Effect: Move inputColorInfo usage to registerInputStream.
To prepare to move `inputColorInfo` from `VFP.Factory.create` to `VFP.registerInputStream`, move all usage of `inputColorInfo` to be *after* `registerInputStream`. To do this, defer creation of `externalShaderProgram` instances, which require `inputColorInfo`. However, we must still initialize `InputSwitcher` and OpenGL ES 3.0 contexts in the VFP create, to create and request the input surface from ExternalTextureManager. PiperOrigin-RevId: 590937251
This commit is contained in:
huangdarwin
Copybara-Service
parent
01578780a6
commit
97e9ed3f7b
@ -45,7 +45,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
+ " RGB channel.";
|
||||
|
||||
private final GlObjectsProvider glObjectsProvider;
|
||||
private final GlShaderProgram shaderProgram;
|
||||
private @MonotonicNonNull GlShaderProgram shaderProgram;
|
||||
// The queue holds all bitmaps with one or more frames pending to be sent downstream.
|
||||
private final Queue<BitmapFrameSequenceInfo> pendingBitmaps;
|
||||
|
||||
@ -63,21 +63,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
* Creates a new instance.
|
||||
*
|
||||
* @param glObjectsProvider The {@link GlObjectsProvider} for using EGL and GLES.
|
||||
* @param shaderProgram The {@link GlShaderProgram} for which this {@code BitmapTextureManager}
|
||||
* will be set as the {@link GlShaderProgram.InputListener}.
|
||||
* @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor} that the
|
||||
* methods of this class run on.
|
||||
*/
|
||||
public BitmapTextureManager(
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
GlShaderProgram shaderProgram,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) {
|
||||
super(videoFrameProcessingTaskExecutor);
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
this.shaderProgram = shaderProgram;
|
||||
pendingBitmaps = new LinkedBlockingQueue<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setSamplingGlShaderProgram(GlShaderProgram samplingGlShaderProgram) {
|
||||
this.shaderProgram = samplingGlShaderProgram;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onReadyToAcceptInputFrame() {
|
||||
videoFrameProcessingTaskExecutor.submit(
|
||||
@ -111,7 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
videoFrameProcessingTaskExecutor.submit(
|
||||
() -> {
|
||||
if (pendingBitmaps.isEmpty()) {
|
||||
shaderProgram.signalEndOfCurrentInputStream();
|
||||
checkNotNull(shaderProgram).signalEndOfCurrentInputStream();
|
||||
DebugTraceUtil.logEvent(
|
||||
DebugTraceUtil.EVENT_BITMAP_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE);
|
||||
} else {
|
||||
@ -166,7 +167,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
}
|
||||
|
||||
downstreamShaderProgramCapacity--;
|
||||
shaderProgram.queueInputFrame(
|
||||
checkNotNull(shaderProgram)
|
||||
.queueInputFrame(
|
||||
glObjectsProvider, checkNotNull(currentGlTextureInfo), currentPresentationTimeUs);
|
||||
DebugTraceUtil.logEvent(
|
||||
DebugTraceUtil.EVENT_VFP_QUEUE_BITMAP,
|
||||
@ -181,7 +183,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
finishedBitmapInfo.bitmap.recycle();
|
||||
if (pendingBitmaps.isEmpty() && currentInputStreamEnded) {
|
||||
// Only signal end of stream after all pending bitmaps are processed.
|
||||
shaderProgram.signalEndOfCurrentInputStream();
|
||||
checkNotNull(shaderProgram).signalEndOfCurrentInputStream();
|
||||
DebugTraceUtil.logEvent(
|
||||
DebugTraceUtil.EVENT_BITMAP_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE);
|
||||
currentInputStreamEnded = false;
|
||||
|
||||
@ -254,26 +254,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
throws VideoFrameProcessingException {
|
||||
// TODO(b/261188041) Add tests to verify the Listener is invoked on the given Executor.
|
||||
|
||||
checkArgument(inputColorInfo.isDataSpaceValid());
|
||||
checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
|
||||
checkArgument(outputColorInfo.isDataSpaceValid());
|
||||
checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
|
||||
if (ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo)) {
|
||||
checkArgument(enableColorTransfers);
|
||||
}
|
||||
|
||||
if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
|
||||
|| ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
|
||||
// OpenGL tone mapping is only implemented for BT2020 to BT709 and HDR to SDR (Gamma 2.2).
|
||||
// Gamma 2.2 is used instead of SMPTE 170M for SDR, despite MediaFormat's
|
||||
// COLOR_TRANSFER_SDR_VIDEO being defined as SMPTE 170M. This is to match
|
||||
// other known tone-mapping behavior within the Android ecosystem.
|
||||
checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
|
||||
checkArgument(outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020);
|
||||
checkArgument(ColorInfo.isTransferHdr(inputColorInfo));
|
||||
checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2);
|
||||
}
|
||||
|
||||
boolean shouldShutdownExecutorService = executorService == null;
|
||||
ExecutorService instanceExecutorService =
|
||||
executorService == null ? Util.newSingleThreadExecutor(THREAD_NAME) : executorService;
|
||||
@ -340,6 +320,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
|
||||
private final List<Effect> activeEffects;
|
||||
private final Object lock;
|
||||
private final boolean enableColorTransfers;
|
||||
private final ColorInfo firstInputColorInfo;
|
||||
private final ColorInfo outputColorInfo;
|
||||
|
||||
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
|
||||
@ -356,6 +338,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
Executor listenerExecutor,
|
||||
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
||||
boolean renderFramesAutomatically,
|
||||
boolean enableColorTransfers,
|
||||
ColorInfo firstInputColorInfo,
|
||||
ColorInfo outputColorInfo) {
|
||||
this.context = context;
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
@ -368,6 +352,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
this.renderFramesAutomatically = renderFramesAutomatically;
|
||||
this.activeEffects = new ArrayList<>();
|
||||
this.lock = new Object();
|
||||
this.enableColorTransfers = enableColorTransfers;
|
||||
this.firstInputColorInfo = firstInputColorInfo;
|
||||
this.outputColorInfo = outputColorInfo;
|
||||
this.finalShaderProgramWrapper = finalShaderProgramWrapper;
|
||||
this.intermediateGlShaderPrograms = new ArrayList<>();
|
||||
@ -477,7 +463,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
|
||||
synchronized (lock) {
|
||||
// An input stream is pending until its effects are configured.
|
||||
InputStreamInfo pendingInputStreamInfo = new InputStreamInfo(inputType, effects, frameInfo);
|
||||
// TODO: b/307952514 - Move inputColorInfo's API from Factory.create into registerInputStream,
|
||||
// and from StreamInfo into FrameInfo.
|
||||
InputStreamInfo pendingInputStreamInfo =
|
||||
new InputStreamInfo(inputType, effects, this.firstInputColorInfo, frameInfo);
|
||||
if (!registeredFirstInputStream) {
|
||||
registeredFirstInputStream = true;
|
||||
inputStreamRegisteredCondition.close();
|
||||
@ -640,11 +629,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
: GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888;
|
||||
EGLContext eglContext =
|
||||
createFocusedEglContextWithFallback(glObjectsProvider, eglDisplay, configAttributes);
|
||||
if ((ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo))
|
||||
&& GlUtil.getContextMajorVersion() != 3) {
|
||||
throw new VideoFrameProcessingException(
|
||||
"OpenGL ES 3.0 context support is required for HDR input or output.");
|
||||
}
|
||||
|
||||
// Not renderFramesAutomatically means outputting to a display surface. HDR display surfaces
|
||||
// require the BT2020 PQ GL extension.
|
||||
@ -689,16 +673,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
textureOutputListener,
|
||||
textureOutputCapacity);
|
||||
|
||||
inputSwitcher.registerInput(inputColorInfo, INPUT_TYPE_SURFACE);
|
||||
if (!ColorInfo.isTransferHdr(inputColorInfo)) {
|
||||
// HDR bitmap input is not supported. Bitmaps are always sRGB/Full range/BT.709.
|
||||
inputSwitcher.registerInput(ColorInfo.SRGB_BT709_FULL, INPUT_TYPE_BITMAP);
|
||||
}
|
||||
if (inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB) {
|
||||
// Image and textureId concatenation not supported.
|
||||
inputSwitcher.registerInput(inputColorInfo, INPUT_TYPE_TEXTURE_ID);
|
||||
}
|
||||
|
||||
return new DefaultVideoFrameProcessor(
|
||||
context,
|
||||
glObjectsProvider,
|
||||
@ -710,6 +684,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
videoFrameProcessorListenerExecutor,
|
||||
finalShaderProgramWrapper,
|
||||
renderFramesAutomatically,
|
||||
enableColorTransfers,
|
||||
/* firstInputColorInfo= */ inputColorInfo,
|
||||
outputColorInfo);
|
||||
}
|
||||
|
||||
@ -826,6 +802,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
*/
|
||||
private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure)
|
||||
throws VideoFrameProcessingException {
|
||||
checkColors(firstInputColorInfo, outputColorInfo, enableColorTransfers);
|
||||
if (forceReconfigure || !activeEffects.equals(inputStreamInfo.effects)) {
|
||||
if (!intermediateGlShaderPrograms.isEmpty()) {
|
||||
for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) {
|
||||
@ -853,7 +830,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
activeEffects.addAll(inputStreamInfo.effects);
|
||||
}
|
||||
|
||||
inputSwitcher.switchToInput(inputStreamInfo.inputType, inputStreamInfo.frameInfo);
|
||||
inputSwitcher.switchToInput(
|
||||
inputStreamInfo.inputType, inputStreamInfo.frameInfo, inputStreamInfo.colorInfo);
|
||||
inputStreamRegisteredCondition.open();
|
||||
listenerExecutor.execute(
|
||||
() ->
|
||||
@ -861,6 +839,42 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
inputStreamInfo.inputType, inputStreamInfo.effects, inputStreamInfo.frameInfo));
|
||||
}
|
||||
|
||||
/** Checks that color configuration is valid for {@link DefaultVideoFrameProcessor}. */
|
||||
private static void checkColors(
|
||||
ColorInfo inputColorInfo, ColorInfo outputColorInfo, boolean enableColorTransfers)
|
||||
throws VideoFrameProcessingException {
|
||||
if ((ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo))) {
|
||||
checkArgument(enableColorTransfers);
|
||||
long glVersion;
|
||||
try {
|
||||
glVersion = GlUtil.getContextMajorVersion();
|
||||
} catch (GlUtil.GlException e) {
|
||||
throw VideoFrameProcessingException.from(e);
|
||||
}
|
||||
if (glVersion != 3) {
|
||||
throw new VideoFrameProcessingException(
|
||||
"OpenGL ES 3.0 context support is required for HDR input or output.");
|
||||
}
|
||||
}
|
||||
|
||||
checkArgument(inputColorInfo.isDataSpaceValid());
|
||||
checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
|
||||
checkArgument(outputColorInfo.isDataSpaceValid());
|
||||
checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
|
||||
|
||||
if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
|
||||
|| ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
|
||||
// OpenGL tone mapping is only implemented for BT2020 to BT709 and HDR to SDR (Gamma 2.2).
|
||||
// Gamma 2.2 is used instead of SMPTE 170M for SDR, despite MediaFormat's
|
||||
// COLOR_TRANSFER_SDR_VIDEO being defined as SMPTE 170M. This is to match
|
||||
// other known tone-mapping behavior within the Android ecosystem.
|
||||
checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
|
||||
checkArgument(outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020);
|
||||
checkArgument(ColorInfo.isTransferHdr(inputColorInfo));
|
||||
checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the {@link GlShaderProgram} instances and destroys the OpenGL context.
|
||||
*
|
||||
@ -925,11 +939,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||
private static final class InputStreamInfo {
|
||||
public final @InputType int inputType;
|
||||
public final List<Effect> effects;
|
||||
public final ColorInfo colorInfo;
|
||||
public final FrameInfo frameInfo;
|
||||
|
||||
public InputStreamInfo(@InputType int inputType, List<Effect> effects, FrameInfo frameInfo) {
|
||||
public InputStreamInfo(
|
||||
@InputType int inputType, List<Effect> effects, ColorInfo colorInfo, FrameInfo frameInfo) {
|
||||
this.inputType = inputType;
|
||||
this.effects = effects;
|
||||
this.colorInfo = colorInfo;
|
||||
this.frameInfo = frameInfo;
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,6 +15,8 @@
|
||||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
import static androidx.media3.common.util.Util.isRunningOnEmulator;
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
@ -30,12 +32,12 @@ import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.Log;
|
||||
import androidx.media3.common.util.Util;
|
||||
import androidx.media3.effect.GlShaderProgram.InputListener;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
* Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
|
||||
@ -58,7 +60,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
private static final long SURFACE_TEXTURE_TIMEOUT_MS = isRunningOnEmulator() ? 10_000 : 500;
|
||||
|
||||
private final GlObjectsProvider glObjectsProvider;
|
||||
private final ExternalShaderProgram externalShaderProgram;
|
||||
private @MonotonicNonNull ExternalShaderProgram externalShaderProgram;
|
||||
private final int externalTexId;
|
||||
private final Surface surface;
|
||||
private final SurfaceTexture surfaceTexture;
|
||||
@ -82,8 +84,6 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
* Creates a new instance. The caller's thread must have a current GL context.
|
||||
*
|
||||
* @param glObjectsProvider The {@link GlObjectsProvider} for using EGL and GLES.
|
||||
* @param externalShaderProgram The {@link ExternalShaderProgram} for which this {@code
|
||||
* ExternalTextureManager} will be set as the {@link InputListener}.
|
||||
* @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor}.
|
||||
* @throws VideoFrameProcessingException If a problem occurs while creating the external texture.
|
||||
*/
|
||||
@ -91,12 +91,10 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
@SuppressWarnings("nullness:method.invocation.invalid")
|
||||
public ExternalTextureManager(
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
ExternalShaderProgram externalShaderProgram,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor)
|
||||
throws VideoFrameProcessingException {
|
||||
super(videoFrameProcessingTaskExecutor);
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
this.externalShaderProgram = externalShaderProgram;
|
||||
try {
|
||||
externalTexId = GlUtil.createExternalTexture();
|
||||
} catch (GlUtil.GlException e) {
|
||||
@ -134,6 +132,17 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
surface = new Surface(surfaceTexture);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*
|
||||
* <p>{@code glShaderProgram} must be an {@link ExternalShaderProgram}.
|
||||
*/
|
||||
@Override
|
||||
public void setSamplingGlShaderProgram(GlShaderProgram samplingGlShaderProgram) {
|
||||
checkState(samplingGlShaderProgram instanceof ExternalShaderProgram);
|
||||
this.externalShaderProgram = (ExternalShaderProgram) samplingGlShaderProgram;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setDefaultBufferSize(int width, int height) {
|
||||
surfaceTexture.setDefaultBufferSize(width, height);
|
||||
@ -161,7 +170,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
if (currentInputStreamEnded && pendingFrames.isEmpty()) {
|
||||
// Reset because there could be further input streams after the current one ends.
|
||||
currentInputStreamEnded = false;
|
||||
externalShaderProgram.signalEndOfCurrentInputStream();
|
||||
checkNotNull(externalShaderProgram).signalEndOfCurrentInputStream();
|
||||
DebugTraceUtil.logEvent(
|
||||
DebugTraceUtil.EVENT_EXTERNAL_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE);
|
||||
cancelForceSignalEndOfStreamTimer();
|
||||
@ -200,7 +209,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
videoFrameProcessingTaskExecutor.submit(
|
||||
() -> {
|
||||
if (pendingFrames.isEmpty() && currentFrame == null) {
|
||||
externalShaderProgram.signalEndOfCurrentInputStream();
|
||||
checkNotNull(externalShaderProgram).signalEndOfCurrentInputStream();
|
||||
DebugTraceUtil.logEvent(
|
||||
DebugTraceUtil.EVENT_EXTERNAL_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE);
|
||||
cancelForceSignalEndOfStreamTimer();
|
||||
@ -294,12 +303,13 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
FrameInfo currentFrame = checkStateNotNull(this.currentFrame);
|
||||
externalShaderProgramInputCapacity.decrementAndGet();
|
||||
surfaceTexture.getTransformMatrix(textureTransformMatrix);
|
||||
externalShaderProgram.setTextureTransformMatrix(textureTransformMatrix);
|
||||
checkNotNull(externalShaderProgram).setTextureTransformMatrix(textureTransformMatrix);
|
||||
long frameTimeNs = surfaceTexture.getTimestamp();
|
||||
long offsetToAddUs = currentFrame.offsetToAddUs;
|
||||
// Correct presentationTimeUs so that GlShaderPrograms don't see the stream offset.
|
||||
long presentationTimeUs = (frameTimeNs / 1000) + offsetToAddUs;
|
||||
externalShaderProgram.queueInputFrame(
|
||||
checkNotNull(externalShaderProgram)
|
||||
.queueInputFrame(
|
||||
glObjectsProvider,
|
||||
new GlTextureInfo(
|
||||
externalTexId,
|
||||
|
||||
@ -28,6 +28,7 @@ import static androidx.media3.common.util.Util.contains;
|
||||
import android.content.Context;
|
||||
import android.util.SparseArray;
|
||||
import android.view.Surface;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.FrameInfo;
|
||||
import androidx.media3.common.GlObjectsProvider;
|
||||
@ -38,6 +39,7 @@ import androidx.media3.common.VideoFrameProcessor;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.util.concurrent.Executor;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
import org.checkerframework.checker.nullness.qual.Nullable;
|
||||
|
||||
/**
|
||||
* A switcher to switch between {@linkplain TextureManager texture managers} of different
|
||||
@ -63,7 +65,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||
Executor errorListenerExecutor,
|
||||
GlShaderProgram.ErrorListener samplingShaderProgramErrorListener,
|
||||
boolean enableColorTransfers) {
|
||||
boolean enableColorTransfers)
|
||||
throws VideoFrameProcessingException {
|
||||
this.context = context;
|
||||
this.outputColorInfo = outputColorInfo;
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
@ -72,30 +75,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
this.samplingShaderProgramErrorListener = samplingShaderProgramErrorListener;
|
||||
this.inputs = new SparseArray<>();
|
||||
this.enableColorTransfers = enableColorTransfers;
|
||||
|
||||
// TODO(b/274109008): Investigate lazy instantiating the texture managers.
|
||||
inputs.put(
|
||||
INPUT_TYPE_SURFACE,
|
||||
new Input(new ExternalTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor)));
|
||||
inputs.put(
|
||||
INPUT_TYPE_BITMAP,
|
||||
new Input(new BitmapTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor)));
|
||||
inputs.put(
|
||||
INPUT_TYPE_TEXTURE_ID,
|
||||
new Input(new TexIdTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers for a new {@link VideoFrameProcessor.InputType input}.
|
||||
*
|
||||
* <p>Can be called multiple times on the same {@link VideoFrameProcessor.InputType inputType},
|
||||
* with the new inputs overwriting the old ones. For example, a new instance of {@link
|
||||
* ExternalTextureManager} is created following each call to this method with {@link
|
||||
* VideoFrameProcessor#INPUT_TYPE_SURFACE}. Effectively, the {@code inputSwitcher} keeps exactly
|
||||
* one {@link TextureManager} per {@linkplain VideoFrameProcessor.InputType input type}.
|
||||
*
|
||||
* <p>Creates an {@link TextureManager} and an appropriate {@linkplain DefaultShaderProgram
|
||||
* sampler} to sample from the input.
|
||||
*
|
||||
* @param inputColorInfo The {@link ColorInfo} for the input frames.
|
||||
* @param inputType The {@linkplain VideoFrameProcessor.InputType type} of the input being
|
||||
* registered.
|
||||
*/
|
||||
public void registerInput(ColorInfo inputColorInfo, @VideoFrameProcessor.InputType int inputType)
|
||||
private DefaultShaderProgram createSamplingShaderProgram(
|
||||
ColorInfo inputColorInfo, @VideoFrameProcessor.InputType int inputType)
|
||||
throws VideoFrameProcessingException {
|
||||
// TODO(b/274109008): Investigate lazy instantiating the texture managers.
|
||||
DefaultShaderProgram samplingShaderProgram;
|
||||
TextureManager textureManager;
|
||||
// TODO(b/274109008): Refactor DefaultShaderProgram to create a class just for sampling.
|
||||
DefaultShaderProgram samplingShaderProgram;
|
||||
switch (inputType) {
|
||||
case INPUT_TYPE_SURFACE:
|
||||
samplingShaderProgram =
|
||||
@ -106,29 +103,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
inputColorInfo,
|
||||
outputColorInfo,
|
||||
enableColorTransfers);
|
||||
samplingShaderProgram.setErrorListener(
|
||||
errorListenerExecutor, samplingShaderProgramErrorListener);
|
||||
textureManager =
|
||||
new ExternalTextureManager(
|
||||
glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
break;
|
||||
case INPUT_TYPE_BITMAP:
|
||||
// HDR bitmap input is not supported. Bitmaps are always sRGB/Full range/BT.709.
|
||||
checkState(!ColorInfo.isTransferHdr(inputColorInfo));
|
||||
ColorInfo bitmapColorInfo = ColorInfo.SRGB_BT709_FULL;
|
||||
samplingShaderProgram =
|
||||
DefaultShaderProgram.createWithInternalSampler(
|
||||
context,
|
||||
/* matrixTransformations= */ ImmutableList.of(),
|
||||
/* rgbMatrices= */ ImmutableList.of(),
|
||||
inputColorInfo,
|
||||
bitmapColorInfo,
|
||||
outputColorInfo,
|
||||
enableColorTransfers,
|
||||
inputType);
|
||||
samplingShaderProgram.setErrorListener(
|
||||
errorListenerExecutor, samplingShaderProgramErrorListener);
|
||||
textureManager =
|
||||
new BitmapTextureManager(
|
||||
glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
break;
|
||||
case INPUT_TYPE_TEXTURE_ID:
|
||||
// Image and textureId concatenation not supported.
|
||||
checkState(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB);
|
||||
samplingShaderProgram =
|
||||
DefaultShaderProgram.createWithInternalSampler(
|
||||
context,
|
||||
@ -138,16 +130,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
outputColorInfo,
|
||||
enableColorTransfers,
|
||||
inputType);
|
||||
samplingShaderProgram.setErrorListener(
|
||||
errorListenerExecutor, samplingShaderProgramErrorListener);
|
||||
textureManager =
|
||||
new TexIdTextureManager(
|
||||
glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
break;
|
||||
default:
|
||||
throw new VideoFrameProcessingException("Unsupported input type " + inputType);
|
||||
}
|
||||
inputs.put(inputType, new Input(textureManager, samplingShaderProgram));
|
||||
samplingShaderProgram.setErrorListener(
|
||||
errorListenerExecutor, samplingShaderProgramErrorListener);
|
||||
return samplingShaderProgram;
|
||||
}
|
||||
|
||||
/** Sets the {@link GlShaderProgram} that {@code InputSwitcher} outputs to. */
|
||||
@ -158,14 +147,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
/**
|
||||
* Switches to a new source of input.
|
||||
*
|
||||
* <p>Must be called after the corresponding {@code newInputType} is {@linkplain #registerInput
|
||||
* registered}.
|
||||
* <p>The first time this is called for each {@link VideoFrameProcessor.InputType}, a sampling
|
||||
* {@link GlShaderProgram} is created for the {@code newInputType}.
|
||||
*
|
||||
* @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to.
|
||||
* @param inputFrameInfo The {@link FrameInfo} associated with the new input.
|
||||
* @param inputColorInfo The {@link ColorInfo} associated with the new input.
|
||||
*/
|
||||
public void switchToInput(
|
||||
@VideoFrameProcessor.InputType int newInputType, FrameInfo inputFrameInfo) {
|
||||
@VideoFrameProcessor.InputType int newInputType,
|
||||
FrameInfo inputFrameInfo,
|
||||
ColorInfo inputColorInfo)
|
||||
throws VideoFrameProcessingException {
|
||||
checkStateNotNull(downstreamShaderProgram);
|
||||
checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType);
|
||||
|
||||
@ -173,10 +166,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
|
||||
Input input = inputs.get(inputType);
|
||||
if (inputType == newInputType) {
|
||||
if (input.getSamplingGlShaderProgram() == null) {
|
||||
// TODO: b/307952514 - When switchToInput is called, and the inputColorInfo doesn't match
|
||||
// the prior inputColorInfo, recreate and reinitialize the input.samplingGlShaderProgram.
|
||||
input.setSamplingGlShaderProgram(
|
||||
createSamplingShaderProgram(inputColorInfo, newInputType));
|
||||
}
|
||||
input.setChainingListener(
|
||||
new GatedChainingListenerWrapper(
|
||||
glObjectsProvider,
|
||||
input.samplingGlShaderProgram,
|
||||
checkNotNull(input.getSamplingGlShaderProgram()),
|
||||
this.downstreamShaderProgram,
|
||||
videoFrameProcessingTaskExecutor));
|
||||
input.setActive(true);
|
||||
@ -217,31 +216,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
*
|
||||
* @return The input {@link Surface}, regardless if the current input is {@linkplain
|
||||
* #switchToInput set} to {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}.
|
||||
* @throws IllegalStateException If {@link VideoFrameProcessor#INPUT_TYPE_SURFACE} is not
|
||||
* {@linkplain #registerInput registered}.
|
||||
*/
|
||||
public Surface getInputSurface() {
|
||||
checkState(contains(inputs, INPUT_TYPE_SURFACE));
|
||||
return inputs.get(INPUT_TYPE_SURFACE).textureManager.getInputSurface();
|
||||
}
|
||||
|
||||
/**
|
||||
* See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}.
|
||||
*
|
||||
* @throws IllegalStateException If {@link VideoFrameProcessor#INPUT_TYPE_SURFACE} is not
|
||||
* {@linkplain #registerInput registered}.
|
||||
*/
|
||||
/** See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. */
|
||||
public void setInputDefaultBufferSize(int width, int height) {
|
||||
checkState(contains(inputs, INPUT_TYPE_SURFACE));
|
||||
inputs.get(INPUT_TYPE_SURFACE).textureManager.setDefaultBufferSize(width, height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the {@link OnInputFrameProcessedListener}.
|
||||
*
|
||||
* @throws IllegalStateException If {@link VideoFrameProcessor#INPUT_TYPE_TEXTURE_ID} is not
|
||||
* {@linkplain #registerInput registered}.
|
||||
*/
|
||||
/** Sets the {@link OnInputFrameProcessedListener}. */
|
||||
public void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener) {
|
||||
checkState(contains(inputs, INPUT_TYPE_TEXTURE_ID));
|
||||
inputs.get(INPUT_TYPE_TEXTURE_ID).textureManager.setOnInputFrameProcessedListener(listener);
|
||||
@ -262,19 +249,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
*/
|
||||
private static final class Input {
|
||||
public final TextureManager textureManager;
|
||||
public final GlShaderProgram samplingGlShaderProgram;
|
||||
|
||||
private @MonotonicNonNull ExternalShaderProgram samplingGlShaderProgram;
|
||||
private @MonotonicNonNull GatedChainingListenerWrapper gatedChainingListenerWrapper;
|
||||
|
||||
public Input(TextureManager textureManager, GlShaderProgram samplingGlShaderProgram) {
|
||||
public Input(TextureManager textureManager) {
|
||||
this.textureManager = textureManager;
|
||||
}
|
||||
|
||||
public void setSamplingGlShaderProgram(ExternalShaderProgram samplingGlShaderProgram) {
|
||||
this.samplingGlShaderProgram = samplingGlShaderProgram;
|
||||
textureManager.setSamplingGlShaderProgram(samplingGlShaderProgram);
|
||||
samplingGlShaderProgram.setInputListener(textureManager);
|
||||
}
|
||||
|
||||
public void setChainingListener(GatedChainingListenerWrapper gatedChainingListenerWrapper) {
|
||||
this.gatedChainingListenerWrapper = gatedChainingListenerWrapper;
|
||||
samplingGlShaderProgram.setOutputListener(gatedChainingListenerWrapper);
|
||||
checkNotNull(samplingGlShaderProgram).setOutputListener(gatedChainingListenerWrapper);
|
||||
}
|
||||
|
||||
public @Nullable ExternalShaderProgram getSamplingGlShaderProgram() {
|
||||
return samplingGlShaderProgram;
|
||||
}
|
||||
|
||||
public void setActive(boolean active) {
|
||||
@ -286,9 +281,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
public void release() throws VideoFrameProcessingException {
|
||||
textureManager.release();
|
||||
if (samplingGlShaderProgram != null) {
|
||||
samplingGlShaderProgram.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a {@link ChainingGlShaderProgramListener}, with the ability to turn off the event
|
||||
|
||||
@ -31,31 +31,28 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
* a {@link GlShaderProgram} for consumption.
|
||||
*/
|
||||
/* package */ final class TexIdTextureManager extends TextureManager {
|
||||
private final FrameConsumptionManager frameConsumptionManager;
|
||||
private @MonotonicNonNull FrameConsumptionManager frameConsumptionManager;
|
||||
|
||||
private @MonotonicNonNull OnInputFrameProcessedListener frameProcessedListener;
|
||||
private @MonotonicNonNull FrameInfo inputFrameInfo;
|
||||
private final GlObjectsProvider glObjectsProvider;
|
||||
|
||||
/**
|
||||
* Creates a new instance.
|
||||
*
|
||||
* @param glObjectsProvider The {@link GlObjectsProvider} for using EGL and GLES.
|
||||
* @param shaderProgram The {@link GlShaderProgram} for which this {@code texIdTextureManager}
|
||||
* will be set as the {@link GlShaderProgram.InputListener}.
|
||||
* @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor}.
|
||||
*/
|
||||
public TexIdTextureManager(
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
GlShaderProgram shaderProgram,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) {
|
||||
super(videoFrameProcessingTaskExecutor);
|
||||
frameConsumptionManager =
|
||||
new FrameConsumptionManager(
|
||||
glObjectsProvider, shaderProgram, videoFrameProcessingTaskExecutor);
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onReadyToAcceptInputFrame() {
|
||||
checkNotNull(frameConsumptionManager);
|
||||
videoFrameProcessingTaskExecutor.submit(frameConsumptionManager::onReadyToAcceptInputFrame);
|
||||
}
|
||||
|
||||
@ -67,6 +64,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
.onInputFrameProcessed(inputTexture.texId, GlUtil.createGlSyncFence()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setSamplingGlShaderProgram(GlShaderProgram samplingGlShaderProgram) {
|
||||
frameConsumptionManager =
|
||||
new FrameConsumptionManager(
|
||||
glObjectsProvider, samplingGlShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void queueInputTexture(int inputTexId, long presentationTimeUs) {
|
||||
FrameInfo frameInfo = checkNotNull(this.inputFrameInfo);
|
||||
@ -80,7 +84,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
frameInfo.width,
|
||||
frameInfo.height);
|
||||
frameConsumptionManager.queueInputFrame(inputTexture, presentationTimeUs);
|
||||
checkNotNull(frameConsumptionManager).queueInputFrame(inputTexture, presentationTimeUs);
|
||||
DebugTraceUtil.logEvent(
|
||||
DebugTraceUtil.EVENT_VFP_QUEUE_TEXTURE,
|
||||
presentationTimeUs,
|
||||
@ -102,14 +106,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
@Override
|
||||
public int getPendingFrameCount() {
|
||||
return frameConsumptionManager.getPendingFrameCount();
|
||||
return checkNotNull(frameConsumptionManager).getPendingFrameCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void signalEndOfCurrentInputStream() {
|
||||
videoFrameProcessingTaskExecutor.submit(
|
||||
() -> {
|
||||
frameConsumptionManager.signalEndOfCurrentStream();
|
||||
checkNotNull(frameConsumptionManager).signalEndOfCurrentStream();
|
||||
DebugTraceUtil.logEvent(
|
||||
DebugTraceUtil.EVENT_TEX_ID_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE);
|
||||
});
|
||||
@ -124,7 +128,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
@Override
|
||||
protected synchronized void flush() {
|
||||
frameConsumptionManager.onFlush();
|
||||
checkNotNull(frameConsumptionManager).onFlush();
|
||||
super.flush();
|
||||
}
|
||||
}
|
||||
|
||||
@ -64,6 +64,16 @@ import androidx.media3.common.util.TimestampIterator;
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the {@link GlShaderProgram} that consumes the {@link TextureManager}'s output.
|
||||
*
|
||||
* <p>Must be called before any method that queues input or {@link
|
||||
* #signalEndOfCurrentInputStream()}.
|
||||
*
|
||||
* <p>This must only be called once.
|
||||
*/
|
||||
public abstract void setSamplingGlShaderProgram(GlShaderProgram samplingGlShaderProgram);
|
||||
|
||||
/**
|
||||
* Provides an input {@link Bitmap} to put into the video frames.
|
||||
*
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user