From 97e9ed3f7b827c8691357902a911e7b4aad91a47 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Thu, 14 Dec 2023 07:45:58 -0800 Subject: [PATCH] Effect: Move inputColorInfo usage to registerInputStream. To prepare to move `inputColorInfo` from `VFP.Factory.create` to `VFP.registerInputStream`, move all usage of `inputColorInfo` to be *after* `registerInputStream`. To do this, defer creation of `externalShaderProgram` instances, which require `inputColorInfo`. However, we must still initialize `InputSwitcher` and OpenGL ES 3.0 contexts in the VFP create, to create and request the input surface from ExternalTextureManager. PiperOrigin-RevId: 590937251 --- .../media3/effect/BitmapTextureManager.java | 20 +-- .../effect/DefaultVideoFrameProcessor.java | 93 ++++++++------ .../media3/effect/ExternalTextureManager.java | 46 ++++--- .../androidx/media3/effect/InputSwitcher.java | 117 +++++++++--------- .../media3/effect/TexIdTextureManager.java | 26 ++-- .../media3/effect/TextureManager.java | 10 ++ 6 files changed, 176 insertions(+), 136 deletions(-) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java index 4b4849ea77..28c60d42d5 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java @@ -45,7 +45,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + " RGB channel."; private final GlObjectsProvider glObjectsProvider; - private final GlShaderProgram shaderProgram; + private @MonotonicNonNull GlShaderProgram shaderProgram; // The queue holds all bitmaps with one or more frames pending to be sent downstream. private final Queue pendingBitmaps; @@ -63,21 +63,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * Creates a new instance. * * @param glObjectsProvider The {@link GlObjectsProvider} for using EGL and GLES. - * @param shaderProgram The {@link GlShaderProgram} for which this {@code BitmapTextureManager} - * will be set as the {@link GlShaderProgram.InputListener}. * @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor} that the * methods of this class run on. */ public BitmapTextureManager( GlObjectsProvider glObjectsProvider, - GlShaderProgram shaderProgram, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) { super(videoFrameProcessingTaskExecutor); this.glObjectsProvider = glObjectsProvider; - this.shaderProgram = shaderProgram; pendingBitmaps = new LinkedBlockingQueue<>(); } + @Override + public void setSamplingGlShaderProgram(GlShaderProgram samplingGlShaderProgram) { + this.shaderProgram = samplingGlShaderProgram; + } + @Override public void onReadyToAcceptInputFrame() { videoFrameProcessingTaskExecutor.submit( @@ -111,7 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; videoFrameProcessingTaskExecutor.submit( () -> { if (pendingBitmaps.isEmpty()) { - shaderProgram.signalEndOfCurrentInputStream(); + checkNotNull(shaderProgram).signalEndOfCurrentInputStream(); DebugTraceUtil.logEvent( DebugTraceUtil.EVENT_BITMAP_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE); } else { @@ -166,8 +167,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } downstreamShaderProgramCapacity--; - shaderProgram.queueInputFrame( - glObjectsProvider, checkNotNull(currentGlTextureInfo), currentPresentationTimeUs); + checkNotNull(shaderProgram) + .queueInputFrame( + glObjectsProvider, checkNotNull(currentGlTextureInfo), currentPresentationTimeUs); DebugTraceUtil.logEvent( DebugTraceUtil.EVENT_VFP_QUEUE_BITMAP, currentPresentationTimeUs, @@ -181,7 +183,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; finishedBitmapInfo.bitmap.recycle(); if (pendingBitmaps.isEmpty() && currentInputStreamEnded) { // Only signal end of stream after all pending bitmaps are processed. - shaderProgram.signalEndOfCurrentInputStream(); + checkNotNull(shaderProgram).signalEndOfCurrentInputStream(); DebugTraceUtil.logEvent( DebugTraceUtil.EVENT_BITMAP_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE); currentInputStreamEnded = false; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index e445877310..1a6e572f80 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -254,26 +254,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { throws VideoFrameProcessingException { // TODO(b/261188041) Add tests to verify the Listener is invoked on the given Executor. - checkArgument(inputColorInfo.isDataSpaceValid()); - checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR); - checkArgument(outputColorInfo.isDataSpaceValid()); - checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR); - if (ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo)) { - checkArgument(enableColorTransfers); - } - - if (inputColorInfo.colorSpace != outputColorInfo.colorSpace - || ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) { - // OpenGL tone mapping is only implemented for BT2020 to BT709 and HDR to SDR (Gamma 2.2). - // Gamma 2.2 is used instead of SMPTE 170M for SDR, despite MediaFormat's - // COLOR_TRANSFER_SDR_VIDEO being defined as SMPTE 170M. This is to match - // other known tone-mapping behavior within the Android ecosystem. - checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020); - checkArgument(outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020); - checkArgument(ColorInfo.isTransferHdr(inputColorInfo)); - checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2); - } - boolean shouldShutdownExecutorService = executorService == null; ExecutorService instanceExecutorService = executorService == null ? Util.newSingleThreadExecutor(THREAD_NAME) : executorService; @@ -340,6 +320,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final List activeEffects; private final Object lock; + private final boolean enableColorTransfers; + private final ColorInfo firstInputColorInfo; private final ColorInfo outputColorInfo; private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; @@ -356,6 +338,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { Executor listenerExecutor, FinalShaderProgramWrapper finalShaderProgramWrapper, boolean renderFramesAutomatically, + boolean enableColorTransfers, + ColorInfo firstInputColorInfo, ColorInfo outputColorInfo) { this.context = context; this.glObjectsProvider = glObjectsProvider; @@ -368,6 +352,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { this.renderFramesAutomatically = renderFramesAutomatically; this.activeEffects = new ArrayList<>(); this.lock = new Object(); + this.enableColorTransfers = enableColorTransfers; + this.firstInputColorInfo = firstInputColorInfo; this.outputColorInfo = outputColorInfo; this.finalShaderProgramWrapper = finalShaderProgramWrapper; this.intermediateGlShaderPrograms = new ArrayList<>(); @@ -477,7 +463,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { synchronized (lock) { // An input stream is pending until its effects are configured. - InputStreamInfo pendingInputStreamInfo = new InputStreamInfo(inputType, effects, frameInfo); + // TODO: b/307952514 - Move inputColorInfo's API from Factory.create into registerInputStream, + // and from StreamInfo into FrameInfo. + InputStreamInfo pendingInputStreamInfo = + new InputStreamInfo(inputType, effects, this.firstInputColorInfo, frameInfo); if (!registeredFirstInputStream) { registeredFirstInputStream = true; inputStreamRegisteredCondition.close(); @@ -640,11 +629,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888; EGLContext eglContext = createFocusedEglContextWithFallback(glObjectsProvider, eglDisplay, configAttributes); - if ((ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo)) - && GlUtil.getContextMajorVersion() != 3) { - throw new VideoFrameProcessingException( - "OpenGL ES 3.0 context support is required for HDR input or output."); - } // Not renderFramesAutomatically means outputting to a display surface. HDR display surfaces // require the BT2020 PQ GL extension. @@ -689,16 +673,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { textureOutputListener, textureOutputCapacity); - inputSwitcher.registerInput(inputColorInfo, INPUT_TYPE_SURFACE); - if (!ColorInfo.isTransferHdr(inputColorInfo)) { - // HDR bitmap input is not supported. Bitmaps are always sRGB/Full range/BT.709. - inputSwitcher.registerInput(ColorInfo.SRGB_BT709_FULL, INPUT_TYPE_BITMAP); - } - if (inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB) { - // Image and textureId concatenation not supported. - inputSwitcher.registerInput(inputColorInfo, INPUT_TYPE_TEXTURE_ID); - } - return new DefaultVideoFrameProcessor( context, glObjectsProvider, @@ -710,6 +684,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { videoFrameProcessorListenerExecutor, finalShaderProgramWrapper, renderFramesAutomatically, + enableColorTransfers, + /* firstInputColorInfo= */ inputColorInfo, outputColorInfo); } @@ -826,6 +802,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { */ private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure) throws VideoFrameProcessingException { + checkColors(firstInputColorInfo, outputColorInfo, enableColorTransfers); if (forceReconfigure || !activeEffects.equals(inputStreamInfo.effects)) { if (!intermediateGlShaderPrograms.isEmpty()) { for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) { @@ -853,7 +830,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { activeEffects.addAll(inputStreamInfo.effects); } - inputSwitcher.switchToInput(inputStreamInfo.inputType, inputStreamInfo.frameInfo); + inputSwitcher.switchToInput( + inputStreamInfo.inputType, inputStreamInfo.frameInfo, inputStreamInfo.colorInfo); inputStreamRegisteredCondition.open(); listenerExecutor.execute( () -> @@ -861,6 +839,42 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { inputStreamInfo.inputType, inputStreamInfo.effects, inputStreamInfo.frameInfo)); } + /** Checks that color configuration is valid for {@link DefaultVideoFrameProcessor}. */ + private static void checkColors( + ColorInfo inputColorInfo, ColorInfo outputColorInfo, boolean enableColorTransfers) + throws VideoFrameProcessingException { + if ((ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo))) { + checkArgument(enableColorTransfers); + long glVersion; + try { + glVersion = GlUtil.getContextMajorVersion(); + } catch (GlUtil.GlException e) { + throw VideoFrameProcessingException.from(e); + } + if (glVersion != 3) { + throw new VideoFrameProcessingException( + "OpenGL ES 3.0 context support is required for HDR input or output."); + } + } + + checkArgument(inputColorInfo.isDataSpaceValid()); + checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR); + checkArgument(outputColorInfo.isDataSpaceValid()); + checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR); + + if (inputColorInfo.colorSpace != outputColorInfo.colorSpace + || ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) { + // OpenGL tone mapping is only implemented for BT2020 to BT709 and HDR to SDR (Gamma 2.2). + // Gamma 2.2 is used instead of SMPTE 170M for SDR, despite MediaFormat's + // COLOR_TRANSFER_SDR_VIDEO being defined as SMPTE 170M. This is to match + // other known tone-mapping behavior within the Android ecosystem. + checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020); + checkArgument(outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020); + checkArgument(ColorInfo.isTransferHdr(inputColorInfo)); + checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2); + } + } + /** * Releases the {@link GlShaderProgram} instances and destroys the OpenGL context. * @@ -925,11 +939,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private static final class InputStreamInfo { public final @InputType int inputType; public final List effects; + public final ColorInfo colorInfo; public final FrameInfo frameInfo; - public InputStreamInfo(@InputType int inputType, List effects, FrameInfo frameInfo) { + public InputStreamInfo( + @InputType int inputType, List effects, ColorInfo colorInfo, FrameInfo frameInfo) { this.inputType = inputType; this.effects = effects; + this.colorInfo = colorInfo; this.frameInfo = frameInfo; } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index 2625996f88..cf7d98fce2 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -15,6 +15,8 @@ */ package androidx.media3.effect; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.common.util.Util.isRunningOnEmulator; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -30,12 +32,12 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; -import androidx.media3.effect.GlShaderProgram.InputListener; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Forwards externally produced frames that become available via a {@link SurfaceTexture} to an @@ -58,7 +60,7 @@ import java.util.concurrent.atomic.AtomicInteger; private static final long SURFACE_TEXTURE_TIMEOUT_MS = isRunningOnEmulator() ? 10_000 : 500; private final GlObjectsProvider glObjectsProvider; - private final ExternalShaderProgram externalShaderProgram; + private @MonotonicNonNull ExternalShaderProgram externalShaderProgram; private final int externalTexId; private final Surface surface; private final SurfaceTexture surfaceTexture; @@ -82,8 +84,6 @@ import java.util.concurrent.atomic.AtomicInteger; * Creates a new instance. The caller's thread must have a current GL context. * * @param glObjectsProvider The {@link GlObjectsProvider} for using EGL and GLES. - * @param externalShaderProgram The {@link ExternalShaderProgram} for which this {@code - * ExternalTextureManager} will be set as the {@link InputListener}. * @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor}. * @throws VideoFrameProcessingException If a problem occurs while creating the external texture. */ @@ -91,12 +91,10 @@ import java.util.concurrent.atomic.AtomicInteger; @SuppressWarnings("nullness:method.invocation.invalid") public ExternalTextureManager( GlObjectsProvider glObjectsProvider, - ExternalShaderProgram externalShaderProgram, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) throws VideoFrameProcessingException { super(videoFrameProcessingTaskExecutor); this.glObjectsProvider = glObjectsProvider; - this.externalShaderProgram = externalShaderProgram; try { externalTexId = GlUtil.createExternalTexture(); } catch (GlUtil.GlException e) { @@ -134,6 +132,17 @@ import java.util.concurrent.atomic.AtomicInteger; surface = new Surface(surfaceTexture); } + /** + * {@inheritDoc} + * + *

{@code glShaderProgram} must be an {@link ExternalShaderProgram}. + */ + @Override + public void setSamplingGlShaderProgram(GlShaderProgram samplingGlShaderProgram) { + checkState(samplingGlShaderProgram instanceof ExternalShaderProgram); + this.externalShaderProgram = (ExternalShaderProgram) samplingGlShaderProgram; + } + @Override public void setDefaultBufferSize(int width, int height) { surfaceTexture.setDefaultBufferSize(width, height); @@ -161,7 +170,7 @@ import java.util.concurrent.atomic.AtomicInteger; if (currentInputStreamEnded && pendingFrames.isEmpty()) { // Reset because there could be further input streams after the current one ends. currentInputStreamEnded = false; - externalShaderProgram.signalEndOfCurrentInputStream(); + checkNotNull(externalShaderProgram).signalEndOfCurrentInputStream(); DebugTraceUtil.logEvent( DebugTraceUtil.EVENT_EXTERNAL_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE); cancelForceSignalEndOfStreamTimer(); @@ -200,7 +209,7 @@ import java.util.concurrent.atomic.AtomicInteger; videoFrameProcessingTaskExecutor.submit( () -> { if (pendingFrames.isEmpty() && currentFrame == null) { - externalShaderProgram.signalEndOfCurrentInputStream(); + checkNotNull(externalShaderProgram).signalEndOfCurrentInputStream(); DebugTraceUtil.logEvent( DebugTraceUtil.EVENT_EXTERNAL_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE); cancelForceSignalEndOfStreamTimer(); @@ -294,20 +303,21 @@ import java.util.concurrent.atomic.AtomicInteger; FrameInfo currentFrame = checkStateNotNull(this.currentFrame); externalShaderProgramInputCapacity.decrementAndGet(); surfaceTexture.getTransformMatrix(textureTransformMatrix); - externalShaderProgram.setTextureTransformMatrix(textureTransformMatrix); + checkNotNull(externalShaderProgram).setTextureTransformMatrix(textureTransformMatrix); long frameTimeNs = surfaceTexture.getTimestamp(); long offsetToAddUs = currentFrame.offsetToAddUs; // Correct presentationTimeUs so that GlShaderPrograms don't see the stream offset. long presentationTimeUs = (frameTimeNs / 1000) + offsetToAddUs; - externalShaderProgram.queueInputFrame( - glObjectsProvider, - new GlTextureInfo( - externalTexId, - /* fboId= */ C.INDEX_UNSET, - /* rboId= */ C.INDEX_UNSET, - currentFrame.width, - currentFrame.height), - presentationTimeUs); + checkNotNull(externalShaderProgram) + .queueInputFrame( + glObjectsProvider, + new GlTextureInfo( + externalTexId, + /* fboId= */ C.INDEX_UNSET, + /* rboId= */ C.INDEX_UNSET, + currentFrame.width, + currentFrame.height), + presentationTimeUs); checkStateNotNull(pendingFrames.remove()); DebugTraceUtil.logEvent(DebugTraceUtil.EVENT_VFP_QUEUE_FRAME, presentationTimeUs); // If the queued frame is the last frame, end of stream will be signaled onInputFrameProcessed. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index 9bf4c064b4..7ed39555eb 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -28,6 +28,7 @@ import static androidx.media3.common.util.Util.contains; import android.content.Context; import android.util.SparseArray; import android.view.Surface; +import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.FrameInfo; import androidx.media3.common.GlObjectsProvider; @@ -38,6 +39,7 @@ import androidx.media3.common.VideoFrameProcessor; import com.google.common.collect.ImmutableList; import java.util.concurrent.Executor; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.Nullable; /** * A switcher to switch between {@linkplain TextureManager texture managers} of different @@ -63,7 +65,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, Executor errorListenerExecutor, GlShaderProgram.ErrorListener samplingShaderProgramErrorListener, - boolean enableColorTransfers) { + boolean enableColorTransfers) + throws VideoFrameProcessingException { this.context = context; this.outputColorInfo = outputColorInfo; this.glObjectsProvider = glObjectsProvider; @@ -72,30 +75,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.samplingShaderProgramErrorListener = samplingShaderProgramErrorListener; this.inputs = new SparseArray<>(); this.enableColorTransfers = enableColorTransfers; + + // TODO(b/274109008): Investigate lazy instantiating the texture managers. + inputs.put( + INPUT_TYPE_SURFACE, + new Input(new ExternalTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor))); + inputs.put( + INPUT_TYPE_BITMAP, + new Input(new BitmapTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor))); + inputs.put( + INPUT_TYPE_TEXTURE_ID, + new Input(new TexIdTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor))); } - /** - * Registers for a new {@link VideoFrameProcessor.InputType input}. - * - *

Can be called multiple times on the same {@link VideoFrameProcessor.InputType inputType}, - * with the new inputs overwriting the old ones. For example, a new instance of {@link - * ExternalTextureManager} is created following each call to this method with {@link - * VideoFrameProcessor#INPUT_TYPE_SURFACE}. Effectively, the {@code inputSwitcher} keeps exactly - * one {@link TextureManager} per {@linkplain VideoFrameProcessor.InputType input type}. - * - *

Creates an {@link TextureManager} and an appropriate {@linkplain DefaultShaderProgram - * sampler} to sample from the input. - * - * @param inputColorInfo The {@link ColorInfo} for the input frames. - * @param inputType The {@linkplain VideoFrameProcessor.InputType type} of the input being - * registered. - */ - public void registerInput(ColorInfo inputColorInfo, @VideoFrameProcessor.InputType int inputType) + private DefaultShaderProgram createSamplingShaderProgram( + ColorInfo inputColorInfo, @VideoFrameProcessor.InputType int inputType) throws VideoFrameProcessingException { - // TODO(b/274109008): Investigate lazy instantiating the texture managers. - DefaultShaderProgram samplingShaderProgram; - TextureManager textureManager; // TODO(b/274109008): Refactor DefaultShaderProgram to create a class just for sampling. + DefaultShaderProgram samplingShaderProgram; switch (inputType) { case INPUT_TYPE_SURFACE: samplingShaderProgram = @@ -106,29 +103,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputColorInfo, outputColorInfo, enableColorTransfers); - samplingShaderProgram.setErrorListener( - errorListenerExecutor, samplingShaderProgramErrorListener); - textureManager = - new ExternalTextureManager( - glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor); break; case INPUT_TYPE_BITMAP: + // HDR bitmap input is not supported. Bitmaps are always sRGB/Full range/BT.709. + checkState(!ColorInfo.isTransferHdr(inputColorInfo)); + ColorInfo bitmapColorInfo = ColorInfo.SRGB_BT709_FULL; samplingShaderProgram = DefaultShaderProgram.createWithInternalSampler( context, /* matrixTransformations= */ ImmutableList.of(), /* rgbMatrices= */ ImmutableList.of(), - inputColorInfo, + bitmapColorInfo, outputColorInfo, enableColorTransfers, inputType); - samplingShaderProgram.setErrorListener( - errorListenerExecutor, samplingShaderProgramErrorListener); - textureManager = - new BitmapTextureManager( - glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor); break; case INPUT_TYPE_TEXTURE_ID: + // Image and textureId concatenation not supported. + checkState(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB); samplingShaderProgram = DefaultShaderProgram.createWithInternalSampler( context, @@ -138,16 +130,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputColorInfo, enableColorTransfers, inputType); - samplingShaderProgram.setErrorListener( - errorListenerExecutor, samplingShaderProgramErrorListener); - textureManager = - new TexIdTextureManager( - glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor); break; default: throw new VideoFrameProcessingException("Unsupported input type " + inputType); } - inputs.put(inputType, new Input(textureManager, samplingShaderProgram)); + samplingShaderProgram.setErrorListener( + errorListenerExecutor, samplingShaderProgramErrorListener); + return samplingShaderProgram; } /** Sets the {@link GlShaderProgram} that {@code InputSwitcher} outputs to. */ @@ -158,14 +147,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Switches to a new source of input. * - *

Must be called after the corresponding {@code newInputType} is {@linkplain #registerInput - * registered}. + *

The first time this is called for each {@link VideoFrameProcessor.InputType}, a sampling + * {@link GlShaderProgram} is created for the {@code newInputType}. * * @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to. * @param inputFrameInfo The {@link FrameInfo} associated with the new input. + * @param inputColorInfo The {@link ColorInfo} associated with the new input. */ public void switchToInput( - @VideoFrameProcessor.InputType int newInputType, FrameInfo inputFrameInfo) { + @VideoFrameProcessor.InputType int newInputType, + FrameInfo inputFrameInfo, + ColorInfo inputColorInfo) + throws VideoFrameProcessingException { checkStateNotNull(downstreamShaderProgram); checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType); @@ -173,10 +166,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); Input input = inputs.get(inputType); if (inputType == newInputType) { + if (input.getSamplingGlShaderProgram() == null) { + // TODO: b/307952514 - When switchToInput is called, and the inputColorInfo doesn't match + // the prior inputColorInfo, recreate and reinitialize the input.samplingGlShaderProgram. + input.setSamplingGlShaderProgram( + createSamplingShaderProgram(inputColorInfo, newInputType)); + } input.setChainingListener( new GatedChainingListenerWrapper( glObjectsProvider, - input.samplingGlShaderProgram, + checkNotNull(input.getSamplingGlShaderProgram()), this.downstreamShaderProgram, videoFrameProcessingTaskExecutor)); input.setActive(true); @@ -217,31 +216,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * * @return The input {@link Surface}, regardless if the current input is {@linkplain * #switchToInput set} to {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}. - * @throws IllegalStateException If {@link VideoFrameProcessor#INPUT_TYPE_SURFACE} is not - * {@linkplain #registerInput registered}. */ public Surface getInputSurface() { checkState(contains(inputs, INPUT_TYPE_SURFACE)); return inputs.get(INPUT_TYPE_SURFACE).textureManager.getInputSurface(); } - /** - * See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. - * - * @throws IllegalStateException If {@link VideoFrameProcessor#INPUT_TYPE_SURFACE} is not - * {@linkplain #registerInput registered}. - */ + /** See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. */ public void setInputDefaultBufferSize(int width, int height) { checkState(contains(inputs, INPUT_TYPE_SURFACE)); inputs.get(INPUT_TYPE_SURFACE).textureManager.setDefaultBufferSize(width, height); } - /** - * Sets the {@link OnInputFrameProcessedListener}. - * - * @throws IllegalStateException If {@link VideoFrameProcessor#INPUT_TYPE_TEXTURE_ID} is not - * {@linkplain #registerInput registered}. - */ + /** Sets the {@link OnInputFrameProcessedListener}. */ public void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener) { checkState(contains(inputs, INPUT_TYPE_TEXTURE_ID)); inputs.get(INPUT_TYPE_TEXTURE_ID).textureManager.setOnInputFrameProcessedListener(listener); @@ -262,19 +249,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; */ private static final class Input { public final TextureManager textureManager; - public final GlShaderProgram samplingGlShaderProgram; + private @MonotonicNonNull ExternalShaderProgram samplingGlShaderProgram; private @MonotonicNonNull GatedChainingListenerWrapper gatedChainingListenerWrapper; - public Input(TextureManager textureManager, GlShaderProgram samplingGlShaderProgram) { + public Input(TextureManager textureManager) { this.textureManager = textureManager; + } + + public void setSamplingGlShaderProgram(ExternalShaderProgram samplingGlShaderProgram) { this.samplingGlShaderProgram = samplingGlShaderProgram; + textureManager.setSamplingGlShaderProgram(samplingGlShaderProgram); samplingGlShaderProgram.setInputListener(textureManager); } public void setChainingListener(GatedChainingListenerWrapper gatedChainingListenerWrapper) { this.gatedChainingListenerWrapper = gatedChainingListenerWrapper; - samplingGlShaderProgram.setOutputListener(gatedChainingListenerWrapper); + checkNotNull(samplingGlShaderProgram).setOutputListener(gatedChainingListenerWrapper); + } + + public @Nullable ExternalShaderProgram getSamplingGlShaderProgram() { + return samplingGlShaderProgram; } public void setActive(boolean active) { @@ -286,7 +281,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public void release() throws VideoFrameProcessingException { textureManager.release(); - samplingGlShaderProgram.release(); + if (samplingGlShaderProgram != null) { + samplingGlShaderProgram.release(); + } } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java index f3b926a3a5..1c04d37b32 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java @@ -31,31 +31,28 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * a {@link GlShaderProgram} for consumption. */ /* package */ final class TexIdTextureManager extends TextureManager { - private final FrameConsumptionManager frameConsumptionManager; + private @MonotonicNonNull FrameConsumptionManager frameConsumptionManager; private @MonotonicNonNull OnInputFrameProcessedListener frameProcessedListener; private @MonotonicNonNull FrameInfo inputFrameInfo; + private final GlObjectsProvider glObjectsProvider; /** * Creates a new instance. * * @param glObjectsProvider The {@link GlObjectsProvider} for using EGL and GLES. - * @param shaderProgram The {@link GlShaderProgram} for which this {@code texIdTextureManager} - * will be set as the {@link GlShaderProgram.InputListener}. * @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor}. */ public TexIdTextureManager( GlObjectsProvider glObjectsProvider, - GlShaderProgram shaderProgram, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) { super(videoFrameProcessingTaskExecutor); - frameConsumptionManager = - new FrameConsumptionManager( - glObjectsProvider, shaderProgram, videoFrameProcessingTaskExecutor); + this.glObjectsProvider = glObjectsProvider; } @Override public void onReadyToAcceptInputFrame() { + checkNotNull(frameConsumptionManager); videoFrameProcessingTaskExecutor.submit(frameConsumptionManager::onReadyToAcceptInputFrame); } @@ -67,6 +64,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; .onInputFrameProcessed(inputTexture.texId, GlUtil.createGlSyncFence())); } + @Override + public void setSamplingGlShaderProgram(GlShaderProgram samplingGlShaderProgram) { + frameConsumptionManager = + new FrameConsumptionManager( + glObjectsProvider, samplingGlShaderProgram, videoFrameProcessingTaskExecutor); + } + @Override public void queueInputTexture(int inputTexId, long presentationTimeUs) { FrameInfo frameInfo = checkNotNull(this.inputFrameInfo); @@ -80,7 +84,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* rboId= */ C.INDEX_UNSET, frameInfo.width, frameInfo.height); - frameConsumptionManager.queueInputFrame(inputTexture, presentationTimeUs); + checkNotNull(frameConsumptionManager).queueInputFrame(inputTexture, presentationTimeUs); DebugTraceUtil.logEvent( DebugTraceUtil.EVENT_VFP_QUEUE_TEXTURE, presentationTimeUs, @@ -102,14 +106,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public int getPendingFrameCount() { - return frameConsumptionManager.getPendingFrameCount(); + return checkNotNull(frameConsumptionManager).getPendingFrameCount(); } @Override public void signalEndOfCurrentInputStream() { videoFrameProcessingTaskExecutor.submit( () -> { - frameConsumptionManager.signalEndOfCurrentStream(); + checkNotNull(frameConsumptionManager).signalEndOfCurrentStream(); DebugTraceUtil.logEvent( DebugTraceUtil.EVENT_TEX_ID_TEXTURE_MANAGER_SIGNAL_EOS, C.TIME_END_OF_SOURCE); }); @@ -124,7 +128,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override protected synchronized void flush() { - frameConsumptionManager.onFlush(); + checkNotNull(frameConsumptionManager).onFlush(); super.flush(); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java index f2a0c95152..47a61f9c4e 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java @@ -64,6 +64,16 @@ import androidx.media3.common.util.TimestampIterator; throw new UnsupportedOperationException(); } + /** + * Sets the {@link GlShaderProgram} that consumes the {@link TextureManager}'s output. + * + *

Must be called before any method that queues input or {@link + * #signalEndOfCurrentInputStream()}. + * + *

This must only be called once. + */ + public abstract void setSamplingGlShaderProgram(GlShaderProgram samplingGlShaderProgram); + /** * Provides an input {@link Bitmap} to put into the video frames. *