diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java index d5e4f7c96e..06512ca0e9 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java @@ -356,6 +356,16 @@ public final class GlEffectsFrameProcessorPixelTest { GlEffectsFrameProcessor.create( context, new FrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + outputImageReader = + ImageReader.newInstance( + width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); + checkNotNull(glEffectsFrameProcessor) + .setOutputSurfaceInfo( + new SurfaceInfo(outputImageReader.getSurface(), width, height)); + } + @Override public void onFrameProcessingError(FrameProcessingException exception) { frameProcessingException.set(exception); @@ -368,16 +378,6 @@ public final class GlEffectsFrameProcessorPixelTest { }, /* streamOffsetUs= */ 0L, effects, - /* outputSurfaceProvider= */ (requestedWidth, requestedHeight) -> { - outputImageReader = - ImageReader.newInstance( - requestedWidth, - requestedHeight, - PixelFormat.RGBA_8888, - /* maxImages= */ 1); - return new SurfaceInfo( - outputImageReader.getSurface(), requestedWidth, requestedHeight); - }, Transformer.DebugViewProvider.NONE, /* enableExperimentalHdrEditing= */ false)); glEffectsFrameProcessor.setInputFrameInfo( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java index 996fce157b..a927c5ce3c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java @@ -34,6 +34,7 @@ import androidx.annotation.WorkerThread; import androidx.media3.common.C; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; +import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -57,7 +58,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final ImmutableList matrixTransformations; private final EGLDisplay eglDisplay; private final EGLContext eglContext; - private final SurfaceInfo.Provider outputSurfaceProvider; private final long streamOffsetUs; private final Transformer.DebugViewProvider debugViewProvider; private final FrameProcessor.Listener frameProcessorListener; @@ -66,17 +66,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private int inputWidth; private int inputHeight; @Nullable private MatrixTransformationProcessor matrixTransformationProcessor; - @Nullable private SurfaceInfo outputSurfaceInfo; - @Nullable private EGLSurface outputEglSurface; @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper; private @MonotonicNonNull Listener listener; + private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation; + + @GuardedBy("this") + @Nullable + private SurfaceInfo outputSurfaceInfo; + + @GuardedBy("this") + @Nullable + private EGLSurface outputEglSurface; public FinalMatrixTransformationProcessorWrapper( Context context, EGLDisplay eglDisplay, EGLContext eglContext, ImmutableList matrixTransformations, - SurfaceInfo.Provider outputSurfaceProvider, long streamOffsetUs, FrameProcessor.Listener frameProcessorListener, Transformer.DebugViewProvider debugViewProvider, @@ -85,7 +91,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.matrixTransformations = matrixTransformations; this.eglDisplay = eglDisplay; this.eglContext = eglContext; - this.outputSurfaceProvider = outputSurfaceProvider; this.streamOffsetUs = streamOffsetUs; this.debugViewProvider = debugViewProvider; this.frameProcessorListener = frameProcessorListener; @@ -107,28 +112,30 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { try { - if (!ensureConfigured(inputTexture.width, inputTexture.height)) { - return false; + synchronized (this) { + if (!ensureConfigured(inputTexture.width, inputTexture.height)) { + return false; + } + + EGLSurface outputEglSurface = this.outputEglSurface; + SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; + MatrixTransformationProcessor matrixTransformationProcessor = + this.matrixTransformationProcessor; + + GlUtil.focusEglSurface( + eglDisplay, + eglContext, + outputEglSurface, + outputSurfaceInfo.width, + outputSurfaceInfo.height); + GlUtil.clearOutputFrame(); + matrixTransformationProcessor.drawFrame(inputTexture.texId, presentationTimeUs); + EGLExt.eglPresentationTimeANDROID( + eglDisplay, + outputEglSurface, + /* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000); + EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); } - - EGLSurface outputEglSurface = this.outputEglSurface; - SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; - MatrixTransformationProcessor matrixTransformationProcessor = - this.matrixTransformationProcessor; - - GlUtil.focusEglSurface( - eglDisplay, - eglContext, - outputEglSurface, - outputSurfaceInfo.width, - outputSurfaceInfo.height); - GlUtil.clearOutputFrame(); - matrixTransformationProcessor.drawFrame(inputTexture.texId, presentationTimeUs); - EGLExt.eglPresentationTimeANDROID( - eglDisplay, - outputEglSurface, - /* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000); - EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); } catch (FrameProcessingException | GlUtil.GlException e) { frameProcessorListener.onFrameProcessingError( FrameProcessingException.from(e, presentationTimeUs)); @@ -156,24 +163,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @EnsuresNonNullIf( expression = {"outputSurfaceInfo", "outputEglSurface", "matrixTransformationProcessor"}, result = true) - private boolean ensureConfigured(int inputWidth, int inputHeight) + private synchronized boolean ensureConfigured(int inputWidth, int inputHeight) throws FrameProcessingException, GlUtil.GlException { - if (inputWidth == this.inputWidth - && inputHeight == this.inputHeight - && outputSurfaceInfo != null - && outputEglSurface != null - && matrixTransformationProcessor != null) { - return true; + + if (this.inputWidth != inputWidth + || this.inputHeight != inputHeight + || this.outputSizeBeforeSurfaceTransformation == null) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + Size outputSizeBeforeSurfaceTransformation = + MatrixUtils.configureAndGetOutputSize(inputWidth, inputHeight, matrixTransformations); + if (!Util.areEqual( + this.outputSizeBeforeSurfaceTransformation, outputSizeBeforeSurfaceTransformation)) { + this.outputSizeBeforeSurfaceTransformation = outputSizeBeforeSurfaceTransformation; + frameProcessorListener.onOutputSizeChanged( + outputSizeBeforeSurfaceTransformation.getWidth(), + outputSizeBeforeSurfaceTransformation.getHeight()); + } } - this.inputWidth = inputWidth; - this.inputHeight = inputHeight; - Size requestedOutputSize = - MatrixUtils.configureAndGetOutputSize(inputWidth, inputHeight, matrixTransformations); - @Nullable - SurfaceInfo outputSurfaceInfo = - outputSurfaceProvider.getSurfaceInfo( - requestedOutputSize.getWidth(), requestedOutputSize.getHeight()); if (outputSurfaceInfo == null) { if (matrixTransformationProcessor != null) { matrixTransformationProcessor.release(); @@ -182,40 +190,44 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputEglSurface = null; return false; } - if (outputSurfaceInfo == this.outputSurfaceInfo - && outputEglSurface != null - && matrixTransformationProcessor != null) { - return true; + + SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; + @Nullable EGLSurface outputEglSurface = this.outputEglSurface; + if (outputEglSurface == null) { // This means that outputSurfaceInfo changed. + if (enableExperimentalHdrEditing) { + // TODO(b/227624622): Don't assume BT.2020 PQ input/output. + outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface); + } else { + outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface); + } + + @Nullable + SurfaceView debugSurfaceView = + debugViewProvider.getDebugPreviewSurfaceView( + outputSurfaceInfo.width, outputSurfaceInfo.height); + if (debugSurfaceView != null) { + debugSurfaceViewWrapper = + new SurfaceViewWrapper( + eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView); + } + if (matrixTransformationProcessor != null) { + matrixTransformationProcessor.release(); + matrixTransformationProcessor = null; + } } - EGLSurface outputEglSurface; - if (enableExperimentalHdrEditing) { - // TODO(b/227624622): Don't assume BT.2020 PQ input/output. - outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface); - } else { - outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface); + if (matrixTransformationProcessor == null) { + matrixTransformationProcessor = + createMatrixTransformationProcessorForOutputSurface(outputSurfaceInfo); } - @Nullable - SurfaceView debugSurfaceView = - debugViewProvider.getDebugPreviewSurfaceView( - outputSurfaceInfo.width, outputSurfaceInfo.height); - if (debugSurfaceView != null) { - debugSurfaceViewWrapper = - new SurfaceViewWrapper( - eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView); - } - - matrixTransformationProcessor = - createMatrixTransformationProcessorForOutputSurface(requestedOutputSize, outputSurfaceInfo); - this.outputSurfaceInfo = outputSurfaceInfo; this.outputEglSurface = outputEglSurface; return true; } private MatrixTransformationProcessor createMatrixTransformationProcessorForOutputSurface( - Size requestedOutputSize, SurfaceInfo outputSurfaceInfo) throws FrameProcessingException { + SurfaceInfo outputSurfaceInfo) throws FrameProcessingException { ImmutableList.Builder matrixTransformationListBuilder = new ImmutableList.Builder().addAll(matrixTransformations); if (outputSurfaceInfo.orientationDegrees != 0) { @@ -224,12 +236,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; .setRotationDegrees(outputSurfaceInfo.orientationDegrees) .build()); } - if (outputSurfaceInfo.width != requestedOutputSize.getWidth() - || outputSurfaceInfo.height != requestedOutputSize.getHeight()) { - matrixTransformationListBuilder.add( - Presentation.createForWidthAndHeight( - outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT)); - } + matrixTransformationListBuilder.add( + Presentation.createForWidthAndHeight( + outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT)); MatrixTransformationProcessor matrixTransformationProcessor = new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build()); @@ -258,6 +267,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { + if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) { + this.outputSurfaceInfo = outputSurfaceInfo; + this.outputEglSurface = null; + } + } + /** * Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid, * and makes rendering a no-op if not. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java index 17c60e801d..01802199d1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; import android.view.Surface; +import androidx.annotation.Nullable; /** Interface for a frame processor that applies changes to individual video frames. */ /* package */ interface FrameProcessor { @@ -26,6 +27,14 @@ import android.view.Surface; */ interface Listener { + /** + * Called when the output size after applying the final effect changes. + * + *

The output size after applying the final effect can differ from the size specified using + * {@link #setOutputSurfaceInfo(SurfaceInfo)}. + */ + void onOutputSizeChanged(int width, int height); + /** * Called when an exception occurs during asynchronous frame processing. * @@ -68,6 +77,23 @@ import android.view.Surface; */ int getPendingInputFrameCount(); + /** + * Sets the output surface and supporting information. + * + *

The new output {@link SurfaceInfo} is applied from the next output frame rendered onwards. + * If the output {@link SurfaceInfo} is {@code null}, the {@code FrameProcessor} will stop + * rendering and resume rendering pending frames once a non-null {@link SurfaceInfo} is set. + * + *

If the dimensions given in {@link SurfaceInfo} do not match the {@linkplain + * Listener#onOutputSizeChanged(int,int) output size after applying the final effect} the frames + * are resized before rendering to the surface and letter/pillar-boxing is applied. + * + *

The caller is responsible for tracking the lifecycle of the {@link SurfaceInfo#surface} + * including calling this method with a new surface if it is destroyed. When this method returns, + * the previous output surface is no longer being used and can safely be released by the caller. + */ + void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo); + /** * Informs the {@code FrameProcessor} that no further input frames should be accepted. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index 9823b57153..af27f46a9f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -23,6 +23,7 @@ import android.graphics.SurfaceTexture; import android.opengl.EGL14; import android.opengl.EGLContext; import android.opengl.EGLDisplay; +import android.util.Pair; import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.WorkerThread; @@ -51,8 +52,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param context A {@link Context}. * @param listener A {@link Listener}. * @param effects The {@link GlEffect GlEffects} to apply to each frame. - * @param outputSurfaceProvider A {@link SurfaceInfo.Provider} managing the output {@link - * Surface}. * @param debugViewProvider A {@link Transformer.DebugViewProvider}. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @return A new instance. @@ -64,7 +63,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; FrameProcessor.Listener listener, long streamOffsetUs, List effects, - SurfaceInfo.Provider outputSurfaceProvider, Transformer.DebugViewProvider debugViewProvider, boolean enableExperimentalHdrEditing) throws FrameProcessingException { @@ -79,7 +77,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; listener, streamOffsetUs, effects, - outputSurfaceProvider, debugViewProvider, enableExperimentalHdrEditing, singleThreadExecutorService)); @@ -108,7 +105,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; FrameProcessor.Listener listener, long streamOffsetUs, List effects, - SurfaceInfo.Provider outputSurfaceProvider, Transformer.DebugViewProvider debugViewProvider, boolean enableExperimentalHdrEditing, ExecutorService singleThreadExecutorService) @@ -131,24 +127,31 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); } - ImmutableList textureProcessors = - getGlTextureProcessorsForGlEffects( - context, - effects, - eglDisplay, - eglContext, - outputSurfaceProvider, - streamOffsetUs, - listener, - debugViewProvider, - enableExperimentalHdrEditing); + Pair, FinalMatrixTransformationProcessorWrapper> + textureProcessors = + getGlTextureProcessorsForGlEffects( + context, + effects, + eglDisplay, + eglContext, + streamOffsetUs, + listener, + debugViewProvider, + enableExperimentalHdrEditing); + ImmutableList intermediateTextureProcessors = textureProcessors.first; + FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper = + textureProcessors.second; ExternalTextureProcessor externalTextureProcessor = new ExternalTextureProcessor(context, enableExperimentalHdrEditing); FrameProcessingTaskExecutor frameProcessingTaskExecutor = new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); chainTextureProcessorsWithListeners( - externalTextureProcessor, textureProcessors, frameProcessingTaskExecutor, listener); + externalTextureProcessor, + intermediateTextureProcessors, + finalTextureProcessorWrapper, + frameProcessingTaskExecutor, + listener); return new GlEffectsFrameProcessor( eglDisplay, @@ -157,7 +160,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, /* inputExternalTextureId= */ GlUtil.createExternalTexture(), externalTextureProcessor, - textureProcessors); + intermediateTextureProcessors, + finalTextureProcessorWrapper); } /** @@ -165,21 +169,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate * {@link GlTextureProcessor} instances. * - *

The final {@link GlTextureProcessor} is wrapped in a {@link - * FinalMatrixTransformationProcessorWrapper} so that it can write directly to the {@linkplain - * SurfaceInfo.Provider provided output surface}. + * @return A {@link Pair} containing a list of {@link GlTextureProcessor} instances to apply in + * the given order and a {@link FinalMatrixTransformationProcessorWrapper} to apply after + * them. */ - private static ImmutableList getGlTextureProcessorsForGlEffects( - Context context, - List effects, - EGLDisplay eglDisplay, - EGLContext eglContext, - SurfaceInfo.Provider outputSurfaceProvider, - long streamOffsetUs, - FrameProcessor.Listener listener, - Transformer.DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing) - throws FrameProcessingException { + private static Pair, FinalMatrixTransformationProcessorWrapper> + getGlTextureProcessorsForGlEffects( + Context context, + List effects, + EGLDisplay eglDisplay, + EGLContext eglContext, + long streamOffsetUs, + FrameProcessor.Listener listener, + Transformer.DebugViewProvider debugViewProvider, + boolean enableExperimentalHdrEditing) + throws FrameProcessingException { ImmutableList.Builder textureProcessorListBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder matrixTransformationListBuilder = @@ -199,18 +203,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } textureProcessorListBuilder.add(effect.toGlTextureProcessor(context)); } - textureProcessorListBuilder.add( + return Pair.create( + textureProcessorListBuilder.build(), new FinalMatrixTransformationProcessorWrapper( context, eglDisplay, eglContext, matrixTransformationListBuilder.build(), - outputSurfaceProvider, streamOffsetUs, listener, debugViewProvider, enableExperimentalHdrEditing)); - return textureProcessorListBuilder.build(); } /** @@ -221,21 +224,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; */ private static void chainTextureProcessorsWithListeners( ExternalTextureProcessor externalTextureProcessor, - ImmutableList textureProcessors, + ImmutableList intermediateTextureProcessors, + FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper, FrameProcessingTaskExecutor frameProcessingTaskExecutor, FrameProcessor.Listener listener) { externalTextureProcessor.setListener( new ChainingGlTextureProcessorListener( /* previousGlTextureProcessor= */ null, - textureProcessors.get(0), + /* nextGlTextureProcessor= */ intermediateTextureProcessors.size() > 0 + ? intermediateTextureProcessors.get(0) + : finalTextureProcessorWrapper, frameProcessingTaskExecutor, listener)); GlTextureProcessor previousGlTextureProcessor = externalTextureProcessor; - for (int i = 0; i < textureProcessors.size(); i++) { - GlTextureProcessor glTextureProcessor = textureProcessors.get(i); + for (int i = 0; i < intermediateTextureProcessors.size(); i++) { + GlTextureProcessor glTextureProcessor = intermediateTextureProcessors.get(i); @Nullable GlTextureProcessor nextGlTextureProcessor = - i + 1 < textureProcessors.size() ? textureProcessors.get(i + 1) : null; + i + 1 < intermediateTextureProcessors.size() + ? intermediateTextureProcessors.get(i + 1) + : finalTextureProcessorWrapper; glTextureProcessor.setListener( new ChainingGlTextureProcessorListener( previousGlTextureProcessor, @@ -244,6 +252,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; listener)); previousGlTextureProcessor = glTextureProcessor; } + finalTextureProcessorWrapper.setListener( + new ChainingGlTextureProcessorListener( + previousGlTextureProcessor, + /* nextGlTextureProcessor= */ null, + frameProcessingTaskExecutor, + listener)); } private static final String TAG = "GlEffectsFrameProcessor"; @@ -267,7 +281,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final float[] inputSurfaceTextureTransformMatrix; private final int inputExternalTextureId; private final ExternalTextureProcessor inputExternalTextureProcessor; - private final ImmutableList textureProcessors; + private final ImmutableList intermediateTextureProcessors; + private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper; private final ConcurrentLinkedQueue pendingInputFrames; private @MonotonicNonNull FrameInfo nextInputFrameInfo; @@ -280,8 +295,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, int inputExternalTextureId, ExternalTextureProcessor inputExternalTextureProcessor, - ImmutableList textureProcessors) { - checkState(!textureProcessors.isEmpty()); + ImmutableList intermediateTextureProcessors, + FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper) { this.eglDisplay = eglDisplay; this.eglContext = eglContext; @@ -289,7 +304,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.streamOffsetUs = streamOffsetUs; this.inputExternalTextureId = inputExternalTextureId; this.inputExternalTextureProcessor = inputExternalTextureProcessor; - this.textureProcessors = textureProcessors; + this.intermediateTextureProcessors = intermediateTextureProcessors; + this.finalTextureProcessorWrapper = finalTextureProcessorWrapper; inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId); inputSurface = new Surface(inputSurfaceTexture); @@ -323,6 +339,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return pendingInputFrames.size(); } + @Override + public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { + finalTextureProcessorWrapper.setOutputSurfaceInfo(outputSurfaceInfo); + } + @Override public void signalEndOfInputStream() { checkState(!inputStreamEnded); @@ -423,9 +444,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private void releaseTextureProcessorsAndDestroyGlContext() throws GlUtil.GlException, FrameProcessingException { inputExternalTextureProcessor.release(); - for (int i = 0; i < textureProcessors.size(); i++) { - textureProcessors.get(i).release(); + for (int i = 0; i < intermediateTextureProcessors.size(); i++) { + intermediateTextureProcessors.get(i).release(); } + finalTextureProcessorWrapper.release(); GlUtil.destroyEglContext(eglDisplay, eglContext); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceInfo.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceInfo.java index 09bc801058..217dd8c75c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceInfo.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceInfo.java @@ -55,16 +55,27 @@ import androidx.annotation.Nullable; this.orientationDegrees = orientationDegrees; } - /** A provider for a {@link SurfaceInfo} instance. */ - public interface Provider { - /** - * Provides a {@linkplain SurfaceInfo surface} for the requested dimensions. - * - *

The dimensions given in the provided {@link SurfaceInfo} may differ from the requested - * dimensions. It is up to the caller to transform frames from the requested dimensions to the - * provided dimensions before rendering them to the {@link SurfaceInfo#surface}. - */ - @Nullable - SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight); + @Override + public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SurfaceInfo)) { + return false; + } + SurfaceInfo that = (SurfaceInfo) o; + return width == that.width + && height == that.height + && orientationDegrees == that.orientationDegrees + && surface.equals(that.surface); + } + + @Override + public int hashCode() { + int result = surface.hashCode(); + result = 31 * result + width; + result = 31 * result + height; + result = 31 * result + orientationDegrees; + return result; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 4304f3483a..44301842fb 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -95,14 +95,23 @@ import org.checkerframework.dataflow.qual.Pure; inputFormat, allowedOutputMimeTypes, transformationRequest, - fallbackListener, - asyncErrorListener); + fallbackListener); try { frameProcessor = GlEffectsFrameProcessor.create( context, new FrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + try { + checkNotNull(frameProcessor) + .setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height)); + } catch (TransformationException exception) { + asyncErrorListener.onTransformationException(exception); + } + } + @Override public void onFrameProcessingError(FrameProcessingException exception) { asyncErrorListener.onTransformationException( @@ -121,7 +130,6 @@ import org.checkerframework.dataflow.qual.Pure; }, streamOffsetUs, effectsListBuilder.build(), - /* outputSurfaceProvider= */ encoderWrapper, debugViewProvider, transformationRequest.enableHdrEditing); } catch (FrameProcessingException e) { @@ -284,14 +292,13 @@ import org.checkerframework.dataflow.qual.Pure; * dimensions, the same encoder is used and the provided dimensions stay fixed. */ @VisibleForTesting - /* package */ static final class EncoderWrapper implements SurfaceInfo.Provider { + /* package */ static final class EncoderWrapper { private final Codec.EncoderFactory encoderFactory; private final Format inputFormat; private final List allowedOutputMimeTypes; private final TransformationRequest transformationRequest; private final FallbackListener fallbackListener; - private final Transformer.AsyncErrorListener asyncErrorListener; private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo; @@ -304,20 +311,18 @@ import org.checkerframework.dataflow.qual.Pure; Format inputFormat, List allowedOutputMimeTypes, TransformationRequest transformationRequest, - FallbackListener fallbackListener, - Transformer.AsyncErrorListener asyncErrorListener) { + FallbackListener fallbackListener) { this.encoderFactory = encoderFactory; this.inputFormat = inputFormat; this.allowedOutputMimeTypes = allowedOutputMimeTypes; this.transformationRequest = transformationRequest; this.fallbackListener = fallbackListener; - this.asyncErrorListener = asyncErrorListener; } - @Override @Nullable - public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight) { + public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight) + throws TransformationException { if (releaseEncoder) { return null; } @@ -349,13 +354,8 @@ import org.checkerframework.dataflow.qual.Pure; : inputFormat.sampleMimeType) .build(); - try { - encoder = - encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); - } catch (TransformationException e) { - asyncErrorListener.onTransformationException(e); - return null; - } + encoder = + encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); Format encoderSupportedFormat = encoder.getConfigurationFormat(); fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java index ac86e370dd..1b8b8b4502 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java @@ -50,8 +50,7 @@ public final class VideoEncoderWrapperTest { /* inputFormat= */ new Format.Builder().build(), /* allowedOutputMimeTypes= */ ImmutableList.of(), emptyTransformationRequest, - fallbackListener, - mock(Transformer.AsyncErrorListener.class)); + fallbackListener); @Before public void registerTrack() { @@ -59,7 +58,7 @@ public final class VideoEncoderWrapperTest { } @Test - public void getSurfaceInfo_landscape_leavesOrientationUnchanged() { + public void getSurfaceInfo_landscape_leavesOrientationUnchanged() throws Exception { int inputWidth = 200; int inputHeight = 150; @@ -71,7 +70,7 @@ public final class VideoEncoderWrapperTest { } @Test - public void getSurfaceInfo_square_leavesOrientationUnchanged() { + public void getSurfaceInfo_square_leavesOrientationUnchanged() throws Exception { int inputWidth = 150; int inputHeight = 150; @@ -83,7 +82,7 @@ public final class VideoEncoderWrapperTest { } @Test - public void getSurfaceInfo_portrait_flipsOrientation() { + public void getSurfaceInfo_portrait_flipsOrientation() throws Exception { int inputWidth = 150; int inputHeight = 200; @@ -95,7 +94,8 @@ public final class VideoEncoderWrapperTest { } @Test - public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution() { + public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution() + throws TransformationException { int inputWidth = 200; int inputHeight = 150; int fallbackWidth = 100;