diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index 6cee080d27..00a9dafa6a 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -41,9 +41,7 @@ import android.util.Size; import androidx.annotation.Nullable; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.google.common.collect.Iterables; import java.nio.ByteBuffer; -import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; import org.junit.Test; @@ -247,24 +245,26 @@ public final class FrameProcessorChainPixelTest { int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); - List frameProcessorsList = asList(frameProcessors); - List sizes = - FrameProcessorChain.configureSizes(inputWidth, inputHeight, frameProcessorsList); - assertThat(sizes).isNotEmpty(); - int outputWidth = Iterables.getLast(sizes).getWidth(); - int outputHeight = Iterables.getLast(sizes).getHeight(); - outputImageReader = - ImageReader.newInstance( - outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1); frameProcessorChain = new FrameProcessorChain( context, PIXEL_WIDTH_HEIGHT_RATIO, - frameProcessorsList, - sizes, + inputWidth, + inputHeight, + asList(frameProcessors), /* enableExperimentalHdrEditing= */ false); + Size outputSize = frameProcessorChain.getOutputSize(); + outputImageReader = + ImageReader.newInstance( + outputSize.getWidth(), + outputSize.getHeight(), + PixelFormat.RGBA_8888, + /* maxImages= */ 1); frameProcessorChain.configure( - outputImageReader.getSurface(), outputWidth, outputHeight, /* debugSurfaceView= */ null); + outputImageReader.getSurface(), + outputSize.getWidth(), + outputSize.getHeight(), + /* debugSurfaceView= */ null); frameProcessorChain.registerInputFrame(); // Queue the first video frame from the extractor. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index 8b445864b9..7972e5f37a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -15,7 +15,6 @@ */ package androidx.media3.transformer; -import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; @@ -29,6 +28,7 @@ import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; import android.opengl.GLES20; +import android.util.Pair; import android.util.Size; import android.view.Surface; import android.view.SurfaceView; @@ -36,8 +36,8 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Util; +import com.google.common.collect.ImmutableList; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; @@ -56,8 +56,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * processed on a background thread as it becomes available. All input frames should be {@link * #registerInputFrame() registered} before they are rendered to the input surface. {@link * #hasPendingFrames()} can be used to check whether there are frames that have not been fully - * processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link - * #configure(Surface, int, int, SurfaceView)}. + * processed yet. Output is written to its {@link #configure(Surface, int, int, SurfaceView) output + * surface}. */ /* package */ final class FrameProcessorChain { @@ -65,32 +65,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; GlUtil.glAssertionsEnabled = true; } - /** - * Configures the output {@link Size sizes} of a list of {@link GlFrameProcessor - * GlFrameProcessors}. - * - * @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}. - * @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}. - * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}. - * @return A mutable {@link List} containing the input {@link Size} as well as the output {@link - * Size} of each {@link GlFrameProcessor}. - */ - // TODO(b/218488308): Return an immutable list once VideoTranscodingSamplePipeline no longer needs - // to modify this list for encoder fallback. - public static List configureSizes( - int inputWidth, int inputHeight, List frameProcessors) { - - List sizes = new ArrayList<>(frameProcessors.size() + 1); - sizes.add(new Size(inputWidth, inputHeight)); - for (int i = 0; i < frameProcessors.size(); i++) { - sizes.add( - frameProcessors - .get(i) - .configureOutputSize(getLast(sizes).getWidth(), getLast(sizes).getHeight())); - } - return sizes; - } - private static final String THREAD_NAME = "Transformer:FrameProcessorChain"; private final boolean enableExperimentalHdrEditing; @@ -129,12 +103,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; *

The {@link ExternalCopyFrameProcessor} writes to the first framebuffer. */ private final int[] framebuffers; - /** - * The input {@link Size}, i.e., the output {@link Size} of the {@link - * ExternalCopyFrameProcessor}), as well as the output {@link Size} of each of the {@code - * frameProcessors}. - */ - private final List sizes; + /** The input {@link Size} of each of the {@code frameProcessors}. */ + private final ImmutableList inputSizes; private int outputWidth; private int outputHeight; @@ -157,23 +127,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * * @param context A {@link Context}. * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. + * @param inputWidth The input frame width, in pixels. + * @param inputHeight The input frame height, in pixels. * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame. - * Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)} - * configured}. - * @param sizes The input {@link Size} as well as the output {@link Size} of each {@link - * GlFrameProcessor}. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1. */ public FrameProcessorChain( Context context, float pixelWidthHeightRatio, + int inputWidth, + int inputHeight, List frameProcessors, - List sizes, boolean enableExperimentalHdrEditing) throws TransformationException { - checkArgument(frameProcessors.size() + 1 == sizes.size()); - if (pixelWidthHeightRatio != 1.0f) { // TODO(b/211782176): Consider implementing support for non-square pixels. throw TransformationException.createForFrameProcessorChain( @@ -186,7 +153,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; this.frameProcessors = frameProcessors; - this.sizes = sizes; try { eglDisplay = GlUtil.createEglDisplay(); @@ -205,12 +171,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; externalCopyFrameProcessor = new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); framebuffers = new int[frameProcessors.size()]; - outputWidth = getLast(sizes).getWidth(); - outputHeight = getLast(sizes).getHeight(); + Pair, Size> sizes = + configureFrameProcessorSizes(inputWidth, inputHeight, frameProcessors); + inputSizes = sizes.first; + outputWidth = sizes.second.getWidth(); + outputHeight = sizes.second.getHeight(); debugPreviewWidth = C.LENGTH_UNSET; debugPreviewHeight = C.LENGTH_UNSET; } + /** Returns the output {@link Size}. */ + public Size getOutputSize() { + return new Size(outputWidth, outputHeight); + } + /** * Configures the {@code FrameProcessorChain} to process frames to the specified output targets. * @@ -399,12 +373,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); inputExternalTexId = GlUtil.createExternalTexture(); - externalCopyFrameProcessor.configureOutputSize( - /* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight()); + Size inputSize = inputSizes.get(0); + externalCopyFrameProcessor.configureOutputSize(inputSize.getWidth(), inputSize.getHeight()); externalCopyFrameProcessor.initialize(inputExternalTexId); for (int i = 0; i < frameProcessors.size(); i++) { - int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight()); + inputSize = inputSizes.get(i); + int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight()); framebuffers[i] = GlUtil.createFboForTexture(inputTexId); frameProcessors.get(i).initialize(inputTexId); } @@ -423,16 +398,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private void processFrame() { checkState(Thread.currentThread().equals(glThread)); + Size outputSize = inputSizes.get(0); if (frameProcessors.isEmpty()) { - GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); + GlUtil.focusEglSurface( + eglDisplay, eglContext, eglSurface, outputSize.getWidth(), outputSize.getHeight()); } else { GlUtil.focusFramebuffer( eglDisplay, eglContext, eglSurface, framebuffers[0], - sizes.get(0).getWidth(), - sizes.get(0).getHeight()); + outputSize.getWidth(), + outputSize.getHeight()); } inputSurfaceTexture.updateTexImage(); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); @@ -441,13 +418,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs); for (int i = 0; i < frameProcessors.size() - 1; i++) { + outputSize = inputSizes.get(i + 1); GlUtil.focusFramebuffer( eglDisplay, eglContext, eglSurface, framebuffers[i + 1], - sizes.get(i + 1).getWidth(), - sizes.get(i + 1).getHeight()); + outputSize.getWidth(), + outputSize.getHeight()); frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs); } if (!frameProcessors.isEmpty()) { @@ -470,4 +448,30 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; checkState(pendingFrameCount.getAndDecrement() > 0); } + + /** + * Configures the input and output {@link Size sizes} of a list of {@link GlFrameProcessor + * GlFrameProcessors}. + * + * @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}, in pixels. + * @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}, in + * pixels. + * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}. + * @return The input {@link Size} of each {@link GlFrameProcessor} and the output {@link Size} of + * the final {@link GlFrameProcessor}. + */ + private static Pair, Size> configureFrameProcessorSizes( + int inputWidth, int inputHeight, List frameProcessors) { + Size size = new Size(inputWidth, inputHeight); + if (frameProcessors.isEmpty()) { + return Pair.create(ImmutableList.of(size), size); + } + + ImmutableList.Builder inputSizes = new ImmutableList.Builder<>(); + for (int i = 0; i < frameProcessors.size(); i++) { + inputSizes.add(size); + size = frameProcessors.get(i).configureOutputSize(size.getWidth(), size.getHeight()); + } + return Pair.create(inputSizes.build(), size); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 1c10c4561b..c5c35b092f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -29,7 +29,6 @@ import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import java.util.List; import org.checkerframework.dataflow.qual.Pure; @@ -70,6 +69,7 @@ import org.checkerframework.dataflow.qual.Pure; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; + // TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline. // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset. ScaleToFitFrameProcessor scaleToFitFrameProcessor = new ScaleToFitFrameProcessor.Builder(context) @@ -80,13 +80,15 @@ import org.checkerframework.dataflow.qual.Pure; new PresentationFrameProcessor.Builder(context) .setResolution(transformationRequest.outputHeight) .build(); - // TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline. - ImmutableList frameProcessors = - ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor); - List frameProcessorSizes = - FrameProcessorChain.configureSizes(decodedWidth, decodedHeight, frameProcessors); - Size requestedEncoderSize = Iterables.getLast(frameProcessorSizes); - // TODO(b/213190310): Move output rotation configuration to PresentationFrameProcessor. + frameProcessorChain = + new FrameProcessorChain( + context, + inputFormat.pixelWidthHeightRatio, + /* inputWidth= */ decodedWidth, + /* inputHeight= */ decodedHeight, + ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor), + transformationRequest.enableHdrEditing); + Size requestedEncoderSize = frameProcessorChain.getOutputSize(); outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees(); Format requestedEncoderFormat = @@ -110,13 +112,6 @@ import org.checkerframework.dataflow.qual.Pure; requestedEncoderFormat, encoderSupportedFormat)); - frameProcessorChain = - new FrameProcessorChain( - context, - inputFormat.pixelWidthHeightRatio, - frameProcessors, - frameProcessorSizes, - transformationRequest.enableHdrEditing); frameProcessorChain.configure( /* outputSurface= */ encoder.getInputSurface(), /* outputWidth= */ encoderSupportedFormat.width, diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java index d1ce4a5141..8749225434 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java @@ -35,6 +35,7 @@ import org.junit.runner.RunWith; */ @RunWith(AndroidJUnit4.class) public final class FrameProcessorChainTest { + @Test public void construct_withSupportedPixelWidthHeightRatio_completesSuccessfully() throws TransformationException { @@ -43,8 +44,9 @@ public final class FrameProcessorChainTest { new FrameProcessorChain( context, /* pixelWidthHeightRatio= */ 1, + /* inputWidth= */ 200, + /* inputHeight= */ 100, /* frameProcessors= */ ImmutableList.of(), - /* sizes= */ ImmutableList.of(new Size(200, 100)), /* enableExperimentalHdrEditing= */ false); } @@ -59,8 +61,9 @@ public final class FrameProcessorChainTest { new FrameProcessorChain( context, /* pixelWidthHeightRatio= */ 2, + /* inputWidth= */ 200, + /* inputHeight= */ 100, /* frameProcessors= */ ImmutableList.of(), - /* sizes= */ ImmutableList.of(new Size(200, 100)), /* enableExperimentalHdrEditing= */ false)); assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class); @@ -68,46 +71,63 @@ public final class FrameProcessorChainTest { } @Test - public void configureOutputDimensions_withEmptyList_returnsInputSize() { + public void getOutputSize_withoutFrameProcessors_returnsInputSize() + throws TransformationException { Size inputSize = new Size(200, 100); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of()); - List sizes = - FrameProcessorChain.configureSizes( - inputSize.getWidth(), inputSize.getHeight(), /* frameProcessors= */ ImmutableList.of()); + Size outputSize = frameProcessorChain.getOutputSize(); - assertThat(sizes).containsExactly(inputSize); + assertThat(outputSize).isEqualTo(inputSize); } @Test - public void configureOutputDimensions_withOneFrameProcessor_returnsItsInputAndOutputDimensions() { + public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize() + throws TransformationException { Size inputSize = new Size(200, 100); - Size outputSize = new Size(300, 250); - GlFrameProcessor frameProcessor = new FakeFrameProcessor(outputSize); + Size frameProcessorOutputSize = new Size(300, 250); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize)); - List sizes = - FrameProcessorChain.configureSizes( - inputSize.getWidth(), inputSize.getHeight(), ImmutableList.of(frameProcessor)); + Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); - assertThat(sizes).containsExactly(inputSize, outputSize).inOrder(); + assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize); } @Test - public void configureOutputDimensions_withThreeFrameProcessors_propagatesOutputDimensions() { + public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize() + throws TransformationException { Size inputSize = new Size(200, 100); Size outputSize1 = new Size(300, 250); Size outputSize2 = new Size(400, 244); Size outputSize3 = new Size(150, 160); - GlFrameProcessor frameProcessor1 = new FakeFrameProcessor(outputSize1); - GlFrameProcessor frameProcessor2 = new FakeFrameProcessor(outputSize2); - GlFrameProcessor frameProcessor3 = new FakeFrameProcessor(outputSize3); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, + /* frameProcessorOutputSizes= */ ImmutableList.of( + outputSize1, outputSize2, outputSize3)); - List sizes = - FrameProcessorChain.configureSizes( - inputSize.getWidth(), - inputSize.getHeight(), - ImmutableList.of(frameProcessor1, frameProcessor2, frameProcessor3)); + Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); - assertThat(sizes).containsExactly(inputSize, outputSize1, outputSize2, outputSize3).inOrder(); + assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3); + } + + private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors( + Size inputSize, List frameProcessorOutputSizes) throws TransformationException { + ImmutableList.Builder frameProcessors = new ImmutableList.Builder<>(); + for (Size element : frameProcessorOutputSizes) { + frameProcessors.add(new FakeFrameProcessor(element)); + } + return new FrameProcessorChain( + getApplicationContext(), + /* pixelWidthHeightRatio= */ 1, + inputSize.getWidth(), + inputSize.getHeight(), + frameProcessors.build(), + /* enableExperimentalHdrEditing= */ false); } private static class FakeFrameProcessor implements GlFrameProcessor {