From dbe57af5530561f47b36b77bfeb43e88b8af83ee Mon Sep 17 00:00:00 2001 From: hschlueter Date: Thu, 24 Mar 2022 18:44:15 +0000 Subject: [PATCH] Configure the frame sizes in FrameProcessorChain instead of caller. Configuring the frame sizes between frame processors is now the FrameProcessorChain's rather than the caller's responsibility. The caller can getOutputSize() and override it for encoder fallback in configure(). PiperOrigin-RevId: 437048436 --- .../FrameProcessorChainPixelTest.java | 28 ++--- .../transformer/FrameProcessorChain.java | 112 +++++++++--------- .../VideoTranscodingSamplePipeline.java | 25 ++-- .../transformer/FrameProcessorChainTest.java | 68 +++++++---- 4 files changed, 126 insertions(+), 107 deletions(-) diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java index 3de44234fc..926c776002 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java @@ -41,9 +41,7 @@ import android.util.Size; import androidx.annotation.Nullable; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.exoplayer2.util.MimeTypes; -import com.google.common.collect.Iterables; import java.nio.ByteBuffer; -import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; import org.junit.Test; @@ -247,24 +245,26 @@ public final class FrameProcessorChainPixelTest { int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); - List frameProcessorsList = asList(frameProcessors); - List sizes = - FrameProcessorChain.configureSizes(inputWidth, inputHeight, frameProcessorsList); - assertThat(sizes).isNotEmpty(); - int outputWidth = Iterables.getLast(sizes).getWidth(); - int outputHeight = Iterables.getLast(sizes).getHeight(); - outputImageReader = - ImageReader.newInstance( - outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1); frameProcessorChain = new FrameProcessorChain( context, PIXEL_WIDTH_HEIGHT_RATIO, - frameProcessorsList, - sizes, + inputWidth, + inputHeight, + asList(frameProcessors), /* enableExperimentalHdrEditing= */ false); + Size outputSize = frameProcessorChain.getOutputSize(); + outputImageReader = + ImageReader.newInstance( + outputSize.getWidth(), + outputSize.getHeight(), + PixelFormat.RGBA_8888, + /* maxImages= */ 1); frameProcessorChain.configure( - outputImageReader.getSurface(), outputWidth, outputHeight, /* debugSurfaceView= */ null); + outputImageReader.getSurface(), + outputSize.getWidth(), + outputSize.getHeight(), + /* debugSurfaceView= */ null); frameProcessorChain.registerInputFrame(); // Queue the first video frame from the extractor. diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java index 65fa498faf..a78e165d5b 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java @@ -15,7 +15,6 @@ */ package com.google.android.exoplayer2.transformer; -import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; @@ -29,6 +28,7 @@ import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; import android.opengl.GLES20; +import android.util.Pair; import android.util.Size; import android.view.Surface; import android.view.SurfaceView; @@ -36,8 +36,8 @@ import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.Util; +import com.google.common.collect.ImmutableList; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; @@ -56,8 +56,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * processed on a background thread as it becomes available. All input frames should be {@link * #registerInputFrame() registered} before they are rendered to the input surface. {@link * #hasPendingFrames()} can be used to check whether there are frames that have not been fully - * processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link - * #configure(Surface, int, int, SurfaceView)}. + * processed yet. Output is written to its {@link #configure(Surface, int, int, SurfaceView) output + * surface}. */ /* package */ final class FrameProcessorChain { @@ -65,32 +65,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; GlUtil.glAssertionsEnabled = true; } - /** - * Configures the output {@link Size sizes} of a list of {@link GlFrameProcessor - * GlFrameProcessors}. - * - * @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}. - * @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}. - * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}. - * @return A mutable {@link List} containing the input {@link Size} as well as the output {@link - * Size} of each {@link GlFrameProcessor}. - */ - // TODO(b/218488308): Return an immutable list once VideoTranscodingSamplePipeline no longer needs - // to modify this list for encoder fallback. - public static List configureSizes( - int inputWidth, int inputHeight, List frameProcessors) { - - List sizes = new ArrayList<>(frameProcessors.size() + 1); - sizes.add(new Size(inputWidth, inputHeight)); - for (int i = 0; i < frameProcessors.size(); i++) { - sizes.add( - frameProcessors - .get(i) - .configureOutputSize(getLast(sizes).getWidth(), getLast(sizes).getHeight())); - } - return sizes; - } - private static final String THREAD_NAME = "Transformer:FrameProcessorChain"; private final boolean enableExperimentalHdrEditing; @@ -129,12 +103,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; *

The {@link ExternalCopyFrameProcessor} writes to the first framebuffer. */ private final int[] framebuffers; - /** - * The input {@link Size}, i.e., the output {@link Size} of the {@link - * ExternalCopyFrameProcessor}), as well as the output {@link Size} of each of the {@code - * frameProcessors}. - */ - private final List sizes; + /** The input {@link Size} of each of the {@code frameProcessors}. */ + private final ImmutableList inputSizes; private int outputWidth; private int outputHeight; @@ -157,23 +127,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * * @param context A {@link Context}. * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. + * @param inputWidth The input frame width, in pixels. + * @param inputHeight The input frame height, in pixels. * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame. - * Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)} - * configured}. - * @param sizes The input {@link Size} as well as the output {@link Size} of each {@link - * GlFrameProcessor}. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1. */ public FrameProcessorChain( Context context, float pixelWidthHeightRatio, + int inputWidth, + int inputHeight, List frameProcessors, - List sizes, boolean enableExperimentalHdrEditing) throws TransformationException { - checkArgument(frameProcessors.size() + 1 == sizes.size()); - if (pixelWidthHeightRatio != 1.0f) { // TODO(b/211782176): Consider implementing support for non-square pixels. throw TransformationException.createForFrameProcessorChain( @@ -186,7 +153,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; this.frameProcessors = frameProcessors; - this.sizes = sizes; try { eglDisplay = GlUtil.createEglDisplay(); @@ -205,12 +171,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; externalCopyFrameProcessor = new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); framebuffers = new int[frameProcessors.size()]; - outputWidth = getLast(sizes).getWidth(); - outputHeight = getLast(sizes).getHeight(); + Pair, Size> sizes = + configureFrameProcessorSizes(inputWidth, inputHeight, frameProcessors); + inputSizes = sizes.first; + outputWidth = sizes.second.getWidth(); + outputHeight = sizes.second.getHeight(); debugPreviewWidth = C.LENGTH_UNSET; debugPreviewHeight = C.LENGTH_UNSET; } + /** Returns the output {@link Size}. */ + public Size getOutputSize() { + return new Size(outputWidth, outputHeight); + } + /** * Configures the {@code FrameProcessorChain} to process frames to the specified output targets. * @@ -399,12 +373,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); inputExternalTexId = GlUtil.createExternalTexture(); - externalCopyFrameProcessor.configureOutputSize( - /* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight()); + Size inputSize = inputSizes.get(0); + externalCopyFrameProcessor.configureOutputSize(inputSize.getWidth(), inputSize.getHeight()); externalCopyFrameProcessor.initialize(inputExternalTexId); for (int i = 0; i < frameProcessors.size(); i++) { - int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight()); + inputSize = inputSizes.get(i); + int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight()); framebuffers[i] = GlUtil.createFboForTexture(inputTexId); frameProcessors.get(i).initialize(inputTexId); } @@ -423,16 +398,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private void processFrame() { checkState(Thread.currentThread().equals(glThread)); + Size outputSize = inputSizes.get(0); if (frameProcessors.isEmpty()) { - GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); + GlUtil.focusEglSurface( + eglDisplay, eglContext, eglSurface, outputSize.getWidth(), outputSize.getHeight()); } else { GlUtil.focusFramebuffer( eglDisplay, eglContext, eglSurface, framebuffers[0], - sizes.get(0).getWidth(), - sizes.get(0).getHeight()); + outputSize.getWidth(), + outputSize.getHeight()); } inputSurfaceTexture.updateTexImage(); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); @@ -441,13 +418,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs); for (int i = 0; i < frameProcessors.size() - 1; i++) { + outputSize = inputSizes.get(i + 1); GlUtil.focusFramebuffer( eglDisplay, eglContext, eglSurface, framebuffers[i + 1], - sizes.get(i + 1).getWidth(), - sizes.get(i + 1).getHeight()); + outputSize.getWidth(), + outputSize.getHeight()); frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs); } if (!frameProcessors.isEmpty()) { @@ -470,4 +448,30 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; checkState(pendingFrameCount.getAndDecrement() > 0); } + + /** + * Configures the input and output {@link Size sizes} of a list of {@link GlFrameProcessor + * GlFrameProcessors}. + * + * @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}, in pixels. + * @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}, in + * pixels. + * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}. + * @return The input {@link Size} of each {@link GlFrameProcessor} and the output {@link Size} of + * the final {@link GlFrameProcessor}. + */ + private static Pair, Size> configureFrameProcessorSizes( + int inputWidth, int inputHeight, List frameProcessors) { + Size size = new Size(inputWidth, inputHeight); + if (frameProcessors.isEmpty()) { + return Pair.create(ImmutableList.of(size), size); + } + + ImmutableList.Builder inputSizes = new ImmutableList.Builder<>(); + for (int i = 0; i < frameProcessors.size(); i++) { + inputSizes.add(size); + size = frameProcessors.get(i).configureOutputSize(size.getWidth(), size.getHeight()); + } + return Pair.create(inputSizes.build(), size); + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index 71e3b8d89e..f6db0a764b 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -29,7 +29,6 @@ import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.util.Util; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import java.util.List; import org.checkerframework.dataflow.qual.Pure; @@ -70,6 +69,7 @@ import org.checkerframework.dataflow.qual.Pure; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; + // TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline. // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset. ScaleToFitFrameProcessor scaleToFitFrameProcessor = new ScaleToFitFrameProcessor.Builder(context) @@ -80,13 +80,15 @@ import org.checkerframework.dataflow.qual.Pure; new PresentationFrameProcessor.Builder(context) .setResolution(transformationRequest.outputHeight) .build(); - // TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline. - ImmutableList frameProcessors = - ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor); - List frameProcessorSizes = - FrameProcessorChain.configureSizes(decodedWidth, decodedHeight, frameProcessors); - Size requestedEncoderSize = Iterables.getLast(frameProcessorSizes); - // TODO(b/213190310): Move output rotation configuration to PresentationFrameProcessor. + frameProcessorChain = + new FrameProcessorChain( + context, + inputFormat.pixelWidthHeightRatio, + /* inputWidth= */ decodedWidth, + /* inputHeight= */ decodedHeight, + ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor), + transformationRequest.enableHdrEditing); + Size requestedEncoderSize = frameProcessorChain.getOutputSize(); outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees(); Format requestedEncoderFormat = @@ -110,13 +112,6 @@ import org.checkerframework.dataflow.qual.Pure; requestedEncoderFormat, encoderSupportedFormat)); - frameProcessorChain = - new FrameProcessorChain( - context, - inputFormat.pixelWidthHeightRatio, - frameProcessors, - frameProcessorSizes, - transformationRequest.enableHdrEditing); frameProcessorChain.configure( /* outputSurface= */ encoder.getInputSurface(), /* outputWidth= */ encoderSupportedFormat.width, diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java index e1b2e1b2dd..e6f873d654 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java @@ -35,6 +35,7 @@ import org.junit.runner.RunWith; */ @RunWith(AndroidJUnit4.class) public final class FrameProcessorChainTest { + @Test public void construct_withSupportedPixelWidthHeightRatio_completesSuccessfully() throws TransformationException { @@ -43,8 +44,9 @@ public final class FrameProcessorChainTest { new FrameProcessorChain( context, /* pixelWidthHeightRatio= */ 1, + /* inputWidth= */ 200, + /* inputHeight= */ 100, /* frameProcessors= */ ImmutableList.of(), - /* sizes= */ ImmutableList.of(new Size(200, 100)), /* enableExperimentalHdrEditing= */ false); } @@ -59,8 +61,9 @@ public final class FrameProcessorChainTest { new FrameProcessorChain( context, /* pixelWidthHeightRatio= */ 2, + /* inputWidth= */ 200, + /* inputHeight= */ 100, /* frameProcessors= */ ImmutableList.of(), - /* sizes= */ ImmutableList.of(new Size(200, 100)), /* enableExperimentalHdrEditing= */ false)); assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class); @@ -68,46 +71,63 @@ public final class FrameProcessorChainTest { } @Test - public void configureOutputDimensions_withEmptyList_returnsInputSize() { + public void getOutputSize_withoutFrameProcessors_returnsInputSize() + throws TransformationException { Size inputSize = new Size(200, 100); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of()); - List sizes = - FrameProcessorChain.configureSizes( - inputSize.getWidth(), inputSize.getHeight(), /* frameProcessors= */ ImmutableList.of()); + Size outputSize = frameProcessorChain.getOutputSize(); - assertThat(sizes).containsExactly(inputSize); + assertThat(outputSize).isEqualTo(inputSize); } @Test - public void configureOutputDimensions_withOneFrameProcessor_returnsItsInputAndOutputDimensions() { + public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize() + throws TransformationException { Size inputSize = new Size(200, 100); - Size outputSize = new Size(300, 250); - GlFrameProcessor frameProcessor = new FakeFrameProcessor(outputSize); + Size frameProcessorOutputSize = new Size(300, 250); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize)); - List sizes = - FrameProcessorChain.configureSizes( - inputSize.getWidth(), inputSize.getHeight(), ImmutableList.of(frameProcessor)); + Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); - assertThat(sizes).containsExactly(inputSize, outputSize).inOrder(); + assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize); } @Test - public void configureOutputDimensions_withThreeFrameProcessors_propagatesOutputDimensions() { + public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize() + throws TransformationException { Size inputSize = new Size(200, 100); Size outputSize1 = new Size(300, 250); Size outputSize2 = new Size(400, 244); Size outputSize3 = new Size(150, 160); - GlFrameProcessor frameProcessor1 = new FakeFrameProcessor(outputSize1); - GlFrameProcessor frameProcessor2 = new FakeFrameProcessor(outputSize2); - GlFrameProcessor frameProcessor3 = new FakeFrameProcessor(outputSize3); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, + /* frameProcessorOutputSizes= */ ImmutableList.of( + outputSize1, outputSize2, outputSize3)); - List sizes = - FrameProcessorChain.configureSizes( - inputSize.getWidth(), - inputSize.getHeight(), - ImmutableList.of(frameProcessor1, frameProcessor2, frameProcessor3)); + Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); - assertThat(sizes).containsExactly(inputSize, outputSize1, outputSize2, outputSize3).inOrder(); + assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3); + } + + private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors( + Size inputSize, List frameProcessorOutputSizes) throws TransformationException { + ImmutableList.Builder frameProcessors = new ImmutableList.Builder<>(); + for (Size element : frameProcessorOutputSizes) { + frameProcessors.add(new FakeFrameProcessor(element)); + } + return new FrameProcessorChain( + getApplicationContext(), + /* pixelWidthHeightRatio= */ 1, + inputSize.getWidth(), + inputSize.getHeight(), + frameProcessors.build(), + /* enableExperimentalHdrEditing= */ false); } private static class FakeFrameProcessor implements GlFrameProcessor {