From 2a363ac3fc7fa8958c7e4fc9c7ff27718ed42d4c Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Thu, 24 Mar 2022 15:42:12 +0000 Subject: [PATCH] Transformer: Always use FrameProcessorChain when decoding. This allows us to bypass many device-specific issues, that only occur when decoding directly to an encoder surface, without OpenGL. This also allows us to maintain fewer code branches, which require additional testing to verify correctness. PiperOrigin-RevId: 437003138 --- .../transformer/TransformerEndToEndTest.java | 30 ------- .../PresentationFrameProcessor.java | 27 +----- .../transformer/ScaleToFitFrameProcessor.java | 12 --- .../VideoTranscodingSamplePipeline.java | 88 ++++++------------- .../PresentationFrameProcessorTest.java | 4 - .../ScaleToFitFrameProcessorTest.java | 6 -- 6 files changed, 31 insertions(+), 136 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 7a710a0d8a..d44f514b9d 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static com.google.common.truth.Truth.assertThat; import android.content.Context; -import androidx.media3.common.MimeTypes; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; @@ -32,37 +31,8 @@ import org.junit.runner.RunWith; @RunWith(AndroidJUnit4.class) public class TransformerEndToEndTest { - private static final String VP9_VIDEO_URI_STRING = "asset:///media/vp9/bear-vp9.webm"; private static final String AVC_VIDEO_URI_STRING = "asset:///media/mp4/sample.mp4"; - @Test - public void videoTranscoding_completesWithConsistentFrameCount() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); - FrameCountingMuxer.Factory muxerFactory = - new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory()); - Transformer transformer = - new Transformer.Builder(context) - .setTransformationRequest( - new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H264).build()) - .setMuxerFactory(muxerFactory) - .setEncoderFactory( - new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false)) - .build(); - // Result of the following command: - // ffprobe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames bear-vp9.webm - int expectedFrameCount = 82; - - new TransformerAndroidTestRunner.Builder(context, transformer) - .build() - .run( - /* testId= */ "videoTranscoding_completesWithConsistentFrameCount", - VP9_VIDEO_URI_STRING); - - FrameCountingMuxer frameCountingMuxer = - checkNotNull(muxerFactory.getLastFrameCountingMuxerCreated()); - assertThat(frameCountingMuxer.getFrameCount()).isEqualTo(expectedFrameCount); - } - @Test public void videoEditing_completesWithConsistentFrameCount() throws Exception { Context context = ApplicationProvider.getApplicationContext(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java index 16c13f0258..31be674a70 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java @@ -81,7 +81,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; private int inputWidth; private int inputHeight; - private int outputHeight; private int outputRotationDegrees; private @MonotonicNonNull Matrix transformationMatrix; @@ -97,7 +96,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputWidth = C.LENGTH_UNSET; inputHeight = C.LENGTH_UNSET; - outputHeight = C.LENGTH_UNSET; outputRotationDegrees = C.LENGTH_UNSET; } @@ -113,18 +111,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return outputRotationDegrees; } - /** - * Returns whether this {@code PresentationFrameProcessor} will apply any changes on a frame. - * - *

The {@code PresentationFrameProcessor} should only be used if this returns true. - * - *

This method can only be called after {@link #configureOutputSize(int, int)}. - */ - public boolean shouldProcess() { - checkStateNotNull(transformationMatrix); - return inputHeight != outputHeight || !transformationMatrix.isIdentity(); - } - @Override public Size configureOutputSize(int inputWidth, int inputHeight) { this.inputWidth = inputWidth; @@ -140,24 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; displayHeight = requestedHeight; } - int outputWidth; // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded // frame before encoding, so the encoded frame's width >= height, and set // outputRotationDegrees to ensure the frame is displayed in the correct orientation. if (displayHeight > displayWidth) { outputRotationDegrees = 90; - outputWidth = displayHeight; - outputHeight = displayWidth; - // TODO(b/201293185): After fragment shader transformations are implemented, put postRotate in - // a later GlFrameProcessor. + // TODO(b/201293185): After fragment shader transformations are implemented, put + // postRotate in a later GlFrameProcessor. transformationMatrix.postRotate(outputRotationDegrees); + return new Size(displayHeight, displayWidth); } else { outputRotationDegrees = 0; - outputWidth = displayWidth; - outputHeight = displayHeight; + return new Size(displayWidth, displayHeight); } - - return new Size(outputWidth, outputHeight); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java index 73fc0076bb..472cf04171 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java @@ -122,18 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputHeight = C.LENGTH_UNSET; } - /** - * Returns whether this ScaleToFitFrameProcessor will apply any changes on a frame. - * - *

The ScaleToFitFrameProcessor should only be used if this returns true. - * - *

This method can only be called after {@link #configureOutputSize(int, int)}. - */ - public boolean shouldProcess() { - checkStateNotNull(adjustedTransformationMatrix); - return !transformationMatrix.isIdentity(); - } - @Override public Size configureOutputSize(int inputWidth, int inputHeight) { this.inputWidth = inputWidth; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 61d120ae4f..4c27eca73c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -42,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure; private final DecoderInputBuffer decoderInputBuffer; private final Codec decoder; - @Nullable private final FrameProcessorChain frameProcessorChain; + private final FrameProcessorChain frameProcessorChain; private final Codec encoder; private final DecoderInputBuffer encoderOutputBuffer; @@ -70,6 +70,7 @@ import org.checkerframework.dataflow.qual.Pure; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; + // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset. ScaleToFitFrameProcessor scaleToFitFrameProcessor = new ScaleToFitFrameProcessor.Builder(context) .setScale(transformationRequest.scaleX, transformationRequest.scaleY) @@ -109,36 +110,24 @@ import org.checkerframework.dataflow.qual.Pure; requestedEncoderFormat, encoderSupportedFormat)); - if (transformationRequest.enableHdrEditing - || inputFormat.height != encoderSupportedFormat.height - || inputFormat.width != encoderSupportedFormat.width - || scaleToFitFrameProcessor.shouldProcess() - || presentationFrameProcessor.shouldProcess() - || shouldAlwaysUseFrameProcessorChain()) { - // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size - // has to change due to encoder fallback or append another GlFrameProcessor. - frameProcessorSizes.set( - frameProcessorSizes.size() - 1, - new Size(encoderSupportedFormat.width, encoderSupportedFormat.height)); - frameProcessorChain = - FrameProcessorChain.create( - context, - inputFormat.pixelWidthHeightRatio, - frameProcessors, - frameProcessorSizes, - /* outputSurface= */ encoder.getInputSurface(), - transformationRequest.enableHdrEditing, - debugViewProvider); - } else { - frameProcessorChain = null; - } + // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size + // has to change due to encoder fallback or append another GlFrameProcessor. + frameProcessorSizes.set( + frameProcessorSizes.size() - 1, + new Size(encoderSupportedFormat.width, encoderSupportedFormat.height)); + frameProcessorChain = + FrameProcessorChain.create( + context, + inputFormat.pixelWidthHeightRatio, + frameProcessors, + frameProcessorSizes, + /* outputSurface= */ encoder.getInputSurface(), + transformationRequest.enableHdrEditing, + debugViewProvider); decoder = decoderFactory.createForVideoDecoding( - inputFormat, - frameProcessorChain == null - ? encoder.getInputSurface() - : frameProcessorChain.createInputSurface()); + inputFormat, frameProcessorChain.createInputSurface()); } @Override @@ -154,15 +143,13 @@ import org.checkerframework.dataflow.qual.Pure; @Override public boolean processData() throws TransformationException { - if (frameProcessorChain != null) { - frameProcessorChain.getAndRethrowBackgroundExceptions(); - if (frameProcessorChain.isEnded()) { - if (!signaledEndOfStreamToEncoder) { - encoder.signalEndOfInputStream(); - signaledEndOfStreamToEncoder = true; - } - return false; + frameProcessorChain.getAndRethrowBackgroundExceptions(); + if (frameProcessorChain.isEnded()) { + if (!signaledEndOfStreamToEncoder) { + encoder.signalEndOfInputStream(); + signaledEndOfStreamToEncoder = true; } + return false; } if (decoder.isEnded()) { return false; @@ -178,13 +165,7 @@ import org.checkerframework.dataflow.qual.Pure; canProcessMoreDataImmediately = processDataDefault(); } if (decoder.isEnded()) { - if (frameProcessorChain != null) { - frameProcessorChain.signalEndOfInputStream(); - } else { - encoder.signalEndOfInputStream(); - signaledEndOfStreamToEncoder = true; - return false; - } + frameProcessorChain.signalEndOfInputStream(); } return canProcessMoreDataImmediately; } @@ -215,7 +196,7 @@ import org.checkerframework.dataflow.qual.Pure; */ private boolean processDataDefault() throws TransformationException { // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29. - if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) { + if (frameProcessorChain.hasPendingFrames()) { return false; } return maybeProcessDecoderOutput(); @@ -255,9 +236,7 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void release() { - if (frameProcessorChain != null) { - frameProcessorChain.release(); - } + frameProcessorChain.release(); decoder.release(); encoder.release(); } @@ -292,17 +271,6 @@ import org.checkerframework.dataflow.qual.Pure; .build(); } - /** Always use {@link FrameProcessorChain} to work around device-specific encoder issues. */ - private static boolean shouldAlwaysUseFrameProcessorChain() { - switch (Util.MODEL) { - case "XT1635-02": - case "Nexus 5": - return true; - default: - return false; - } - } - /** * Feeds at most one decoder output frame to the next step of the pipeline. * @@ -314,9 +282,7 @@ import org.checkerframework.dataflow.qual.Pure; return false; } - if (frameProcessorChain != null) { - frameProcessorChain.registerInputFrame(); - } + frameProcessorChain.registerInputFrame(); decoder.releaseOutputBuffer(/* render= */ true); return true; } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java index 84fa0717ae..0703a2468f 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java @@ -42,7 +42,6 @@ public final class PresentationFrameProcessorTest { Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); - assertThat(presentationFrameProcessor.shouldProcess()).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -57,7 +56,6 @@ public final class PresentationFrameProcessorTest { Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); - assertThat(presentationFrameProcessor.shouldProcess()).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -72,7 +70,6 @@ public final class PresentationFrameProcessorTest { Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); - assertThat(presentationFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputWidth); } @@ -90,7 +87,6 @@ public final class PresentationFrameProcessorTest { Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); - assertThat(presentationFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight); assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java index 96b9485baa..6fd582b74f 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java @@ -42,7 +42,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -69,7 +68,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f)); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -85,7 +83,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -101,7 +98,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2); } @@ -117,7 +113,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputWidth); } @@ -134,7 +129,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight); assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight); }