Transformer: Always use FrameProcessorChain when decoding.

This allows us to bypass many device-specific issues, that only occur when
decoding directly to an encoder surface, without OpenGL. This also allows us
to maintain fewer code branches, which require additional testing to verify
correctness.

PiperOrigin-RevId: 437003138

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
 import static com.google.common.truth.Truth.assertThat;
 
 import android.content.Context;
-import androidx.media3.common.MimeTypes;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import org.junit.Test;
@@ -32,37 +31,8 @@ import org.junit.runner.RunWith;
 @RunWith(AndroidJUnit4.class)
 public class TransformerEndToEndTest {
 
-  private static final String VP9_VIDEO_URI_STRING = "asset:///media/vp9/bear-vp9.webm";
   private static final String AVC_VIDEO_URI_STRING = "asset:///media/mp4/sample.mp4";
 
-  @Test
-  public void videoTranscoding_completesWithConsistentFrameCount() throws Exception {
-    Context context = ApplicationProvider.getApplicationContext();
-    FrameCountingMuxer.Factory muxerFactory =
-        new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory());
-    Transformer transformer =
-        new Transformer.Builder(context)
-            .setTransformationRequest(
-                new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H264).build())
-            .setMuxerFactory(muxerFactory)
-            .setEncoderFactory(
-                new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false))
-            .build();
-    // Result of the following command:
-    // ffprobe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames bear-vp9.webm
-    int expectedFrameCount = 82;
-
-    new TransformerAndroidTestRunner.Builder(context, transformer)
-        .build()
-        .run(
-            /* testId= */ "videoTranscoding_completesWithConsistentFrameCount",
-            VP9_VIDEO_URI_STRING);
-
-    FrameCountingMuxer frameCountingMuxer =
-        checkNotNull(muxerFactory.getLastFrameCountingMuxerCreated());
-    assertThat(frameCountingMuxer.getFrameCount()).isEqualTo(expectedFrameCount);
-  }
-
   @Test
   public void videoEditing_completesWithConsistentFrameCount() throws Exception {
     Context context = ApplicationProvider.getApplicationContext();

libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java

@@ -81,7 +81,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor;
   private int inputWidth;
   private int inputHeight;
-  private int outputHeight;
   private int outputRotationDegrees;
   private @MonotonicNonNull Matrix transformationMatrix;
 
@@ -97,7 +96,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
     inputWidth = C.LENGTH_UNSET;
     inputHeight = C.LENGTH_UNSET;
-    outputHeight = C.LENGTH_UNSET;
     outputRotationDegrees = C.LENGTH_UNSET;
   }
 
@@ -113,18 +111,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     return outputRotationDegrees;
   }
 
-  /**
-   * Returns whether this {@code PresentationFrameProcessor} will apply any changes on a frame.
-   *
-   * <p>The {@code PresentationFrameProcessor} should only be used if this returns true.
-   *
-   * <p>This method can only be called after {@link #configureOutputSize(int, int)}.
-   */
-  public boolean shouldProcess() {
-    checkStateNotNull(transformationMatrix);
-    return inputHeight != outputHeight || !transformationMatrix.isIdentity();
-  }
-
   @Override
   public Size configureOutputSize(int inputWidth, int inputHeight) {
     this.inputWidth = inputWidth;
@@ -140,24 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       displayHeight = requestedHeight;
     }
 
-    int outputWidth;
     // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
     // frame before encoding, so the encoded frame's width >= height, and set
     // outputRotationDegrees to ensure the frame is displayed in the correct orientation.
     if (displayHeight > displayWidth) {
       outputRotationDegrees = 90;
-      outputWidth = displayHeight;
-      outputHeight = displayWidth;
-      // TODO(b/201293185): After fragment shader transformations are implemented, put postRotate in
-      //  a later GlFrameProcessor.
+      // TODO(b/201293185): After fragment shader transformations are implemented, put
+      //  postRotate in a later GlFrameProcessor.
       transformationMatrix.postRotate(outputRotationDegrees);
+      return new Size(displayHeight, displayWidth);
     } else {
       outputRotationDegrees = 0;
-      outputWidth = displayWidth;
-      outputHeight = displayHeight;
+      return new Size(displayWidth, displayHeight);
     }
-
-    return new Size(outputWidth, outputHeight);
   }
 
   @Override

libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java

@@ -122,18 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     inputHeight = C.LENGTH_UNSET;
   }
 
-  /**
-   * Returns whether this ScaleToFitFrameProcessor will apply any changes on a frame.
-   *
-   * <p>The ScaleToFitFrameProcessor should only be used if this returns true.
-   *
-   * <p>This method can only be called after {@link #configureOutputSize(int, int)}.
-   */
-  public boolean shouldProcess() {
-    checkStateNotNull(adjustedTransformationMatrix);
-    return !transformationMatrix.isIdentity();
-  }
-
   @Override
   public Size configureOutputSize(int inputWidth, int inputHeight) {
     this.inputWidth = inputWidth;

libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java

@@ -42,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure;
   private final DecoderInputBuffer decoderInputBuffer;
   private final Codec decoder;
 
-  @Nullable private final FrameProcessorChain frameProcessorChain;
+  private final FrameProcessorChain frameProcessorChain;
 
   private final Codec encoder;
   private final DecoderInputBuffer encoderOutputBuffer;
@@ -70,6 +70,7 @@ import org.checkerframework.dataflow.qual.Pure;
     int decodedHeight =
         (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
 
+    // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset.
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
         new ScaleToFitFrameProcessor.Builder(context)
             .setScale(transformationRequest.scaleX, transformationRequest.scaleY)
@@ -109,36 +110,24 @@ import org.checkerframework.dataflow.qual.Pure;
             requestedEncoderFormat,
             encoderSupportedFormat));
 
-    if (transformationRequest.enableHdrEditing
-        || inputFormat.height != encoderSupportedFormat.height
-        || inputFormat.width != encoderSupportedFormat.width
-        || scaleToFitFrameProcessor.shouldProcess()
-        || presentationFrameProcessor.shouldProcess()
-        || shouldAlwaysUseFrameProcessorChain()) {
-      // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
-      //  has to change due to encoder fallback or append another GlFrameProcessor.
-      frameProcessorSizes.set(
-          frameProcessorSizes.size() - 1,
-          new Size(encoderSupportedFormat.width, encoderSupportedFormat.height));
-      frameProcessorChain =
-          FrameProcessorChain.create(
-              context,
-              inputFormat.pixelWidthHeightRatio,
-              frameProcessors,
-              frameProcessorSizes,
-              /* outputSurface= */ encoder.getInputSurface(),
-              transformationRequest.enableHdrEditing,
-              debugViewProvider);
-    } else {
-      frameProcessorChain = null;
-    }
+    // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
+    //  has to change due to encoder fallback or append another GlFrameProcessor.
+    frameProcessorSizes.set(
+        frameProcessorSizes.size() - 1,
+        new Size(encoderSupportedFormat.width, encoderSupportedFormat.height));
+    frameProcessorChain =
+        FrameProcessorChain.create(
+            context,
+            inputFormat.pixelWidthHeightRatio,
+            frameProcessors,
+            frameProcessorSizes,
+            /* outputSurface= */ encoder.getInputSurface(),
+            transformationRequest.enableHdrEditing,
+            debugViewProvider);
 
     decoder =
         decoderFactory.createForVideoDecoding(
-            inputFormat,
-            frameProcessorChain == null
-                ? encoder.getInputSurface()
-                : frameProcessorChain.createInputSurface());
+            inputFormat, frameProcessorChain.createInputSurface());
   }
 
   @Override
@@ -154,15 +143,13 @@ import org.checkerframework.dataflow.qual.Pure;
 
   @Override
   public boolean processData() throws TransformationException {
-    if (frameProcessorChain != null) {
-      frameProcessorChain.getAndRethrowBackgroundExceptions();
-      if (frameProcessorChain.isEnded()) {
-        if (!signaledEndOfStreamToEncoder) {
-          encoder.signalEndOfInputStream();
-          signaledEndOfStreamToEncoder = true;
-        }
-        return false;
+    frameProcessorChain.getAndRethrowBackgroundExceptions();
+    if (frameProcessorChain.isEnded()) {
+      if (!signaledEndOfStreamToEncoder) {
+        encoder.signalEndOfInputStream();
+        signaledEndOfStreamToEncoder = true;
       }
+      return false;
     }
     if (decoder.isEnded()) {
       return false;
@@ -178,13 +165,7 @@ import org.checkerframework.dataflow.qual.Pure;
       canProcessMoreDataImmediately = processDataDefault();
     }
     if (decoder.isEnded()) {
-      if (frameProcessorChain != null) {
-        frameProcessorChain.signalEndOfInputStream();
-      } else {
-        encoder.signalEndOfInputStream();
-        signaledEndOfStreamToEncoder = true;
-        return false;
-      }
+      frameProcessorChain.signalEndOfInputStream();
     }
     return canProcessMoreDataImmediately;
   }
@@ -215,7 +196,7 @@ import org.checkerframework.dataflow.qual.Pure;
    */
   private boolean processDataDefault() throws TransformationException {
     // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
-    if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) {
+    if (frameProcessorChain.hasPendingFrames()) {
       return false;
     }
     return maybeProcessDecoderOutput();
@@ -255,9 +236,7 @@ import org.checkerframework.dataflow.qual.Pure;
 
   @Override
   public void release() {
-    if (frameProcessorChain != null) {
-      frameProcessorChain.release();
-    }
+    frameProcessorChain.release();
     decoder.release();
     encoder.release();
   }
@@ -292,17 +271,6 @@ import org.checkerframework.dataflow.qual.Pure;
         .build();
   }
 
-  /** Always use {@link FrameProcessorChain} to work around device-specific encoder issues. */
-  private static boolean shouldAlwaysUseFrameProcessorChain() {
-    switch (Util.MODEL) {
-      case "XT1635-02":
-      case "Nexus 5":
-        return true;
-      default:
-        return false;
-    }
-  }
-
   /**
    * Feeds at most one decoder output frame to the next step of the pipeline.
    *
@@ -314,9 +282,7 @@ import org.checkerframework.dataflow.qual.Pure;
       return false;
     }
 
-    if (frameProcessorChain != null) {
-      frameProcessorChain.registerInputFrame();
-    }
+    frameProcessorChain.registerInputFrame();
     decoder.releaseOutputBuffer(/* render= */ true);
     return true;
   }

libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java

@@ -42,7 +42,6 @@ public final class PresentationFrameProcessorTest {
     Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
     assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
-    assertThat(presentationFrameProcessor.shouldProcess()).isFalse();
     assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
@@ -57,7 +56,6 @@ public final class PresentationFrameProcessorTest {
     Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
     assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
-    assertThat(presentationFrameProcessor.shouldProcess()).isFalse();
     assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
@@ -72,7 +70,6 @@ public final class PresentationFrameProcessorTest {
     Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
     assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
-    assertThat(presentationFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
     assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
   }
@@ -90,7 +87,6 @@ public final class PresentationFrameProcessorTest {
     Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
     assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
-    assertThat(presentationFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight);
     assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
   }

libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java

@@ -42,7 +42,6 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.shouldProcess()).isFalse();
     assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
@@ -69,7 +68,6 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f));
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
@@ -85,7 +83,6 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2);
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
@@ -101,7 +98,6 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2);
   }
@@ -117,7 +113,6 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
     assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
   }
@@ -134,7 +129,6 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight);
     assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight);
   }