FrameProcessor: Create EncoderCompatibilityFrameProcessor.

Split rotationDegrees changes to EncoderCompatibilityFrameProcessor, a new
FrameProcessor.

This removes automatic rotationDegrees adjustments from Presentation, which
allows Presentation to be used for changes before the end of a
FrameProcessorChain pipeline.

PiperOrigin-RevId: 443387226

libraries/transformer/src/main/java/androidx/media3/transformer/EncoderCompatibilityFrameProcessor.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
+
+import android.content.Context;
+import android.util.Size;
+import androidx.annotation.VisibleForTesting;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.util.GlUtil;
+import java.io.IOException;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
+/**
+ * Copies frames from a texture and applies {@link Format#rotationDegrees} for encoder
+ * compatibility, if needed.
+ *
+ * <p>Encoders commonly support higher maximum widths than maximum heights. This may rotate the
+ * decoded frame before encoding, so the encoded frame's width >= height, and set {@link
+ * Format#rotationDegrees} to ensure the frame is displayed in the correct orientation.
+ */
+/* package */ class EncoderCompatibilityFrameProcessor implements GlFrameProcessor {
+  // TODO(b/218488308): Allow reconfiguration of the output size, as encoders may not support the
+  //  requested output resolution.
+
+  static {
+    GlUtil.glAssertionsEnabled = true;
+  }
+
+  private int outputRotationDegrees;
+  private @MonotonicNonNull ScaleToFitFrameProcessor rotateFrameProcessor;
+
+  /** Creates a new instance. */
+  /* package */ EncoderCompatibilityFrameProcessor() {
+
+    outputRotationDegrees = C.LENGTH_UNSET;
+  }
+
+  @Override
+  public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
+      throws IOException {
+    configureOutputSizeAndRotation(inputWidth, inputHeight);
+    rotateFrameProcessor =
+        new ScaleToFitFrameProcessor.Builder().setRotationDegrees(outputRotationDegrees).build();
+    rotateFrameProcessor.initialize(context, inputTexId, inputWidth, inputHeight);
+  }
+
+  @Override
+  public Size getOutputSize() {
+    return checkStateNotNull(rotateFrameProcessor).getOutputSize();
+  }
+
+  /**
+   * Returns {@link Format#rotationDegrees} for the output frame.
+   *
+   * <p>Return values may be {@code 0} or {@code 90} degrees.
+   *
+   * <p>The frame processor must be {@linkplain GlFrameProcessor#initialize(Context, int, int, int)
+   * initialized}.
+   */
+  public int getOutputRotationDegrees() {
+    checkState(
+        outputRotationDegrees != C.LENGTH_UNSET,
+        "configureOutputSizeAndTransformationMatrix must be called before"
+            + " getOutputRotationDegrees");
+    return outputRotationDegrees;
+  }
+
+  @Override
+  public void drawFrame(long presentationTimeUs) {
+    checkStateNotNull(rotateFrameProcessor).drawFrame(presentationTimeUs);
+  }
+
+  @Override
+  public void release() {
+    if (rotateFrameProcessor != null) {
+      rotateFrameProcessor.release();
+    }
+  }
+
+  @VisibleForTesting // Allows robolectric testing of output size calculation without OpenGL.
+  /* package */ void configureOutputSizeAndRotation(int inputWidth, int inputHeight) {
+    checkArgument(inputWidth > 0, "inputWidth must be positive");
+    checkArgument(inputHeight > 0, "inputHeight must be positive");
+
+    outputRotationDegrees = (inputHeight > inputWidth) ? 90 : 0;
+  }
+}

libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java

@@ -27,7 +27,6 @@ import android.util.Size;
 import androidx.annotation.IntDef;
 import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.C;
-import androidx.media3.common.Format;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.UnstableApi;
 import java.io.IOException;
@@ -234,7 +233,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
   private final float requestedAspectRatio;
   private final @Layout int layout;
 
-  private int outputRotationDegrees;
   private int outputWidth;
   private int outputHeight;
   private @MonotonicNonNull Matrix transformationMatrix;
@@ -259,7 +257,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
 
     outputWidth = C.LENGTH_UNSET;
     outputHeight = C.LENGTH_UNSET;
-    outputRotationDegrees = C.LENGTH_UNSET;
     transformationMatrix = new Matrix();
   }
 
@@ -279,22 +276,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
     return new Size(outputWidth, outputHeight);
   }
 
-  /**
-   * Returns {@link Format#rotationDegrees} for the output frame.
-   *
-   * <p>Return values may be {@code 0} or {@code 90} degrees.
-   *
-   * <p>The frame processor must be {@linkplain GlFrameProcessor#initialize(Context, int, int, int)
-   * initialized}.
-   */
-  public int getOutputRotationDegrees() {
-    checkState(
-        outputRotationDegrees != C.LENGTH_UNSET,
-        "configureOutputSizeAndTransformationMatrix must be called before"
-            + " getOutputRotationDegrees");
-    return outputRotationDegrees;
-  }
-
   @Override
   public void drawFrame(long presentationTimeUs) {
     checkStateNotNull(advancedFrameProcessor).drawFrame(presentationTimeUs);
@@ -331,20 +312,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
       outputWidth = Math.round((float) requestedHeightPixels * outputWidth / outputHeight);
       outputHeight = requestedHeightPixels;
     }
-
-    // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
-    // frame before encoding, so the encoded frame's width >= height, and set
-    // outputRotationDegrees to ensure the frame is displayed in the correct orientation.
-    if (outputHeight > outputWidth) {
-      outputRotationDegrees = 90;
-      // TODO(b/201293185): Put postRotate in a later GlFrameProcessor.
-      transformationMatrix.postRotate(outputRotationDegrees);
-      int swap = outputWidth;
-      outputWidth = outputHeight;
-      outputHeight = swap;
-    } else {
-      outputRotationDegrees = 0;
-    }
   }
 
   @RequiresNonNull("transformationMatrix")

libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java

@@ -70,7 +70,8 @@ import org.checkerframework.dataflow.qual.Pure;
     int decodedHeight =
         (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
 
-    // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset.
+    // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset,
+    // and don't create a PresentationFrameProcessor if resolution is unset.
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
         new ScaleToFitFrameProcessor.Builder()
             .setScale(transformationRequest.scaleX, transformationRequest.scaleY)
@@ -80,6 +81,8 @@ import org.checkerframework.dataflow.qual.Pure;
         new PresentationFrameProcessor.Builder()
             .setResolution(transformationRequest.outputHeight)
             .build();
+    EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
+        new EncoderCompatibilityFrameProcessor();
     frameProcessorChain =
         FrameProcessorChain.create(
             context,
@@ -90,10 +93,11 @@ import org.checkerframework.dataflow.qual.Pure;
                 .addAll(frameProcessors)
                 .add(scaleToFitFrameProcessor)
                 .add(presentationFrameProcessor)
+                .add(encoderCompatibilityFrameProcessor)
                 .build(),
             transformationRequest.enableHdrEditing);
     Size requestedEncoderSize = frameProcessorChain.getOutputSize();
-    outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees();
+    outputRotationDegrees = encoderCompatibilityFrameProcessor.getOutputRotationDegrees();
 
     Format requestedEncoderFormat =
         new Format.Builder()

libraries/transformer/src/test/java/androidx/media3/transformer/EncoderCompatibilityFrameProcessorTest.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit tests for {@link EncoderCompatibilityFrameProcessor}. */
+@RunWith(AndroidJUnit4.class)
+public final class EncoderCompatibilityFrameProcessorTest {
+  @Test
+  public void getOutputSize_noEditsLandscape_leavesOrientationUnchanged() {
+    int inputWidth = 200;
+    int inputHeight = 150;
+    EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
+        new EncoderCompatibilityFrameProcessor();
+
+    encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight);
+
+    assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
+  }
+
+  @Test
+  public void getOutputSize_noEditsSquare_leavesOrientationUnchanged() {
+    int inputWidth = 150;
+    int inputHeight = 150;
+    EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
+        new EncoderCompatibilityFrameProcessor();
+
+    encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight);
+
+    assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
+  }
+
+  @Test
+  public void getOutputSize_noEditsPortrait_flipsOrientation() {
+    int inputWidth = 150;
+    int inputHeight = 200;
+    EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
+        new EncoderCompatibilityFrameProcessor();
+
+    encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight);
+
+    assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
+  }
+
+  @Test
+  public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
+    EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
+        new EncoderCompatibilityFrameProcessor();
+
+    // configureOutputSize not called before getOutputRotationDegrees.
+    assertThrows(
+        IllegalStateException.class, encoderCompatibilityFrameProcessor::getOutputRotationDegrees);
+  }
+}

libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java

@@ -27,13 +27,13 @@ import org.junit.runner.RunWith;
 /**
  * Unit tests for {@link PresentationFrameProcessor}.
  *
- * <p>See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link
+ * <p>See {@code PresentationFrameProcessorPixelTest} for pixel tests testing {@link
- * AdvancedFrameProcessor} given a transformation matrix.
+ * PresentationFrameProcessor}.
  */
 @RunWith(AndroidJUnit4.class)
 public final class PresentationFrameProcessorTest {
   @Test
-  public void getOutputSize_noEditsLandscape_leavesFramesUnchanged() {
+  public void getOutputSize_noEdits_leavesFramesUnchanged() {
     int inputWidth = 200;
     int inputHeight = 150;
     PresentationFrameProcessor presentationFrameProcessor =
@@ -42,41 +42,10 @@ public final class PresentationFrameProcessorTest {
     presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
     Size outputSize = presentationFrameProcessor.getOutputSize();
 
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
 
-  @Test
-  public void getOutputSize_noEditsSquare_leavesFramesUnchanged() {
-    int inputWidth = 150;
-    int inputHeight = 150;
-    PresentationFrameProcessor presentationFrameProcessor =
-        new PresentationFrameProcessor.Builder().build();
-
-    presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
-    Size outputSize = presentationFrameProcessor.getOutputSize();
-
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
-    assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
-    assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
-  }
-
-  @Test
-  public void getOutputSize_noEditsPortrait_flipsOrientation() {
-    int inputWidth = 150;
-    int inputHeight = 200;
-    PresentationFrameProcessor presentationFrameProcessor =
-        new PresentationFrameProcessor.Builder().build();
-
-    presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
-    Size outputSize = presentationFrameProcessor.getOutputSize();
-
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
-    assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
-    assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
-  }
-
   @Test
   public void getOutputSize_setResolution_changesDimensions() {
     int inputWidth = 200;
@@ -88,7 +57,6 @@ public final class PresentationFrameProcessorTest {
     presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
     Size outputSize = presentationFrameProcessor.getOutputSize();
 
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight);
     assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
   }
@@ -107,7 +75,6 @@ public final class PresentationFrameProcessorTest {
     presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
     Size outputSize = presentationFrameProcessor.getOutputSize();
 
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
     int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
     assertThat(outputSize.getWidth()).isEqualTo(expectedPostCropWidth);
@@ -132,7 +99,6 @@ public final class PresentationFrameProcessorTest {
     presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
     Size outputSize = presentationFrameProcessor.getOutputSize();
 
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
     int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
     assertThat(outputSize.getWidth())
@@ -159,7 +125,6 @@ public final class PresentationFrameProcessorTest {
     presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
     Size outputSize = presentationFrameProcessor.getOutputSize();
 
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
     int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
     assertThat(outputSize.getWidth())
@@ -181,7 +146,6 @@ public final class PresentationFrameProcessorTest {
     presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
     Size outputSize = presentationFrameProcessor.getOutputSize();
 
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * inputHeight));
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
@@ -201,7 +165,6 @@ public final class PresentationFrameProcessorTest {
     presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
     Size outputSize = presentationFrameProcessor.getOutputSize();
 
-    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * requestedHeight));
     assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
   }
@@ -231,13 +194,4 @@ public final class PresentationFrameProcessorTest {
             presentationFrameProcessor.setAspectRatio(
                 /* aspectRatio= */ 2f, PresentationFrameProcessor.LAYOUT_SCALE_TO_FIT));
   }
-
-  @Test
-  public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
-    PresentationFrameProcessor presentationFrameProcessor =
-        new PresentationFrameProcessor.Builder().build();
-
-    // configureOutputSize not called before getOutputRotationDegrees.
-    assertThrows(IllegalStateException.class, presentationFrameProcessor::getOutputRotationDegrees);
-  }
 }