Transformer GL: Add support for pixelWidthHeightRatio.

To ensure frame processor operations operate on square pixels,
make the frame taller or wider for non-square input pixels.

In addition to automated tests, this was tested by changing the
inputFormat.pixelWidthHeightRatio in the TransformerVideoRenderer.

PiperOrigin-RevId: 444553517

libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java

@@ -53,6 +53,8 @@ import org.junit.runner.RunWith;
 public final class FrameProcessorChainPixelTest {
   public static final String ORIGINAL_PNG_ASSET_PATH =
       "media/bitmap/sample_mp4_first_frame/original.png";
+  public static final String SCALE_WIDE_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/scale_wide.png";
   public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH =
       "media/bitmap/sample_mp4_first_frame/translate_right.png";
   public static final String ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH =
@@ -74,7 +76,7 @@ public final class FrameProcessorChainPixelTest {
    */
   private static final int FRAME_PROCESSING_WAIT_MS = 5000;
   /** The ratio of width over height, for each pixel in a frame. */
-  private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
+  private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1;
 
   private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
   private @MonotonicNonNull ImageReader outputImageReader;
@@ -90,7 +92,7 @@ public final class FrameProcessorChainPixelTest {
   @Test
   public void processData_noEdits_producesExpectedOutput() throws Exception {
     String testId = "processData_noEdits";
-    setUpAndPrepareFirstFrame();
+    setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO);
     Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
 
     Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -104,6 +106,23 @@ public final class FrameProcessorChainPixelTest {
     assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
   }
 
+  @Test
+  public void processData_withPixelWidthHeightRatio_producesExpectedOutput() throws Exception {
+    String testId = "processData_withPixelWidthHeightRatio";
+    setUpAndPrepareFirstFrame(/* pixelWidthHeightRatio= */ 2f);
+    Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_WIDE_PNG_ASSET_PATH);
+
+    Bitmap actualBitmap = processFirstFrameAndEnd();
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "actual", actualBitmap);
+    // TODO(b/207848601): switch to using proper tooling for testing against golden data.
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
   @Test
   public void processData_withAdvancedFrameProcessor_translateRight_producesExpectedOutput()
       throws Exception {
@@ -111,7 +130,7 @@ public final class FrameProcessorChainPixelTest {
     Matrix translateRightMatrix = new Matrix();
     translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
     GlFrameProcessor glFrameProcessor = new AdvancedFrameProcessor(translateRightMatrix);
-    setUpAndPrepareFirstFrame(glFrameProcessor);
+    setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
     Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
 
     Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -135,7 +154,8 @@ public final class FrameProcessorChainPixelTest {
         new AdvancedFrameProcessor(translateRightMatrix);
     GlFrameProcessor rotate45FrameProcessor =
         new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build();
-    setUpAndPrepareFirstFrame(translateRightFrameProcessor, rotate45FrameProcessor);
+    setUpAndPrepareFirstFrame(
+        DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, translateRightFrameProcessor, rotate45FrameProcessor);
     Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH);
 
     Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -159,7 +179,8 @@ public final class FrameProcessorChainPixelTest {
     translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
     GlFrameProcessor translateRightFrameProcessor =
         new AdvancedFrameProcessor(translateRightMatrix);
-    setUpAndPrepareFirstFrame(rotate45FrameProcessor, translateRightFrameProcessor);
+    setUpAndPrepareFirstFrame(
+        DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, rotate45FrameProcessor, translateRightFrameProcessor);
     Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH);
 
     Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -179,7 +200,7 @@ public final class FrameProcessorChainPixelTest {
     String testId = "processData_withPresentationFrameProcessor_setResolution";
     GlFrameProcessor glFrameProcessor =
         new PresentationFrameProcessor.Builder().setResolution(480).build();
-    setUpAndPrepareFirstFrame(glFrameProcessor);
+    setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
     Bitmap expectedBitmap = BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH);
 
     Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -199,7 +220,7 @@ public final class FrameProcessorChainPixelTest {
     String testId = "processData_withScaleToFitFrameProcessor_rotate45";
     GlFrameProcessor glFrameProcessor =
         new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build();
-    setUpAndPrepareFirstFrame(glFrameProcessor);
+    setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
     Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH);
 
     Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -218,10 +239,12 @@ public final class FrameProcessorChainPixelTest {
    * infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
    * accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}.
    *
+   * @param pixelWidthHeightRatio The ratio of width over height for each pixel.
    * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes
    *     to the input frame.
    */
-  private void setUpAndPrepareFirstFrame(GlFrameProcessor... frameProcessors) throws Exception {
+  private void setUpAndPrepareFirstFrame(
+      float pixelWidthHeightRatio, GlFrameProcessor... frameProcessors) throws Exception {
     // Set up the extractor to read the first video frame and get its format.
     MediaExtractor mediaExtractor = new MediaExtractor();
     @Nullable MediaCodec mediaCodec = null;
@@ -241,7 +264,7 @@ public final class FrameProcessorChainPixelTest {
       frameProcessorChain =
           FrameProcessorChain.create(
               context,
-              PIXEL_WIDTH_HEIGHT_RATIO,
+              pixelWidthHeightRatio,
               inputWidth,
               inputHeight,
               asList(frameProcessors),

libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java

@@ -17,7 +17,6 @@ package androidx.media3.transformer;
 
 import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
 import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
 
 import android.content.Context;
 import android.util.Size;
@@ -36,46 +35,13 @@ import org.junit.runner.RunWith;
 public final class FrameProcessorChainTest {
 
   @Test
-  public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully()
+  public void getOutputSize_noOperation_returnsInputSize() throws Exception {
-      throws TransformationException {
-    Context context = getApplicationContext();
-
-    FrameProcessorChain.create(
-        context,
-        /* pixelWidthHeightRatio= */ 1,
-        /* inputWidth= */ 200,
-        /* inputHeight= */ 100,
-        /* frameProcessors= */ ImmutableList.of(),
-        /* enableExperimentalHdrEditing= */ false);
-  }
-
-  @Test
-  public void create_withUnsupportedPixelWidthHeightRatio_throwsException() {
-    Context context = getApplicationContext();
-
-    TransformationException exception =
-        assertThrows(
-            TransformationException.class,
-            () ->
-                FrameProcessorChain.create(
-                    context,
-                    /* pixelWidthHeightRatio= */ 2,
-                    /* inputWidth= */ 200,
-                    /* inputHeight= */ 100,
-                    /* frameProcessors= */ ImmutableList.of(),
-                    /* enableExperimentalHdrEditing= */ false));
-
-    assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class);
-    assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio");
-  }
-
-  @Test
-  public void getOutputSize_withoutFrameProcessors_returnsInputSize()
-      throws TransformationException {
     Size inputSize = new Size(200, 100);
     FrameProcessorChain frameProcessorChain =
         createFrameProcessorChainWithFakeFrameProcessors(
-            inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of());
+            /* pixelWidthHeightRatio= */ 1f,
+            inputSize,
+            /* frameProcessorOutputSizes= */ ImmutableList.of());
 
     Size outputSize = frameProcessorChain.getOutputSize();
 
@@ -83,13 +49,42 @@ public final class FrameProcessorChainTest {
   }
 
   @Test
-  public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize()
+  public void getOutputSize_withWidePixels_returnsWiderOutputSize() throws Exception {
-      throws TransformationException {
+    Size inputSize = new Size(200, 100);
+    FrameProcessorChain frameProcessorChain =
+        createFrameProcessorChainWithFakeFrameProcessors(
+            /* pixelWidthHeightRatio= */ 2f,
+            inputSize,
+            /* frameProcessorOutputSizes= */ ImmutableList.of());
+
+    Size outputSize = frameProcessorChain.getOutputSize();
+
+    assertThat(outputSize).isEqualTo(new Size(400, 100));
+  }
+
+  @Test
+  public void getOutputSize_withTallPixels_returnsTallerOutputSize() throws Exception {
+    Size inputSize = new Size(200, 100);
+    FrameProcessorChain frameProcessorChain =
+        createFrameProcessorChainWithFakeFrameProcessors(
+            /* pixelWidthHeightRatio= */ .5f,
+            inputSize,
+            /* frameProcessorOutputSizes= */ ImmutableList.of());
+
+    Size outputSize = frameProcessorChain.getOutputSize();
+
+    assertThat(outputSize).isEqualTo(new Size(200, 200));
+  }
+
+  @Test
+  public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize() throws Exception {
     Size inputSize = new Size(200, 100);
     Size frameProcessorOutputSize = new Size(300, 250);
     FrameProcessorChain frameProcessorChain =
         createFrameProcessorChainWithFakeFrameProcessors(
-            inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize));
+            /* pixelWidthHeightRatio= */ 1f,
+            inputSize,
+            /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize));
 
     Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
 
@@ -97,14 +92,14 @@ public final class FrameProcessorChainTest {
   }
 
   @Test
-  public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize()
+  public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize() throws Exception {
-      throws TransformationException {
     Size inputSize = new Size(200, 100);
     Size outputSize1 = new Size(300, 250);
     Size outputSize2 = new Size(400, 244);
     Size outputSize3 = new Size(150, 160);
     FrameProcessorChain frameProcessorChain =
         createFrameProcessorChainWithFakeFrameProcessors(
+            /* pixelWidthHeightRatio= */ 1f,
             inputSize,
             /* frameProcessorOutputSizes= */ ImmutableList.of(
                 outputSize1, outputSize2, outputSize3));
@@ -115,14 +110,15 @@ public final class FrameProcessorChainTest {
   }
 
   private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
-      Size inputSize, List<Size> frameProcessorOutputSizes) throws TransformationException {
+      float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes)
+      throws TransformationException {
     ImmutableList.Builder<GlFrameProcessor> frameProcessors = new ImmutableList.Builder<>();
     for (Size element : frameProcessorOutputSizes) {
       frameProcessors.add(new FakeFrameProcessor(element));
     }
     return FrameProcessorChain.create(
         getApplicationContext(),
-        /* pixelWidthHeightRatio= */ 1,
+        pixelWidthHeightRatio,
         inputSize.getWidth(),
         inputSize.getHeight(),
         frameProcessors.build(),

libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java

@@ -71,15 +71,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
    * Creates a new instance.
    *
    * @param context A {@link Context}.
-   * @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
+   * @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded
+   *     by this ratio so that the output frame's pixels have a ratio of 1.
    * @param inputWidth The input frame width, in pixels.
    * @param inputHeight The input frame height, in pixels.
    * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
    * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
    * @return A new instance.
-   * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader
+   * @throws TransformationException If reading shader files fails, or an OpenGL error occurs while
-   *     files fails, or an OpenGL error occurs while creating and configuring the OpenGL
+   *     creating and configuring the OpenGL components.
-   *     components.
    */
   public static FrameProcessorChain create(
       Context context,
@@ -92,19 +92,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     checkArgument(inputWidth > 0, "inputWidth must be positive");
     checkArgument(inputHeight > 0, "inputHeight must be positive");
 
-    if (pixelWidthHeightRatio != 1.0f) {
-      // TODO(b/211782176): Consider implementing support for non-square pixels.
-      throw TransformationException.createForFrameProcessorChain(
-          new UnsupportedOperationException(
-              "Transformer's FrameProcessorChain currently does not support frame edits on"
-                  + " non-square pixels. The pixelWidthHeightRatio is: "
-                  + pixelWidthHeightRatio),
-          TransformationException.ERROR_CODE_GL_INIT_FAILED);
-    }
-
     ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
-    ExternalCopyFrameProcessor externalCopyFrameProcessor =
-        new ExternalCopyFrameProcessor(enableExperimentalHdrEditing);
 
     try {
       return singleThreadExecutorService
@@ -112,12 +100,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
               () ->
                   createOpenGlObjectsAndFrameProcessorChain(
                       context,
+                      pixelWidthHeightRatio,
                       inputWidth,
                       inputHeight,
                       frameProcessors,
                       enableExperimentalHdrEditing,
-                      singleThreadExecutorService,
+                      singleThreadExecutorService))
-                      externalCopyFrameProcessor))
           .get();
     } catch (ExecutionException e) {
       throw TransformationException.createForFrameProcessorChain(
@@ -138,16 +126,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   @WorkerThread
   private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
       Context context,
+      float pixelWidthHeightRatio,
       int inputWidth,
       int inputHeight,
       List<GlFrameProcessor> frameProcessors,
       boolean enableExperimentalHdrEditing,
-      ExecutorService singleThreadExecutorService,
+      ExecutorService singleThreadExecutorService)
-      ExternalCopyFrameProcessor externalCopyFrameProcessor)
       throws IOException {
     checkState(Thread.currentThread().getName().equals(THREAD_NAME));
 
     EGLDisplay eglDisplay = GlUtil.createEglDisplay();
+    ExternalCopyFrameProcessor externalCopyFrameProcessor =
+        new ExternalCopyFrameProcessor(enableExperimentalHdrEditing);
     EGLContext eglContext =
         enableExperimentalHdrEditing
             ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay)
@@ -163,18 +153,22 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
     }
 
+    ImmutableList<GlFrameProcessor> expandedFrameProcessors =
+        getExpandedFrameProcessors(
+            externalCopyFrameProcessor, pixelWidthHeightRatio, frameProcessors);
+
+    // Initialize frame processors.
     int inputExternalTexId = GlUtil.createExternalTexture();
     externalCopyFrameProcessor.initialize(context, inputExternalTexId, inputWidth, inputHeight);
 
-    int[] framebuffers = new int[frameProcessors.size()];
+    int[] framebuffers = new int[expandedFrameProcessors.size() - 1];
     Size inputSize = externalCopyFrameProcessor.getOutputSize();
-    for (int i = 0; i < frameProcessors.size(); i++) {
+    for (int i = 1; i < expandedFrameProcessors.size(); i++) {
       int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight());
-      framebuffers[i] = GlUtil.createFboForTexture(inputTexId);
+      framebuffers[i - 1] = GlUtil.createFboForTexture(inputTexId);
-      frameProcessors
+      GlFrameProcessor frameProcessor = expandedFrameProcessors.get(i);
-          .get(i)
+      frameProcessor.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight());
-          .initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight());
+      inputSize = frameProcessor.getOutputSize();
-      inputSize = frameProcessors.get(i).getOutputSize();
     }
     return new FrameProcessorChain(
         eglDisplay,
@@ -182,13 +176,33 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
         singleThreadExecutorService,
         inputExternalTexId,
         framebuffers,
-        new ImmutableList.Builder<GlFrameProcessor>()
+        expandedFrameProcessors,
-            .add(externalCopyFrameProcessor)
-            .addAll(frameProcessors)
-            .build(),
         enableExperimentalHdrEditing);
   }
 
+  private static ImmutableList<GlFrameProcessor> getExpandedFrameProcessors(
+      ExternalCopyFrameProcessor externalCopyFrameProcessor,
+      float pixelWidthHeightRatio,
+      List<GlFrameProcessor> frameProcessors) {
+    ImmutableList.Builder<GlFrameProcessor> frameProcessorListBuilder =
+        new ImmutableList.Builder<GlFrameProcessor>().add(externalCopyFrameProcessor);
+
+    // Scale to expand the frame to apply the pixelWidthHeightRatio.
+    if (pixelWidthHeightRatio > 1f) {
+      frameProcessorListBuilder.add(
+          new ScaleToFitFrameProcessor.Builder()
+              .setScale(/* scaleX= */ pixelWidthHeightRatio, /* scaleY= */ 1f)
+              .build());
+    } else if (pixelWidthHeightRatio < 1f) {
+      frameProcessorListBuilder.add(
+          new ScaleToFitFrameProcessor.Builder()
+              .setScale(/* scaleX= */ 1f, /* scaleY= */ 1f / pixelWidthHeightRatio)
+              .build());
+    }
+    frameProcessorListBuilder.addAll(frameProcessors);
+    return frameProcessorListBuilder.build();
+  }
+
   private static final String TAG = "FrameProcessorChain";
   private static final String THREAD_NAME = "Transformer:FrameProcessorChain";
   private static final long RELEASE_WAIT_TIME_MS = 100;

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java

@@ -121,6 +121,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
         && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
       return false;
     }
+    if (inputFormat.pixelWidthHeightRatio != 1f) {
+      return false;
+    }
     if (transformationRequest.rotationDegrees != 0f) {
       return false;
     }