FrameProcessor: Add setCrop to Presentation.

PiperOrigin-RevId: 440325693

library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java

@@ -55,6 +55,9 @@ public final class GlUtil {
   /** Number of vertices in a rectangle. */
   public static final int RECTANGLE_VERTICES_COUNT = 4;
 
+  /** Length of the normalized device coordinate (NDC) space, which spans from -1 to 1. */
+  public static final float LENGTH_NDC = 2f;
+
   private static final String TAG = "GlUtil";
 
   // https://www.khronos.org/registry/EGL/extensions/EXT/EGL_EXT_protected_content.txt

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/BitmapTestUtil.java

@@ -63,6 +63,10 @@ public class BitmapTestUtil {
       "media/bitmap/sample_mp4_first_frame_request_output_height.png";
   public static final String ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING =
       "media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png";
+  public static final String CROP_SMALLER_EXPECTED_OUTPUT_PNG_ASSET_STRING =
+      "media/bitmap/sample_mp4_first_frame_crop_smaller.png";
+  public static final String CROP_LARGER_EXPECTED_OUTPUT_PNG_ASSET_STRING =
+      "media/bitmap/sample_mp4_first_frame_crop_larger.png";
   /**
    * Maximum allowed average pixel difference between the expected and actual edited images in pixel
    * difference-based tests. The value is chosen so that differences in decoder behavior across

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java

@@ -16,6 +16,8 @@
 package com.google.android.exoplayer2.transformer;
 
 import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+import static com.google.android.exoplayer2.transformer.BitmapTestUtil.CROP_LARGER_EXPECTED_OUTPUT_PNG_ASSET_STRING;
+import static com.google.android.exoplayer2.transformer.BitmapTestUtil.CROP_SMALLER_EXPECTED_OUTPUT_PNG_ASSET_STRING;
 import static com.google.android.exoplayer2.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_STRING;
 import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
 import static com.google.android.exoplayer2.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
@@ -197,6 +199,51 @@ public final class FrameProcessorChainPixelTest {
     assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
   }
 
+  @Test
+  public void processData_withPresentationFrameProcessor_cropSmaller_producesExpectedOutput()
+      throws Exception {
+    String testId = "updateProgramAndDraw_cropSmaller";
+    GlFrameProcessor glFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext())
+            .setCrop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f)
+            .build();
+    setUpAndPrepareFirstFrame(glFrameProcessor);
+    Bitmap expectedBitmap =
+        BitmapTestUtil.readBitmap(CROP_SMALLER_EXPECTED_OUTPUT_PNG_ASSET_STRING);
+
+    Bitmap actualBitmap = processFirstFrameAndEnd();
+
+    // TODO(b/207848601): switch to using proper tooling for testing against golden data.
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    BitmapTestUtil.saveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void processData_withPresentationFrameProcessor_cropLarger_producesExpectedOutput()
+      throws Exception {
+    String testId = "updateProgramAndDraw_cropLarger";
+    GlFrameProcessor glFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext())
+            .setCrop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f)
+            .build();
+    setUpAndPrepareFirstFrame(glFrameProcessor);
+    Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_EXPECTED_OUTPUT_PNG_ASSET_STRING);
+
+    Bitmap actualBitmap = processFirstFrameAndEnd();
+
+    // TODO(b/207848601): switch to using proper tooling for testing against golden data.
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    BitmapTestUtil.saveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
   @Test
   public void processData_withScaleToFitFrameProcessor_rotate45_producesExpectedOutput()
       throws Exception {

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java

@@ -15,6 +15,7 @@
  */
 package com.google.android.exoplayer2.transformer;
 
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
 import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
 
@@ -28,19 +29,27 @@ import com.google.android.exoplayer2.util.GlUtil;
 import java.io.IOException;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
-/** Controls how a frame is viewed, by changing resolution. */
+/**
-// TODO(b/213190310): Implement crop, aspect ratio changes, etc.
+ * Controls how a frame is viewed, by cropping or changing resolution.
+ *
+ * <p>Cropping is applied before setting resolution.
+ */
+// TODO(b/213190310): Implement aspect ratio changes, etc.
 public final class PresentationFrameProcessor implements GlFrameProcessor {
-
   /** A builder for {@link PresentationFrameProcessor} instances. */
   public static final class Builder {
 
     // Mandatory field.
     private final Context context;
 
-    // Optional field.
+    // Optional fields.
-    private int outputHeight;
+    private int heightPixels;
+    private float cropLeft;
+    private float cropRight;
+    private float cropBottom;
+    private float cropTop;
 
     /**
      * Creates a builder with default values.
@@ -49,7 +58,11 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
      */
     public Builder(Context context) {
       this.context = context;
-      outputHeight = C.LENGTH_UNSET;
+      heightPixels = C.LENGTH_UNSET;
+      cropLeft = -1f;
+      cropRight = 1f;
+      cropBottom = -1f;
+      cropTop = 1f;
     }
 
     /**
@@ -59,18 +72,49 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
      * input. Output width of the displayed frame will scale to preserve the frame's aspect ratio
      * after other transformations.
      *
-     * <p>For example, a 1920x1440 frame can be scaled to 640x480 by calling setResolution(480).
+     * <p>For example, a 1920x1440 frame can be scaled to 640x480 by calling {@code
+     * setResolution(480)}.
      *
-     * @param outputHeight The output height of the displayed frame, in pixels.
+     * @param height The output height of the displayed frame, in pixels.
      * @return This builder.
      */
-    public Builder setResolution(int outputHeight) {
+    public Builder setResolution(int height) {
-      this.outputHeight = outputHeight;
+      this.heightPixels = height;
+      return this;
+    }
+
+    /**
+     * Crops a smaller (or larger frame), per normalized device coordinates (NDC), where the input
+     * frame corresponds to the square ranging from -1 to 1 on the x and y axes.
+     *
+     * <p>{@code left} and {@code bottom} default to -1, and {@code right} and {@code top} default
+     * to 1. To crop to a smaller subset of the input frame, use values between -1 and 1. To crop to
+     * a larger frame, use values below -1 and above 1.
+     *
+     * <p>Width and height values set may be rescaled by {@link #setResolution(int)}.
+     *
+     * @param left The left edge of the output frame, in NDC. Must be less than {@code right}.
+     * @param right The right edge of the output frame, in NDC. Must be greater than {@code left}.
+     * @param bottom The bottom edge of the output frame, in NDC. Must be less than {@code top}.
+     * @param top The top edge of the output frame, in NDC. Must be greater than {@code bottom}.
+     * @return This builder.
+     */
+    public Builder setCrop(float left, float right, float bottom, float top) {
+      checkArgument(
+          right > left, "right value " + right + " should be greater than left value " + left);
+      checkArgument(
+          top > bottom, "top value " + top + " should be greater than bottom value " + bottom);
+      cropLeft = left;
+      cropRight = right;
+      cropBottom = bottom;
+      cropTop = top;
+
       return this;
     }
 
     public PresentationFrameProcessor build() {
-      return new PresentationFrameProcessor(context, outputHeight);
+      return new PresentationFrameProcessor(
+          context, heightPixels, cropLeft, cropRight, cropBottom, cropTop);
     }
   }
 
@@ -79,24 +123,34 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
   }
 
   private final Context context;
-  private final int requestedHeight;
+  private final int requestedHeightPixels;
+  private final float cropLeft;
+  private final float cropRight;
+  private final float cropBottom;
+  private final float cropTop;
 
-  private @MonotonicNonNull Size outputSize;
   private int outputRotationDegrees;
+  private @MonotonicNonNull Size outputSize;
   private @MonotonicNonNull Matrix transformationMatrix;
   private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor;
 
-  /**
+  /** Creates a new instance. */
-   * Creates a new instance.
+  private PresentationFrameProcessor(
-   *
+      Context context,
-   * @param context The {@link Context}.
+      int requestedHeightPixels,
-   * @param requestedHeight The height of the output frame, in pixels.
+      float cropLeft,
-   */
+      float cropRight,
-  private PresentationFrameProcessor(Context context, int requestedHeight) {
+      float cropBottom,
+      float cropTop) {
     this.context = context;
-    this.requestedHeight = requestedHeight;
+    this.requestedHeightPixels = requestedHeightPixels;
+    this.cropLeft = cropLeft;
+    this.cropRight = cropRight;
+    this.cropBottom = cropBottom;
+    this.cropTop = cropTop;
 
     outputRotationDegrees = C.LENGTH_UNSET;
+    transformationMatrix = new Matrix();
   }
 
   @Override
@@ -136,16 +190,20 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
   }
 
   @EnsuresNonNull("transformationMatrix")
-  @VisibleForTesting // Allows roboletric testing of output size calculation without OpenGL.
+  @VisibleForTesting // Allows robolectric testing of output size calculation without OpenGL.
   /* package */ void configureOutputSizeAndTransformationMatrix(int inputWidth, int inputHeight) {
     transformationMatrix = new Matrix();
-    int displayWidth = inputWidth;
+
-    int displayHeight = inputHeight;
+    Size cropSize = applyCrop(inputWidth, inputHeight);
-    // Scale width and height to desired requestedHeight, preserving aspect ratio.
+    int displayWidth = cropSize.getWidth();
-    if (requestedHeight != C.LENGTH_UNSET && requestedHeight != displayHeight) {
+    int displayHeight = cropSize.getHeight();
-      displayWidth = Math.round((float) requestedHeight * displayWidth / displayHeight);
+
-      displayHeight = requestedHeight;
+    // Scale width and height to desired requestedHeightPixels, preserving aspect ratio.
+    if (requestedHeightPixels != C.LENGTH_UNSET && requestedHeightPixels != displayHeight) {
+      displayWidth = Math.round((float) requestedHeightPixels * displayWidth / displayHeight);
+      displayHeight = requestedHeightPixels;
     }
+
     // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
     // frame before encoding, so the encoded frame's width >= height, and set
     // outputRotationDegrees to ensure the frame is displayed in the correct orientation.
@@ -160,4 +218,19 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
       outputSize = new Size(displayWidth, displayHeight);
     }
   }
+
+  @RequiresNonNull("transformationMatrix")
+  private Size applyCrop(int inputWidth, int inputHeight) {
+    float scaleX = (cropRight - cropLeft) / GlUtil.LENGTH_NDC;
+    float scaleY = (cropTop - cropBottom) / GlUtil.LENGTH_NDC;
+    float centerX = (cropLeft + cropRight) / 2;
+    float centerY = (cropBottom + cropTop) / 2;
+
+    transformationMatrix.postTranslate(-centerX, -centerY);
+    transformationMatrix.postScale(1f / scaleX, 1f / scaleY);
+
+    int outputWidth = Math.round(inputWidth * scaleX);
+    int outputHeight = Math.round(inputHeight * scaleY);
+    return new Size(outputWidth, outputHeight);
+  }
 }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java

@@ -144,7 +144,7 @@ public final class ScaleToFitFrameProcessor implements GlFrameProcessor {
   }
 
   @EnsuresNonNull("adjustedTransformationMatrix")
-  @VisibleForTesting // Allows roboletric testing of output size calculation without OpenGL.
+  @VisibleForTesting // Allows robolectric testing of output size calculation without OpenGL.
   /* package */ void configureOutputSizeAndTransformationMatrix(int inputWidth, int inputHeight) {
     adjustedTransformationMatrix = new Matrix(transformationMatrix);
 
@@ -164,22 +164,21 @@ public final class ScaleToFitFrameProcessor implements GlFrameProcessor {
 
     // Modify transformationMatrix to keep input pixels.
     float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}};
-    float xMin = Float.MAX_VALUE;
+    float minX = Float.MAX_VALUE;
-    float xMax = Float.MIN_VALUE;
+    float maxX = Float.MIN_VALUE;
-    float yMin = Float.MAX_VALUE;
+    float minY = Float.MAX_VALUE;
-    float yMax = Float.MIN_VALUE;
+    float maxY = Float.MIN_VALUE;
     for (float[] transformOnNdcPoint : transformOnNdcPoints) {
       adjustedTransformationMatrix.mapPoints(transformOnNdcPoint);
-      xMin = min(xMin, transformOnNdcPoint[0]);
+      minX = min(minX, transformOnNdcPoint[0]);
-      xMax = max(xMax, transformOnNdcPoint[0]);
+      maxX = max(maxX, transformOnNdcPoint[0]);
-      yMin = min(yMin, transformOnNdcPoint[1]);
+      minY = min(minY, transformOnNdcPoint[1]);
-      yMax = max(yMax, transformOnNdcPoint[1]);
+      maxY = max(maxY, transformOnNdcPoint[1]);
     }
 
-    float ndcWidthAndHeight = 2f; // Length from -1 to 1.
+    float scaleX = (maxX - minX) / GlUtil.LENGTH_NDC;
-    float xScale = (xMax - xMin) / ndcWidthAndHeight;
+    float scaleY = (maxY - minY) / GlUtil.LENGTH_NDC;
-    float yScale = (yMax - yMin) / ndcWidthAndHeight;
+    adjustedTransformationMatrix.postScale(1f / scaleX, 1f / scaleY);
-    adjustedTransformationMatrix.postScale(1f / xScale, 1f / yScale);
+    outputSize = new Size(Math.round(inputWidth * scaleX), Math.round(inputHeight * scaleY));
-    outputSize = new Size(Math.round(inputWidth * xScale), Math.round(inputHeight * yScale));
   }
 }

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java

@@ -21,6 +21,7 @@ import static org.junit.Assert.assertThrows;
 
 import android.util.Size;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.util.GlUtil;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
@@ -95,6 +96,83 @@ public final class PresentationFrameProcessorTest {
     assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
   }
 
+  @Test
+  public void getOutputSize_setCrop_changesDimensions() {
+    int inputWidth = 300;
+    int inputHeight = 200;
+    float left = -.5f;
+    float right = .5f;
+    float bottom = .5f;
+    float top = 1f;
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext())
+            .setCrop(left, right, bottom, top)
+            .build();
+
+    presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
+    Size outputSize = presentationFrameProcessor.getOutputSize();
+
+    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
+    int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
+    int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
+    assertThat(outputSize.getWidth()).isEqualTo(expectedPostCropWidth);
+    assertThat(outputSize.getHeight()).isEqualTo(expectedPostCropHeight);
+  }
+
+  @Test
+  public void getOutputSize_setCropAndSetResolution_changesDimensions() {
+    int inputWidth = 300;
+    int inputHeight = 200;
+    float left = -.5f;
+    float right = .5f;
+    float bottom = .5f;
+    float top = 1f;
+    int requestedHeight = 100;
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext())
+            .setCrop(left, right, bottom, top)
+            .setResolution(100)
+            .build();
+
+    presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
+    Size outputSize = presentationFrameProcessor.getOutputSize();
+
+    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
+    int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
+    int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
+    assertThat(outputSize.getWidth())
+        .isEqualTo(
+            Math.round((float) requestedHeight * expectedPostCropWidth / expectedPostCropHeight));
+    assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
+  }
+
+  @Test
+  public void getOutputSize_setResolutionAndCrop_changesDimensions() {
+    int inputWidth = 300;
+    int inputHeight = 200;
+    float left = -.5f;
+    float right = .5f;
+    float bottom = .5f;
+    float top = 1f;
+    int requestedHeight = 100;
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext())
+            .setResolution(100)
+            .setCrop(left, right, bottom, top)
+            .build();
+
+    presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
+    Size outputSize = presentationFrameProcessor.getOutputSize();
+
+    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
+    int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
+    int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
+    assertThat(outputSize.getWidth())
+        .isEqualTo(
+            Math.round((float) requestedHeight * expectedPostCropWidth / expectedPostCropHeight));
+    assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
+  }
+
   @Test
   public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
     PresentationFrameProcessor presentationFrameProcessor =