Transformer GL: Implement auto-scaling to preserve input frame.

PiperOrigin-RevId: 427982223

demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java

@@ -152,7 +152,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
     rotateAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item);
     rotateSpinner = findViewById(R.id.rotate_spinner);
     rotateSpinner.setAdapter(rotateAdapter);
-    rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "90", "180");
+    rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "60", "90", "180");
 
     enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox);
   }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java

@@ -105,7 +105,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
     checkArgument(format.width != Format.NO_VALUE);
     checkArgument(format.height != Format.NO_VALUE);
     // According to interface Javadoc, format.rotationDegrees should be 0. The video should always
-    // be in landscape orientation.
+    // be encoded in landscape orientation.
     checkArgument(format.height <= format.width);
     checkArgument(format.rotationDegrees == 0);
     checkNotNull(format.sampleMimeType);

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java

@@ -25,7 +25,6 @@ import com.google.android.exoplayer2.extractor.mp4.Mp4Extractor;
 import com.google.android.exoplayer2.source.MediaSource;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.Util;
-import com.google.common.collect.ImmutableSet;
 
 /** A media transformation request. */
 public final class TransformationRequest {
@@ -33,9 +32,6 @@ public final class TransformationRequest {
   /** A builder for {@link TransformationRequest} instances. */
   public static final class Builder {
 
-    private static final ImmutableSet<Integer> SUPPORTED_OUTPUT_HEIGHTS =
-        ImmutableSet.of(144, 240, 360, 480, 720, 1080, 1440, 2160);
-
     private Matrix transformationMatrix;
     private boolean flattenForSlowMotion;
     private int outputHeight;
@@ -122,27 +118,17 @@ public final class TransformationRequest {
 
     /**
      * Sets the output resolution using the output height. The default value {@link C#LENGTH_UNSET}
-     * corresponds to using the same height as the input. Output width will scale to preserve the
+     * corresponds to using the same height as the input. Output width of the displayed video will
-     * input video's aspect ratio.
+     * scale to preserve the video's aspect ratio after other transformations.
-     *
-     * <p>For now, only "popular" heights like 144, 240, 360, 480, 720, 1080, 1440, or 2160 are
-     * supported, to ensure compatibility on different devices.
      *
      * <p>For example, a 1920x1440 video can be scaled to 640x480 by calling setResolution(480).
      *
-     * @param outputHeight The output height in pixels.
+     * @param outputHeight The output height of the displayed video, in pixels.
      * @return This builder.
      * @throws IllegalArgumentException If the {@code outputHeight} is not supported.
      */
     public Builder setResolution(int outputHeight) {
-      // TODO(b/209781577): Define outputHeight in the javadoc as height can be ambiguous for videos
-      // where rotationDegrees is set in the Format.
       // TODO(b/201293185): Restructure to input a Presentation class.
-      // TODO(b/201293185): Check encoder codec capabilities in order to allow arbitrary
-      // resolutions and reasonable fallbacks.
-      checkArgument(
-          outputHeight == C.LENGTH_UNSET || SUPPORTED_OUTPUT_HEIGHTS.contains(outputHeight),
-          "Unsupported outputHeight: " + outputHeight);
       this.outputHeight = outputHeight;
       return this;
     }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -18,6 +18,8 @@ package com.google.android.exoplayer2.transformer;
 
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Util.SDK_INT;
+import static java.lang.Math.max;
+import static java.lang.Math.min;
 
 import android.content.Context;
 import android.graphics.Matrix;
@@ -63,76 +65,106 @@ import org.checkerframework.dataflow.qual.Pure;
     encoderOutputBuffer =
         new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
 
-    // Scale width and height to desired transformationRequest.outputHeight, preserving aspect
+    // The decoder rotates encoded frames for display by inputFormat.rotationDegrees.
-    // ratio.
+    int decodedWidth =
-    // TODO(b/209781577): Think about which edge length should be set for portrait videos.
+        (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
-    float inputFormatAspectRatio = (float) inputFormat.width / inputFormat.height;
+    int decodedHeight =
-    int outputWidth = inputFormat.width;
+        (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
-    int outputHeight = inputFormat.height;
+    float decodedAspectRatio = (float) decodedWidth / decodedHeight;
+
+    Matrix transformationMatrix = new Matrix(transformationRequest.transformationMatrix);
+
+    int outputWidth = decodedWidth;
+    int outputHeight = decodedHeight;
+    if (!transformationMatrix.isIdentity()) {
+      // Scale frames by decodedAspectRatio, to account for FrameEditor's normalized device
+      // coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular display
+      // of input pixels during transformations (ex. rotations). With scaling, transformationMatrix
+      // operations operate on a rectangle for x from -decodedAspectRatio to decodedAspectRatio, and
+      // y from -1 to 1.
+      transformationMatrix.preScale(/* sx= */ decodedAspectRatio, /* sy= */ 1f);
+      transformationMatrix.postScale(/* sx= */ 1f / decodedAspectRatio, /* sy= */ 1f);
+
+      float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}};
+      float xMin = Float.MAX_VALUE;
+      float xMax = Float.MIN_VALUE;
+      float yMin = Float.MAX_VALUE;
+      float yMax = Float.MIN_VALUE;
+      for (float[] transformOnNdcPoint : transformOnNdcPoints) {
+        transformationMatrix.mapPoints(transformOnNdcPoint);
+        xMin = min(xMin, transformOnNdcPoint[0]);
+        xMax = max(xMax, transformOnNdcPoint[0]);
+        yMin = min(yMin, transformOnNdcPoint[1]);
+        yMax = max(yMax, transformOnNdcPoint[1]);
+      }
+
+      float xCenter = (xMax + xMin) / 2f;
+      float yCenter = (yMax + yMin) / 2f;
+      transformationMatrix.postTranslate(-xCenter, -yCenter);
+
+      float ndcWidthAndHeight = 2f; // Length from -1 to 1.
+      float xScale = (xMax - xMin) / ndcWidthAndHeight;
+      float yScale = (yMax - yMin) / ndcWidthAndHeight;
+      transformationMatrix.postScale(1f / xScale, 1f / yScale);
+      outputWidth = Math.round(decodedWidth * xScale);
+      outputHeight = Math.round(decodedHeight * yScale);
+    }
+    // Scale width and height to desired transformationRequest.outputHeight, preserving
+    // aspect ratio.
     if (transformationRequest.outputHeight != C.LENGTH_UNSET
-        && transformationRequest.outputHeight != inputFormat.height) {
+        && transformationRequest.outputHeight != outputHeight) {
-      outputWidth = Math.round(inputFormatAspectRatio * transformationRequest.outputHeight);
+      outputWidth =
+          Math.round((float) transformationRequest.outputHeight * outputWidth / outputHeight);
       outputHeight = transformationRequest.outputHeight;
     }
 
-    // The encoder may not support encoding in portrait orientation, so the decoded video is
+    // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
-    // rotated to landscape orientation and a rotation is added back later to the output format.
+    // video before encoding, so the encoded video's width >= height, and set outputRotationDegrees
-    boolean swapEncodingDimensions = inputFormat.height > inputFormat.width;
+    // to ensure the video is displayed in the correct orientation.
+    int requestedEncoderWidth;
+    int requestedEncoderHeight;
+    boolean swapEncodingDimensions = outputHeight > outputWidth;
     if (swapEncodingDimensions) {
-      outputRotationDegrees = (inputFormat.rotationDegrees + 90) % 360;
+      outputRotationDegrees = 90;
-      int temp = outputWidth;
+      requestedEncoderWidth = outputHeight;
-      outputWidth = outputHeight;
+      requestedEncoderHeight = outputWidth;
-      outputHeight = temp;
+      // TODO(b/201293185): After fragment shader transformations are implemented, put
+      // postRotate in a later vertex shader.
+      transformationMatrix.postRotate(outputRotationDegrees);
     } else {
-      outputRotationDegrees = inputFormat.rotationDegrees;
+      outputRotationDegrees = 0;
+      requestedEncoderWidth = outputWidth;
+      requestedEncoderHeight = outputHeight;
     }
-    float displayAspectRatio =
-        (inputFormat.rotationDegrees % 180) == 0
-            ? inputFormatAspectRatio
-            : 1.0f / inputFormatAspectRatio;
 
-    Matrix transformationMatrix = new Matrix(transformationRequest.transformationMatrix);
+    Format requestedEncoderFormat =
-    // Scale frames by input aspect ratio, to account for FrameEditor's square normalized device
-    // coordinates (-1 to 1) and preserve frame relative dimensions during transformations
-    // (ex. rotations). After this scaling, transformationMatrix operations operate on a rectangle
-    // for x from -displayAspectRatio to displayAspectRatio, and y from -1 to 1
-    transformationMatrix.preScale(displayAspectRatio, 1);
-    transformationMatrix.postScale(1.0f / displayAspectRatio, 1);
-
-    // The decoder rotates videos to their intended display orientation. The frameEditor rotates
-    // them back for improved encoder compatibility.
-    // TODO(b/201293185): After fragment shader transformations are implemented, put
-    // postRotate in a later vertex shader.
-    transformationMatrix.postRotate(outputRotationDegrees);
-
-    Format requestedOutputFormat =
         new Format.Builder()
-            .setWidth(outputWidth)
+            .setWidth(requestedEncoderWidth)
-            .setHeight(outputHeight)
+            .setHeight(requestedEncoderHeight)
             .setRotationDegrees(0)
             .setSampleMimeType(
                 transformationRequest.videoMimeType != null
                     ? transformationRequest.videoMimeType
                     : inputFormat.sampleMimeType)
             .build();
-    encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat, allowedOutputMimeTypes);
+    encoder = encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
-    Format actualOutputFormat = encoder.getConfigurationFormat();
+    Format encoderSupportedFormat = encoder.getConfigurationFormat();
     fallbackListener.onTransformationRequestFinalized(
         createFallbackTransformationRequest(
             transformationRequest,
             !swapEncodingDimensions,
-            requestedOutputFormat,
+            requestedEncoderFormat,
-            actualOutputFormat));
+            encoderSupportedFormat));
 
     if (transformationRequest.enableHdrEditing
-        || inputFormat.height != actualOutputFormat.height
+        || inputFormat.height != encoderSupportedFormat.height
-        || inputFormat.width != actualOutputFormat.width
+        || inputFormat.width != encoderSupportedFormat.width
         || !transformationMatrix.isIdentity()) {
       frameEditor =
           FrameEditor.create(
               context,
-              actualOutputFormat.width,
+              encoderSupportedFormat.width,
-              actualOutputFormat.height,
+              encoderSupportedFormat.height,
               inputFormat.pixelWidthHeightRatio,
               transformationMatrix,
               /* outputSurface= */ encoder.getInputSurface(),