Transformer GL: Create setTransformationMatrix().

Allows a transformation matrix to be input into Transformer,
to apply vertex transformations like cropping, rotation,
and other transformations built into android.graphics.Matrix.

Not building out into a VertexTransformation class yet, as
that class structure wouldn't make sense until we can modify
resolution, per TODOs.

PiperOrigin-RevId: 413384409

libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java

@@ -24,6 +24,7 @@ import static java.lang.Math.max;
 import android.content.res.AssetFileDescriptor;
 import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
+import android.graphics.Matrix;
 import android.graphics.PixelFormat;
 import android.media.Image;
 import android.media.ImageReader;
@@ -84,9 +85,14 @@ public final class FrameEditorTest {
       int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
       frameEditorOutputImageReader =
           ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
+      Matrix identityMatrix = new Matrix();
       frameEditor =
           FrameEditor.create(
-              getApplicationContext(), width, height, frameEditorOutputImageReader.getSurface());
+              getApplicationContext(),
+              width,
+              height,
+              identityMatrix,
+              frameEditorOutputImageReader.getSurface());
 
       // Queue the first video frame from the extractor.
       String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));

libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer.mh;
+
+import static androidx.media3.transformer.mh.AndroidTestUtil.REMOTE_MP4_10_SECONDS_URI_STRING;
+import static androidx.media3.transformer.mh.AndroidTestUtil.runTransformer;
+
+import android.content.Context;
+import android.graphics.Matrix;
+import androidx.media3.transformer.Transformer;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** {@link Transformer} instrumentation test for setting a transformation matrix. */
+@RunWith(AndroidJUnit4.class)
+public class SetTransformationMatrixTransformationTest {
+  @Test
+  public void setTransformationMatrixTransform() throws Exception {
+    Context context = ApplicationProvider.getApplicationContext();
+    Matrix transformationMatrix = new Matrix();
+    transformationMatrix.postTranslate(/* dx= */ .2f, /* dy= */ .1f);
+    Transformer transformer =
+        new Transformer.Builder(context).setTransformationMatrix(transformationMatrix).build();
+
+    runTransformer(
+        context,
+        /* testId= */ "setTransformationMatrixTransform",
+        transformer,
+        REMOTE_MP4_10_SECONDS_URI_STRING,
+        /* timeoutSeconds= */ 120);
+  }
+}

libraries/transformer/src/main/assets/shaders/vertex_shader.glsl

@@ -14,8 +14,9 @@
 attribute vec4 a_position;
 attribute vec4 a_texcoord;
 uniform mat4 tex_transform;
+uniform mat4 transformation_matrix;
 varying vec2 v_texcoord;
 void main() {
   gl_Position = a_position;
-  v_texcoord = (tex_transform * a_texcoord).xy;
+  v_texcoord = (transformation_matrix * tex_transform * a_texcoord).xy;
 }

libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java

@@ -16,6 +16,7 @@
 package androidx.media3.transformer;
 
 import android.content.Context;
+import android.graphics.Matrix;
 import android.graphics.SurfaceTexture;
 import android.opengl.EGL14;
 import android.opengl.EGLContext;
@@ -27,10 +28,7 @@ import android.view.Surface;
 import androidx.media3.common.util.GlUtil;
 import java.io.IOException;
 
-/**
+/** FrameEditor applies changes to individual video frames. */
- * FrameEditor applies changes to individual video frames. Changes include just resolution for now,
- * but may later include brightness, cropping, rotation, etc.
- */
 /* package */ final class FrameEditor {
 
   static {
@@ -43,11 +41,16 @@ import java.io.IOException;
    * @param context A {@link Context}.
    * @param outputWidth The output width in pixels.
    * @param outputHeight The output height in pixels.
+   * @param transformationMatrix The transformation matrix to apply to each frame.
    * @param outputSurface The {@link Surface}.
    * @return A configured {@code FrameEditor}.
    */
   public static FrameEditor create(
-      Context context, int outputWidth, int outputHeight, Surface outputSurface) {
+      Context context,
+      int outputWidth,
+      int outputHeight,
+      Matrix transformationMatrix,
+      Surface outputSurface) {
     EGLDisplay eglDisplay = GlUtil.createEglDisplay();
     EGLContext eglContext;
     try {
@@ -58,15 +61,16 @@ import java.io.IOException;
     EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
     GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
     int textureId = GlUtil.createExternalTexture();
-    GlUtil.Program copyProgram;
+    GlUtil.Program glProgram;
     try {
       // TODO(internal b/205002913): check the loaded program is consistent with the attributes
       // and uniforms expected in the code.
-      copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
+      glProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
     } catch (IOException e) {
       throw new IllegalStateException(e);
     }
-    copyProgram.setBufferAttribute(
+
+    glProgram.setBufferAttribute(
         "a_position",
         new float[] {
           -1.0f, -1.0f, 0.0f, 1.0f,
@@ -75,7 +79,7 @@ import java.io.IOException;
           1.0f, 1.0f, 0.0f, 1.0f,
         },
         /* size= */ 4);
-    copyProgram.setBufferAttribute(
+    glProgram.setBufferAttribute(
         "a_texcoord",
         new float[] {
           0.0f, 0.0f, 0.0f, 1.0f,
@@ -84,14 +88,57 @@ import java.io.IOException;
           1.0f, 1.0f, 0.0f, 1.0f,
         },
         /* size= */ 4);
-    copyProgram.setSamplerTexIdUniform("tex_sampler", textureId, /* unit= */ 0);
+    glProgram.setSamplerTexIdUniform("tex_sampler", textureId, /* unit= */ 0);
-    return new FrameEditor(eglDisplay, eglContext, eglSurface, textureId, copyProgram);
+
+    float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix);
+    glProgram.setFloatsUniform("transformation_matrix", transformationMatrixArray);
+
+    return new FrameEditor(eglDisplay, eglContext, eglSurface, textureId, glProgram);
+  }
+
+  /**
+   * Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful
+   * for converting to the 4x4 column-major format commonly used in OpenGL.
+   */
+  private static final float[] getGlMatrixArray(Matrix matrix) {
+    float[] matrix3x3Array = new float[9];
+    matrix.getValues(matrix3x3Array);
+    float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array);
+
+    // Transpose from row-major to column-major representations.
+    float[] transposedMatrix4x4Array = new float[16];
+    android.opengl.Matrix.transposeM(
+        transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0);
+
+    return transposedMatrix4x4Array;
+  }
+
+  /**
+   * Returns a 4x4 matrix array containing the 3x3 matrix array's contents.
+   *
+   * <p>The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected
+   * to be in 3 dimensions. The output will have the third row/column's values be an identity
+   * matrix's values, so that vertex transformations using this matrix will not affect the z axis.
+   * <br>
+   * Input format: [a, b, c, d, e, f, g, h, i] <br>
+   * Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i]
+   */
+  private static final float[] getMatrix4x4Array(float[] matrix3x3Array) {
+    float[] matrix4x4Array = new float[16];
+    matrix4x4Array[10] = 1;
+    for (int inputRow = 0; inputRow < 3; inputRow++) {
+      for (int inputColumn = 0; inputColumn < 3; inputColumn++) {
+        int outputRow = (inputRow == 2) ? 3 : inputRow;
+        int outputColumn = (inputColumn == 2) ? 3 : inputColumn;
+        matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn];
+      }
+    }
+    return matrix4x4Array;
   }
 
   // Predefined shader values.
-  private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl";
+  private static final String VERTEX_SHADER_FILE_PATH = "shaders/vertex_shader.glsl";
-  private static final String FRAGMENT_SHADER_FILE_PATH =
+  private static final String FRAGMENT_SHADER_FILE_PATH = "shaders/fragment_shader.glsl";
-      "shaders/copy_external_fragment_shader.glsl";
 
   private final float[] textureTransformMatrix;
   private final EGLDisplay eglDisplay;
@@ -101,7 +148,7 @@ import java.io.IOException;
   private final SurfaceTexture inputSurfaceTexture;
   private final Surface inputSurface;
 
-  private final GlUtil.Program copyProgram;
+  private final GlUtil.Program glProgram;
 
   private volatile boolean hasInputData;
 
@@ -110,12 +157,12 @@ import java.io.IOException;
       EGLContext eglContext,
       EGLSurface eglSurface,
       int textureId,
-      GlUtil.Program copyProgram) {
+      GlUtil.Program glProgram) {
     this.eglDisplay = eglDisplay;
     this.eglContext = eglContext;
     this.eglSurface = eglSurface;
     this.textureId = textureId;
-    this.copyProgram = copyProgram;
+    this.glProgram = glProgram;
     textureTransformMatrix = new float[16];
     inputSurfaceTexture = new SurfaceTexture(textureId);
     inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true);
@@ -135,12 +182,12 @@ import java.io.IOException;
     return hasInputData;
   }
 
-  /** Processes pending input data. */
+  /** Processes pending input frame. */
   public void processData() {
     inputSurfaceTexture.updateTexImage();
     inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
-    copyProgram.setFloatsUniform("tex_transform", textureTransformMatrix);
+    glProgram.setFloatsUniform("tex_transform", textureTransformMatrix);
-    copyProgram.bindAttributesAndUniforms();
+    glProgram.bindAttributesAndUniforms();
     GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
     long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp();
     EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs);
@@ -150,7 +197,7 @@ import java.io.IOException;
 
   /** Releases all resources. */
   public void release() {
-    copyProgram.delete();
+    glProgram.delete();
     GlUtil.deleteTexture(textureId);
     GlUtil.destroyEglContext(eglDisplay, eglContext);
     inputSurfaceTexture.release();

libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java

@@ -16,6 +16,7 @@
 
 package androidx.media3.transformer;
 
+import android.graphics.Matrix;
 import androidx.annotation.Nullable;
 
 /** A media transformation configuration. */
@@ -25,6 +26,7 @@ import androidx.annotation.Nullable;
   public final boolean removeVideo;
   public final boolean flattenForSlowMotion;
   public final int outputHeight;
+  public final Matrix transformationMatrix;
   public final String containerMimeType;
   @Nullable public final String audioMimeType;
   @Nullable public final String videoMimeType;
@@ -34,6 +36,7 @@ import androidx.annotation.Nullable;
       boolean removeVideo,
       boolean flattenForSlowMotion,
       int outputHeight,
+      Matrix transformationMatrix,
       String containerMimeType,
       @Nullable String audioMimeType,
       @Nullable String videoMimeType) {
@@ -41,6 +44,7 @@ import androidx.annotation.Nullable;
     this.removeVideo = removeVideo;
     this.flattenForSlowMotion = flattenForSlowMotion;
     this.outputHeight = outputHeight;
+    this.transformationMatrix = transformationMatrix;
     this.containerMimeType = containerMimeType;
     this.audioMimeType = audioMimeType;
     this.videoMimeType = videoMimeType;

libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java

@@ -25,6 +25,7 @@ import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_MIN_BUFFER_MS
 import static java.lang.Math.min;
 
 import android.content.Context;
+import android.graphics.Matrix;
 import android.media.MediaFormat;
 import android.media.MediaMuxer;
 import android.os.Handler;
@@ -102,6 +103,7 @@ public final class Transformer {
     private boolean removeVideo;
     private boolean flattenForSlowMotion;
     private int outputHeight;
+    private Matrix transformationMatrix;
     private String containerMimeType;
     @Nullable private String audioMimeType;
     @Nullable private String videoMimeType;
@@ -114,6 +116,7 @@ public final class Transformer {
     public Builder() {
       muxerFactory = new FrameworkMuxer.Factory();
       outputHeight = Format.NO_VALUE;
+      transformationMatrix = new Matrix();
       containerMimeType = MimeTypes.VIDEO_MP4;
       listener = new Listener() {};
       looper = Util.getCurrentOrMainLooper();
@@ -129,6 +132,7 @@ public final class Transformer {
       this.context = context.getApplicationContext();
       muxerFactory = new FrameworkMuxer.Factory();
       outputHeight = Format.NO_VALUE;
+      transformationMatrix = new Matrix();
       containerMimeType = MimeTypes.VIDEO_MP4;
       listener = new Listener() {};
       looper = Util.getCurrentOrMainLooper();
@@ -144,6 +148,7 @@ public final class Transformer {
       this.removeVideo = transformer.transformation.removeVideo;
       this.flattenForSlowMotion = transformer.transformation.flattenForSlowMotion;
       this.outputHeight = transformer.transformation.outputHeight;
+      this.transformationMatrix = transformer.transformation.transformationMatrix;
       this.containerMimeType = transformer.transformation.containerMimeType;
       this.audioMimeType = transformer.transformation.audioMimeType;
       this.videoMimeType = transformer.transformation.videoMimeType;
@@ -260,6 +265,26 @@ public final class Transformer {
       return this;
     }
 
+    /**
+     * Sets the transformation matrix. The default value is to apply no change.
+     *
+     * <p>This can be used to perform operations supported by {@link Matrix}, like scaling and
+     * rotating the video.
+     *
+     * <p>For now, resolution will not be affected by this method.
+     *
+     * @param transformationMatrix The transformation to apply to video frames.
+     * @return This builder.
+     */
+    public Builder setTransformationMatrix(Matrix transformationMatrix) {
+      // TODO(Internal b/201293185): After {@link #setResolution} supports arbitrary resolutions,
+      // allow transformations to change the resolution, by scaling to the appropriate min/max
+      // values. This will also be required to create the VertexTransformation class, in order to
+      // have aspect ratio helper methods (which require resolution to change).
+      this.transformationMatrix = transformationMatrix;
+      return this;
+    }
+
     /**
      * @deprecated This feature will be removed in a following release and the MIME type of the
      *     output will always be MP4.
@@ -411,6 +436,7 @@ public final class Transformer {
               removeVideo,
               flattenForSlowMotion,
               outputHeight,
+              transformationMatrix,
               containerMimeType,
               audioMimeType,
               videoMimeType);

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java

@@ -68,10 +68,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       return false;
     }
     Format inputFormat = checkNotNull(formatHolder.format);
-    if ((transformation.videoMimeType != null
+    if (shouldTranscode(inputFormat)) {
-            && !transformation.videoMimeType.equals(inputFormat.sampleMimeType))
-        || (transformation.outputHeight != Format.NO_VALUE
-            && transformation.outputHeight != inputFormat.height)) {
       samplePipeline = new VideoSamplePipeline(context, inputFormat, transformation, getIndex());
     } else {
       samplePipeline = new PassthroughSamplePipeline(inputFormat);
@@ -82,6 +79,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     return true;
   }
 
+  private boolean shouldTranscode(Format inputFormat) {
+    if (transformation.videoMimeType != null
+        && !transformation.videoMimeType.equals(inputFormat.sampleMimeType)) {
+      return true;
+    }
+    if (transformation.outputHeight != Format.NO_VALUE
+        && transformation.outputHeight != inputFormat.height) {
+      return true;
+    }
+    if (!transformation.transformationMatrix.isIdentity()) {
+      return true;
+    }
+    return false;
+  }
+
   /**
    * Queues the input buffer to the sample pipeline unless it should be dropped because of slow
    * motion flattening.

libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java

@@ -84,6 +84,7 @@ import java.io.IOException;
             context,
             outputWidth,
             outputHeight,
+            transformation.transformationMatrix,
             /* outputSurface= */ checkNotNull(encoder.getInputSurface()));
     try {
       decoder =