Effect: Flip texture in OpenGL instead of allocating a Bitmap.

Reduce short-lived allocations of potentially large objects, like Bitmap.

Unfortunately, this does make the TextureOverlay interface more messy though, requiring a way to signal whether the texture should be flipped vertically.

PiperOrigin-RevId: 584661400

libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java

@@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
 import android.content.Context;
 import android.graphics.Bitmap;
 import android.net.Uri;
+import android.opengl.Matrix;
 import androidx.media3.common.C;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.util.BitmapLoader;
@@ -39,10 +40,17 @@ import org.checkerframework.checker.nullness.qual.Nullable;
  */
 @UnstableApi
 public abstract class BitmapOverlay extends TextureOverlay {
+
+  private final float[] flipVerticallyMatrix;
+
   private int lastTextureId;
   private @Nullable Bitmap lastBitmap;
 
-  BitmapOverlay() {
+  /* package */ BitmapOverlay() {
+    float[] temp = GlUtil.create4x4IdentityMatrix();
+    Matrix.scaleM(temp, /* offset */ 0, /* x= */ 1f, /* y= */ -1f, /* z= */ 1f);
+    flipVerticallyMatrix = temp;
+
     lastTextureId = C.INDEX_UNSET;
   }
 
@@ -72,11 +80,10 @@ public abstract class BitmapOverlay extends TextureOverlay {
     if (bitmap != lastBitmap) {
       try {
         lastBitmap = bitmap;
-        if (lastTextureId != -1) {
+        if (lastTextureId != C.INDEX_UNSET) {
           GlUtil.deleteTexture(lastTextureId);
         }
-        lastTextureId =
-            GlUtil.createTexture(BitmapUtil.flipBitmapVertically(checkNotNull(lastBitmap)));
+        lastTextureId = GlUtil.createTexture(checkNotNull(lastBitmap));
       } catch (GlUtil.GlException e) {
         throw new VideoFrameProcessingException(e);
       }
@@ -84,6 +91,15 @@ public abstract class BitmapOverlay extends TextureOverlay {
     return lastTextureId;
   }
 
+  @Override
+  public float[] getVertexTransformation(long presentationTimeUs) {
+    // Whereas the Android system uses the top-left corner as (0,0) of the
+    // coordinate system, OpenGL uses the bottom-left corner as (0,0), so the
+    // texture gets flipped. Flip the texture vertically to ensure the
+    // orientation of the output is correct.
+    return flipVerticallyMatrix;
+  }
+
   /**
    * Creates a {@link BitmapOverlay} that shows the {@code overlayBitmap} in the same position and
    * size throughout the whole video.
@@ -164,7 +180,7 @@ public abstract class BitmapOverlay extends TextureOverlay {
   public void release() throws VideoFrameProcessingException {
     super.release();
     lastBitmap = null;
-    if (lastTextureId != -1) {
+    if (lastTextureId != C.INDEX_UNSET) {
       try {
         GlUtil.deleteTexture(lastTextureId);
       } catch (GlUtil.GlException e) {

libraries/effect/src/main/java/androidx/media3/effect/BitmapUtil.java

@@ -1,38 +0,0 @@
-/*
- * Copyright 2022 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package androidx.media3.effect;
-
-import android.graphics.Bitmap;
-import android.graphics.Matrix;
-
-/** Utility functions for working with {@link Bitmap}. */
-/* package */ final class BitmapUtil {
-  public static Bitmap flipBitmapVertically(Bitmap bitmap) {
-    Matrix flip = new Matrix();
-    flip.postScale(1f, -1f);
-    return Bitmap.createBitmap(
-        bitmap,
-        /* x= */ 0,
-        /* y= */ 0,
-        bitmap.getWidth(),
-        bitmap.getHeight(),
-        flip,
-        /* filter= */ true);
-  }
-
-  /** Class only contains static methods. */
-  private BitmapUtil() {}
-}

libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java

@@ -78,26 +78,24 @@ import com.google.common.collect.ImmutableList;
       throws VideoFrameProcessingException {
     try {
       glProgram.use();
-      if (!overlays.isEmpty()) {
-        for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) {
-          TextureOverlay overlay = overlays.get(texUnitIndex - 1);
-
-          glProgram.setSamplerTexIdUniform(
-              Util.formatInvariant("uOverlayTexSampler%d", texUnitIndex),
-              overlay.getTextureId(presentationTimeUs),
-              texUnitIndex);
-          OverlaySettings overlaySettings = overlay.getOverlaySettings(presentationTimeUs);
-          Size overlaySize = overlay.getTextureSize(presentationTimeUs);
-
-          glProgram.setFloatsUniform(
-              Util.formatInvariant("uTransformationMatrix%d", texUnitIndex),
-              samplerOverlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings));
-
-          glProgram.setFloatUniform(
-              Util.formatInvariant("uOverlayAlphaScale%d", texUnitIndex),
-              overlaySettings.alphaScale);
-        }
+      for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) {
+        TextureOverlay overlay = overlays.get(texUnitIndex - 1);
+        glProgram.setSamplerTexIdUniform(
+            Util.formatInvariant("uOverlayTexSampler%d", texUnitIndex),
+            overlay.getTextureId(presentationTimeUs),
+            texUnitIndex);
+        glProgram.setFloatsUniform(
+            Util.formatInvariant("uVertexTransformationMatrix%d", texUnitIndex),
+            overlay.getVertexTransformation(presentationTimeUs));
+        OverlaySettings overlaySettings = overlay.getOverlaySettings(presentationTimeUs);
+        Size overlaySize = overlay.getTextureSize(presentationTimeUs);
+        glProgram.setFloatsUniform(
+            Util.formatInvariant("uTransformationMatrix%d", texUnitIndex),
+            samplerOverlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings));
+        glProgram.setFloatUniform(
+            Util.formatInvariant("uOverlayAlphaScale%d", texUnitIndex), overlaySettings.alphaScale);
       }
+
       glProgram.setSamplerTexIdUniform("uVideoTexSampler0", inputTexId, /* texUnitIndex= */ 0);
       glProgram.bindAttributesAndUniforms();
       // The four-vertex triangle strip forms a quad.
@@ -131,6 +129,8 @@ import com.google.common.collect.ImmutableList;
     for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
       shader
           .append(Util.formatInvariant("uniform mat4 uTransformationMatrix%s;\n", texUnitIndex))
+          .append(
+              Util.formatInvariant("uniform mat4 uVertexTransformationMatrix%s;\n", texUnitIndex))
           .append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%s;\n", texUnitIndex));
     }
 
@@ -146,7 +146,9 @@ import com.google.common.collect.ImmutableList;
       shader
           .append(Util.formatInvariant("  vec4 aOverlayPosition%d = \n", texUnitIndex))
           .append(
-              Util.formatInvariant("  uTransformationMatrix%s * aFramePosition;\n", texUnitIndex))
+              Util.formatInvariant(
+                  "  uVertexTransformationMatrix%s * uTransformationMatrix%s * aFramePosition;\n",
+                  texUnitIndex, texUnitIndex))
           .append(
               Util.formatInvariant(
                   "  vOverlayTexSamplingCoord%d = getTexSamplingCoord(aOverlayPosition%d.xy);\n",

libraries/effect/src/main/java/androidx/media3/effect/TextureOverlay.java

@@ -16,12 +16,15 @@
 package androidx.media3.effect;
 
 import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.Size;
 import androidx.media3.common.util.UnstableApi;
 
 /** Creates overlays from OpenGL textures. */
 @UnstableApi
 public abstract class TextureOverlay {
+  private static final float[] IDENTITY_MATRIX = GlUtil.create4x4IdentityMatrix();
+
   /**
    * Returns the overlay texture identifier displayed at the specified timestamp.
    *
@@ -41,6 +44,18 @@ public abstract class TextureOverlay {
    */
   public abstract Size getTextureSize(long presentationTimeUs);
 
+  /**
+   * Returns a 4x4 OpenGL matrix, controlling how the vertices of the overlay are displayed at the
+   * specified timestamp.
+   *
+   * <p>Applied before {@linkplain #getOverlaySettings overlay settings}.
+   *
+   * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
+   */
+  public float[] getVertexTransformation(long presentationTimeUs) {
+    return IDENTITY_MATRIX;
+  }
+
   /**
    * Set up resources for the overlay given the input video’s dimensions.
    *