From dc037b22cd6ed5f138e3b1b277d8e4807ba61fd3 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Wed, 22 Nov 2023 10:14:28 -0800 Subject: [PATCH] Effect: Flip texture in OpenGL instead of allocating a Bitmap. Reduce short-lived allocations of potentially large objects, like Bitmap. Unfortunately, this does make the TextureOverlay interface more messy though, requiring a way to signal whether the texture should be flipped vertically. PiperOrigin-RevId: 584661400 --- .../androidx/media3/effect/BitmapOverlay.java | 26 +++++++++--- .../androidx/media3/effect/BitmapUtil.java | 38 ----------------- .../media3/effect/OverlayShaderProgram.java | 42 ++++++++++--------- .../media3/effect/TextureOverlay.java | 15 +++++++ 4 files changed, 58 insertions(+), 63 deletions(-) delete mode 100644 libraries/effect/src/main/java/androidx/media3/effect/BitmapUtil.java diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java index feb7e95297..d171b18349 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import android.content.Context; import android.graphics.Bitmap; import android.net.Uri; +import android.opengl.Matrix; import androidx.media3.common.C; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.BitmapLoader; @@ -39,10 +40,17 @@ import org.checkerframework.checker.nullness.qual.Nullable; */ @UnstableApi public abstract class BitmapOverlay extends TextureOverlay { + + private final float[] flipVerticallyMatrix; + private int lastTextureId; private @Nullable Bitmap lastBitmap; - BitmapOverlay() { + /* package */ BitmapOverlay() { + float[] temp = GlUtil.create4x4IdentityMatrix(); + Matrix.scaleM(temp, /* offset */ 0, /* x= */ 1f, /* y= */ -1f, /* z= */ 1f); + flipVerticallyMatrix = temp; + lastTextureId = C.INDEX_UNSET; } @@ -72,11 +80,10 @@ public abstract class BitmapOverlay extends TextureOverlay { if (bitmap != lastBitmap) { try { lastBitmap = bitmap; - if (lastTextureId != -1) { + if (lastTextureId != C.INDEX_UNSET) { GlUtil.deleteTexture(lastTextureId); } - lastTextureId = - GlUtil.createTexture(BitmapUtil.flipBitmapVertically(checkNotNull(lastBitmap))); + lastTextureId = GlUtil.createTexture(checkNotNull(lastBitmap)); } catch (GlUtil.GlException e) { throw new VideoFrameProcessingException(e); } @@ -84,6 +91,15 @@ public abstract class BitmapOverlay extends TextureOverlay { return lastTextureId; } + @Override + public float[] getVertexTransformation(long presentationTimeUs) { + // Whereas the Android system uses the top-left corner as (0,0) of the + // coordinate system, OpenGL uses the bottom-left corner as (0,0), so the + // texture gets flipped. Flip the texture vertically to ensure the + // orientation of the output is correct. + return flipVerticallyMatrix; + } + /** * Creates a {@link BitmapOverlay} that shows the {@code overlayBitmap} in the same position and * size throughout the whole video. @@ -164,7 +180,7 @@ public abstract class BitmapOverlay extends TextureOverlay { public void release() throws VideoFrameProcessingException { super.release(); lastBitmap = null; - if (lastTextureId != -1) { + if (lastTextureId != C.INDEX_UNSET) { try { GlUtil.deleteTexture(lastTextureId); } catch (GlUtil.GlException e) { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapUtil.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapUtil.java deleted file mode 100644 index 5375c20684..0000000000 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapUtil.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.effect; - -import android.graphics.Bitmap; -import android.graphics.Matrix; - -/** Utility functions for working with {@link Bitmap}. */ -/* package */ final class BitmapUtil { - public static Bitmap flipBitmapVertically(Bitmap bitmap) { - Matrix flip = new Matrix(); - flip.postScale(1f, -1f); - return Bitmap.createBitmap( - bitmap, - /* x= */ 0, - /* y= */ 0, - bitmap.getWidth(), - bitmap.getHeight(), - flip, - /* filter= */ true); - } - - /** Class only contains static methods. */ - private BitmapUtil() {} -} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java index 7a9dd98444..a21ebcc76a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java @@ -78,26 +78,24 @@ import com.google.common.collect.ImmutableList; throws VideoFrameProcessingException { try { glProgram.use(); - if (!overlays.isEmpty()) { - for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) { - TextureOverlay overlay = overlays.get(texUnitIndex - 1); - - glProgram.setSamplerTexIdUniform( - Util.formatInvariant("uOverlayTexSampler%d", texUnitIndex), - overlay.getTextureId(presentationTimeUs), - texUnitIndex); - OverlaySettings overlaySettings = overlay.getOverlaySettings(presentationTimeUs); - Size overlaySize = overlay.getTextureSize(presentationTimeUs); - - glProgram.setFloatsUniform( - Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), - samplerOverlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings)); - - glProgram.setFloatUniform( - Util.formatInvariant("uOverlayAlphaScale%d", texUnitIndex), - overlaySettings.alphaScale); - } + for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) { + TextureOverlay overlay = overlays.get(texUnitIndex - 1); + glProgram.setSamplerTexIdUniform( + Util.formatInvariant("uOverlayTexSampler%d", texUnitIndex), + overlay.getTextureId(presentationTimeUs), + texUnitIndex); + glProgram.setFloatsUniform( + Util.formatInvariant("uVertexTransformationMatrix%d", texUnitIndex), + overlay.getVertexTransformation(presentationTimeUs)); + OverlaySettings overlaySettings = overlay.getOverlaySettings(presentationTimeUs); + Size overlaySize = overlay.getTextureSize(presentationTimeUs); + glProgram.setFloatsUniform( + Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), + samplerOverlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings)); + glProgram.setFloatUniform( + Util.formatInvariant("uOverlayAlphaScale%d", texUnitIndex), overlaySettings.alphaScale); } + glProgram.setSamplerTexIdUniform("uVideoTexSampler0", inputTexId, /* texUnitIndex= */ 0); glProgram.bindAttributesAndUniforms(); // The four-vertex triangle strip forms a quad. @@ -131,6 +129,8 @@ import com.google.common.collect.ImmutableList; for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) { shader .append(Util.formatInvariant("uniform mat4 uTransformationMatrix%s;\n", texUnitIndex)) + .append( + Util.formatInvariant("uniform mat4 uVertexTransformationMatrix%s;\n", texUnitIndex)) .append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%s;\n", texUnitIndex)); } @@ -146,7 +146,9 @@ import com.google.common.collect.ImmutableList; shader .append(Util.formatInvariant(" vec4 aOverlayPosition%d = \n", texUnitIndex)) .append( - Util.formatInvariant(" uTransformationMatrix%s * aFramePosition;\n", texUnitIndex)) + Util.formatInvariant( + " uVertexTransformationMatrix%s * uTransformationMatrix%s * aFramePosition;\n", + texUnitIndex, texUnitIndex)) .append( Util.formatInvariant( " vOverlayTexSamplingCoord%d = getTexSamplingCoord(aOverlayPosition%d.xy);\n", diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureOverlay.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureOverlay.java index 6a7c2b0c40..04cc186b3f 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TextureOverlay.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureOverlay.java @@ -16,12 +16,15 @@ package androidx.media3.effect; import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; /** Creates overlays from OpenGL textures. */ @UnstableApi public abstract class TextureOverlay { + private static final float[] IDENTITY_MATRIX = GlUtil.create4x4IdentityMatrix(); + /** * Returns the overlay texture identifier displayed at the specified timestamp. * @@ -41,6 +44,18 @@ public abstract class TextureOverlay { */ public abstract Size getTextureSize(long presentationTimeUs); + /** + * Returns a 4x4 OpenGL matrix, controlling how the vertices of the overlay are displayed at the + * specified timestamp. + * + *

Applied before {@linkplain #getOverlaySettings overlay settings}. + * + * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + */ + public float[] getVertexTransformation(long presentationTimeUs) { + return IDENTITY_MATRIX; + } + /** * Set up resources for the overlay given the input video’s dimensions. *