diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 59c8e9d669..f2aaebef1c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -43,6 +43,7 @@ * Remove `TransformationRequest.HdrMode` annotation type and its associated constants. Use `Composition.HdrMode` and its associated constants instead. + * Simplify the `OverlaySettings` to fix rotation issues. * Track Selection: * Extractors: * MPEG-TS: Ensure the last frame is rendered by passing the last access diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TimerOverlay.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TimerOverlay.java index 98fe7b7cfa..f12aa024e0 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TimerOverlay.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TimerOverlay.java @@ -16,12 +16,10 @@ package androidx.media3.demo.transformer; import android.graphics.Color; -import android.opengl.Matrix; import android.text.Spannable; import android.text.SpannableString; import android.text.style.ForegroundColorSpan; import androidx.media3.common.C; -import androidx.media3.common.util.GlUtil; import androidx.media3.effect.OverlaySettings; import androidx.media3.effect.TextOverlay; import androidx.media3.effect.TextureOverlay; @@ -36,13 +34,12 @@ import java.util.Locale; private final OverlaySettings overlaySettings; public TimerOverlay() { - float[] positioningMatrix = GlUtil.create4x4IdentityMatrix(); - Matrix.translateM( - positioningMatrix, /* mOffset= */ 0, /* x= */ -0.7f, /* y= */ -0.95f, /* z= */ 1); overlaySettings = new OverlaySettings.Builder() - .setAnchor(/* x= */ -1f, /* y= */ -1f) - .setMatrix(positioningMatrix) + // Place the timer in the bottom left corner of the screen with some padding from the + // edges. + .setOverlayAnchor(/* x= */ 1f, /* y= */ 1f) + .setVideoFrameAnchor(/* x= */ -0.7f, /* y= */ -0.95f) .build(); } diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 4222ba1525..69b735e53e 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -31,7 +31,6 @@ import android.graphics.Bitmap; import android.graphics.Color; import android.graphics.drawable.Drawable; import android.net.Uri; -import android.opengl.Matrix; import android.os.Bundle; import android.os.Handler; import android.text.Spannable; @@ -58,7 +57,6 @@ import androidx.media3.common.audio.ChannelMixingAudioProcessor; import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.media3.common.audio.SonicAudioProcessor; import androidx.media3.common.util.BitmapLoader; -import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; import androidx.media3.datasource.DataSourceBitmapLoader; import androidx.media3.effect.BitmapOverlay; @@ -580,13 +578,12 @@ public final class TransformerActivity extends AppCompatActivity { private OverlayEffect createOverlayEffectFromBundle(Bundle bundle, boolean[] selectedEffects) { ImmutableList.Builder overlaysBuilder = new ImmutableList.Builder<>(); if (selectedEffects[ConfigurationActivity.OVERLAY_LOGO_AND_TIMER_INDEX]) { - float[] logoPositioningMatrix = GlUtil.create4x4IdentityMatrix(); - Matrix.translateM( - logoPositioningMatrix, /* mOffset= */ 0, /* x= */ -0.95f, /* y= */ -0.95f, /* z= */ 1); OverlaySettings logoSettings = new OverlaySettings.Builder() - .setMatrix(logoPositioningMatrix) - .setAnchor(/* x= */ -1f, /* y= */ -1f) + // Place the logo in the bottom left corner of the screen with some padding from the + // edges. + .setOverlayAnchor(/* x= */ 1f, /* y= */ 1f) + .setVideoFrameAnchor(/* x= */ -0.95f, /* y= */ -0.95f) .build(); Drawable logo; try { diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayShaderProgramPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayShaderProgramPixelTest.java index fa3f9a4aeb..0659ec7c14 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayShaderProgramPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayShaderProgramPixelTest.java @@ -31,7 +31,6 @@ import android.graphics.Color; import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLSurface; -import android.opengl.Matrix; import android.text.Spannable; import android.text.SpannableString; import android.text.style.ForegroundColorSpan; @@ -64,10 +63,15 @@ public class OverlayShaderProgramPixelTest { "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; private static final String OVERLAY_BITMAP_DEFAULT = "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png"; - private static final String OVERLAY_BITMAP_ANCHORED = + private static final String OVERLAY_BITMAP_DOUBLY_ANCHORED = "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_anchored.png"; + + private static final String OVERLAY_BITMAP_OVERLAY_ANCHORED = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_overlayAnchored.png"; private static final String OVERLAY_BITMAP_SCALED = "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png"; + private static final String OVERLAY_BITMAP_ROTATED90 = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_rotated90.png"; private static final String OVERLAY_BITMAP_TRANSLUCENT = "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_translucent.png"; private static final String OVERLAY_TEXT_DEFAULT = @@ -154,39 +158,23 @@ public class OverlayShaderProgramPixelTest { } @Test - public void drawFrame_scaledBitmapOverlay_letterboxStretchesOverlay() throws Exception { - String testId = "drawFrame_scaledBitmapOverlay"; + public void drawFrame_anchoredAndTranslatedBitmapOverlay_blendsBitmapIntoTopLeftOfFrame() + throws Exception { + String testId = "drawFrame_anchoredAndTranslatedBitmapOverlay"; Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); - float[] scaleMatrix = GlUtil.create4x4IdentityMatrix(); - OverlaySettings overlaySettings = new OverlaySettings.Builder().setMatrix(scaleMatrix).build(); + OverlaySettings overlaySettings = + new OverlaySettings.Builder() + .setOverlayAnchor(/* x= */ 1f, /* y= */ -1f) + .setVideoFrameAnchor(/* x= */ -1f, /* y= */ 1f) + .build(); BitmapOverlay staticBitmapOverlay = - new BitmapOverlay() { - @Override - public Bitmap getBitmap(long presentationTimeUs) { - return overlayBitmap; - } - - @Override - public void configure(Size videoSize) { - Matrix.scaleM( - scaleMatrix, - /* mOffset= */ 0, - /* x= */ videoSize.getWidth() / (float) overlayBitmap.getWidth(), - /* y= */ 1, - /* z= */ 1); - } - - @Override - public OverlaySettings getOverlaySettings(long presentationTimeUs) { - return overlaySettings; - } - }; + BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings); overlayShaderProgram = new OverlayEffect(ImmutableList.of(staticBitmapOverlay)) .toGlShaderProgram(context, /* useHdr= */ false); Size outputSize = overlayShaderProgram.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); - Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_SCALED); + Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_DOUBLY_ANCHORED); overlayShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = @@ -199,13 +187,12 @@ public class OverlayShaderProgramPixelTest { } @Test - public void drawFrame_anchoredBitmapOverlay_blendsBitmapIntoTopLeftOfFrame() throws Exception { - String testId = "drawFrame_anchoredBitmapOverlay"; + public void drawFrame_overlayAnchoredOnlyBitmapOverlay_anchorsOverlayFromTopLeftCornerOfFrame() + throws Exception { + String testId = "drawFrame_anchoredAndTranslatedBitmapOverlay"; Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); - float[] translateMatrix = GlUtil.create4x4IdentityMatrix(); - Matrix.translateM(translateMatrix, /* mOffset= */ 0, /* x= */ -1f, /* y= */ 1f, /* z= */ 1); OverlaySettings overlaySettings = - new OverlaySettings.Builder().setMatrix(translateMatrix).setAnchor(-1f, 1f).build(); + new OverlaySettings.Builder().setOverlayAnchor(/* x= */ 1f, /* y= */ -1f).build(); BitmapOverlay staticBitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings); overlayShaderProgram = @@ -213,7 +200,31 @@ public class OverlayShaderProgramPixelTest { .toGlShaderProgram(context, /* useHdr= */ false); Size outputSize = overlayShaderProgram.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); - Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_ANCHORED); + Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_OVERLAY_ANCHORED); + + overlayShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_rotatedBitmapOverlay_blendsBitmapRotated90degrees() throws Exception { + String testId = "drawFrame_rotatedBitmapOverlay"; + Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); + OverlaySettings overlaySettings = new OverlaySettings.Builder().setRotationDegrees(90f).build(); + BitmapOverlay staticBitmapOverlay = + BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings); + overlayShaderProgram = + new OverlayEffect(ImmutableList.of(staticBitmapOverlay)) + .toGlShaderProgram(context, /* useHdr= */ false); + Size outputSize = overlayShaderProgram.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_ROTATED90); overlayShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = @@ -333,10 +344,9 @@ public class OverlayShaderProgramPixelTest { } @Test - public void drawFrame_translatedTextOverlay_blendsTextIntoFrame() throws Exception { - String testId = "drawFrame_translatedTextOverlay"; - float[] translateMatrix = GlUtil.create4x4IdentityMatrix(); - Matrix.translateM(translateMatrix, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1); + public void drawFrame_anchoredTextOverlay_blendsTextIntoTheTopRightQuadrantOfFrame() + throws Exception { + String testId = "drawFrame_anchoredTextOverlay"; SpannableString overlayText = new SpannableString(/* source= */ "Text styling"); overlayText.setSpan( new ForegroundColorSpan(Color.GRAY), @@ -344,7 +354,7 @@ public class OverlayShaderProgramPixelTest { /* end= */ 4, Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); OverlaySettings overlaySettings = - new OverlaySettings.Builder().setMatrix(translateMatrix).build(); + new OverlaySettings.Builder().setVideoFrameAnchor(0.5f, 0.5f).build(); TextOverlay staticTextOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings); overlayShaderProgram = @@ -367,8 +377,6 @@ public class OverlayShaderProgramPixelTest { @Test public void drawFrame_multipleOverlays_blendsBothIntoFrame() throws Exception { String testId = "drawFrame_multipleOverlays"; - float[] translateMatrix1 = GlUtil.create4x4IdentityMatrix(); - Matrix.translateM(translateMatrix1, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1); SpannableString overlayText = new SpannableString(/* source= */ "Overlay 1"); overlayText.setSpan( new ForegroundColorSpan(Color.GRAY), @@ -376,7 +384,7 @@ public class OverlayShaderProgramPixelTest { /* end= */ 4, Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); OverlaySettings overlaySettings1 = - new OverlaySettings.Builder().setMatrix(translateMatrix1).build(); + new OverlaySettings.Builder().setVideoFrameAnchor(0.5f, 0.5f).build(); TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1); Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setAlpha(0.5f).build(); @@ -407,15 +415,12 @@ public class OverlayShaderProgramPixelTest { /* start= */ 0, /* end= */ overlayText.length(), Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); - float[] scaleTextMatrix = GlUtil.create4x4IdentityMatrix(); - Matrix.scaleM(scaleTextMatrix, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1); OverlaySettings overlaySettings1 = - new OverlaySettings.Builder().setMatrix(scaleTextMatrix).build(); + new OverlaySettings.Builder().setScale(/* x= */ 0.5f, /* y= */ 0.5f).build(); TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1); Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); - float[] scaleMatrix = GlUtil.create4x4IdentityMatrix(); - Matrix.scaleM(scaleMatrix, /* mOffset= */ 0, /* x= */ 3, /* y= */ 3, /* z= */ 1); - OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setMatrix(scaleMatrix).build(); + OverlaySettings overlaySettings2 = + new OverlaySettings.Builder().setScale(/* x= */ 3, /* y= */ 3).build(); BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2); overlayShaderProgram = @@ -435,6 +440,57 @@ public class OverlayShaderProgramPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void drawFrame_scaledBitmapOverlay_letterboxStretchesOverlay() throws Exception { + String testId = "drawFrame_scaledBitmapOverlay"; + Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); + overlayShaderProgram = + new OverlayEffect(ImmutableList.of(new LetterBoxStretchedBitmapOverlay(overlayBitmap))) + .toGlShaderProgram(context, /* useHdr= */ false); + Size outputSize = overlayShaderProgram.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_SCALED); + + overlayShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + private static final class LetterBoxStretchedBitmapOverlay extends BitmapOverlay { + + private final OverlaySettings.Builder overlaySettingsBuilder; + private final Bitmap overlayBitmap; + + private @MonotonicNonNull OverlaySettings overlaySettings; + + public LetterBoxStretchedBitmapOverlay(Bitmap overlayBitmap) { + this.overlayBitmap = overlayBitmap; + overlaySettingsBuilder = new OverlaySettings.Builder(); + } + + @Override + public Bitmap getBitmap(long presentationTimeUs) { + return overlayBitmap; + } + + @Override + public void configure(Size videoSize) { + overlaySettingsBuilder.setScale( + /* x= */ videoSize.getWidth() / (float) overlayBitmap.getWidth(), /* y= */ 1); + overlaySettings = overlaySettingsBuilder.build(); + } + + @Override + public OverlaySettings getOverlaySettings(long presentationTimeUs) { + return checkNotNull(overlaySettings); + } + } + private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { int outputTexId = GlUtil.createTexture( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java index eaa6919304..b5c0372525 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java @@ -19,37 +19,50 @@ import static androidx.media3.common.util.Assertions.checkArgument; import android.util.Pair; import androidx.annotation.FloatRange; -import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; import com.google.errorprone.annotations.CanIgnoreReturnValue; -/** Contains information to control how an {@link TextureOverlay} is displayed on the screen. */ +/** Contains information to control how a {@link TextureOverlay} is displayed on the screen. */ @UnstableApi public final class OverlaySettings { public final boolean useHdr; public final float alpha; - public final float[] matrix; - public final Pair anchor; + public final Pair videoFrameAnchor; + public final Pair overlayAnchor; + public final Pair scale; + public final float rotationDegrees; - private OverlaySettings(boolean useHdr, float alpha, float[] matrix, Pair anchor) { + private OverlaySettings( + boolean useHdr, + float alpha, + Pair videoFrameAnchor, + Pair overlayAnchor, + Pair scale, + float rotationDegrees) { this.useHdr = useHdr; this.alpha = alpha; - this.matrix = matrix; - this.anchor = anchor; + this.videoFrameAnchor = videoFrameAnchor; + this.overlayAnchor = overlayAnchor; + this.scale = scale; + this.rotationDegrees = rotationDegrees; } /** A builder for {@link OverlaySettings} instances. */ public static final class Builder { private boolean useHdr; private float alpha; - private float[] matrix; - private Pair anchor; + private Pair videoFrameAnchor; + private Pair overlayAnchor; + private Pair scale; + private float rotationDegrees; /** Creates a new {@link Builder}. */ public Builder() { alpha = 1f; - matrix = GlUtil.create4x4IdentityMatrix(); - anchor = Pair.create(0f, 0f); + videoFrameAnchor = Pair.create(0f, 0f); + overlayAnchor = Pair.create(0f, 0f); + scale = Pair.create(1f, 1f); + rotationDegrees = 0f; } /** @@ -64,18 +77,6 @@ public final class OverlaySettings { return this; } - /** - * Sets the {@link android.opengl.Matrix} used to transform the overlay before applying it to a - * frame. - * - *

Set to always return the identity matrix by default. - */ - @CanIgnoreReturnValue - public Builder setMatrix(float[] matrix) { - this.matrix = matrix; - return this; - } - /** * Sets the alpha value of the overlay, altering its transparency. * @@ -91,28 +92,79 @@ public final class OverlaySettings { } /** - * Sets the coordinates for the anchor point of the overlay. + * Sets the coordinates for the anchor point of the overlay within the video frame. * - *

The anchor point is the point inside the overlay that the overlay is positioned from. + *

The coordinates are specified in Normalised Device Coordinates (NDCs) relative to the + * video frame. Set to always return {@code (0,0)} (the center of the video frame) by default. * - *

The coordinates are specified in Normalised Device Coordinates (NDCs). Set to always - * return {@code (0,0)} (the center) by default. + *

For example, a value of {@code (+1,+1)} will move the overlay's {@linkplain + * #setOverlayAnchor anchor point} to the top right corner of the video frame. * * @param x The NDC x-coordinate in the range [-1, 1]. * @param y The NDC y-coordinate in the range [-1, 1]. */ @CanIgnoreReturnValue - public Builder setAnchor( + public Builder setVideoFrameAnchor( @FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) { checkArgument(-1 <= x && x <= 1); checkArgument(-1 <= y && y <= 1); - this.anchor = Pair.create(x, y); + this.videoFrameAnchor = Pair.create(x, y); + return this; + } + + /** + * Sets the coordinates for the anchor point of the overlay. + * + *

The anchor point is the point inside the overlay that is placed on the {@linkplain + * #setVideoFrameAnchor video frame anchor} + * + *

The coordinates are specified in Normalised Device Coordinates (NDCs) relative to the + * overlay frame. Set to return {@code (0,0)} (the center of the overlay) by default. + * + *

For example, a value of {@code (+1,-1)} will result in the overlay being positioned from + * the bottom right corner of its frame. + * + * @param x The NDC x-coordinate in the range [-1, 1]. + * @param y The NDC y-coordinate in the range [-1, 1]. + */ + @CanIgnoreReturnValue + public Builder setOverlayAnchor( + @FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) { + checkArgument(-1 <= x && x <= 1); + checkArgument(-1 <= y && y <= 1); + this.overlayAnchor = Pair.create(x, y); + return this; + } + + /** + * Sets the scaling of the overlay. + * + * @param x The desired scaling in the x axis of the overlay. + * @param y The desired scaling in the y axis of the overlay. + */ + @CanIgnoreReturnValue + public Builder setScale(float x, float y) { + this.scale = Pair.create(x, y); + return this; + } + + /** + * Sets the rotation of the overlay, counter-clockwise. + * + *

The overlay is rotated at the center of its frame. + * + * @param rotationDegree The desired degrees of rotation, counter-clockwise. + */ + @CanIgnoreReturnValue + public Builder setRotationDegrees(float rotationDegree) { + this.rotationDegrees = rotationDegree; return this; } /** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */ public OverlaySettings build() { - return new OverlaySettings(useHdr, alpha, matrix, anchor); + return new OverlaySettings( + useHdr, alpha, videoFrameAnchor, overlayAnchor, scale, rotationDegrees); } } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java index d32ef91d44..c6526dc881 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java @@ -35,9 +35,16 @@ import com.google.common.collect.ImmutableList; private final GlProgram glProgram; private final ImmutableList overlays; + private final float[] videoFrameAnchorMatrix; + private final float[] videoFrameAnchorMatrixInv; private final float[] aspectRatioMatrix; - private final float[] overlayMatrix; - private final float[] anchorMatrix; + private final float[] scaleMatrix; + private final float[] scaleMatrixInv; + private final float[] overlayAnchorMatrix; + private final float[] overlayAnchorMatrixInv; + private final float[] rotateMatrix; + private final float[] overlayAspectRatioMatrix; + private final float[] overlayAspectRatioMatrixInv; private final float[] transformationMatrix; private int videoWidth; @@ -63,8 +70,15 @@ import com.google.common.collect.ImmutableList; "OverlayShaderProgram does not support more than 15 overlays in the same instance."); this.overlays = overlays; aspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); - overlayMatrix = GlUtil.create4x4IdentityMatrix(); - anchorMatrix = GlUtil.create4x4IdentityMatrix(); + videoFrameAnchorMatrix = GlUtil.create4x4IdentityMatrix(); + videoFrameAnchorMatrixInv = GlUtil.create4x4IdentityMatrix(); + overlayAnchorMatrix = GlUtil.create4x4IdentityMatrix(); + overlayAnchorMatrixInv = GlUtil.create4x4IdentityMatrix(); + rotateMatrix = GlUtil.create4x4IdentityMatrix(); + scaleMatrix = GlUtil.create4x4IdentityMatrix(); + scaleMatrixInv = GlUtil.create4x4IdentityMatrix(); + overlayAspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); + overlayAspectRatioMatrixInv = GlUtil.create4x4IdentityMatrix(); transformationMatrix = GlUtil.create4x4IdentityMatrix(); try { glProgram = @@ -105,43 +119,163 @@ import com.google.common.collect.ImmutableList; texUnitIndex); GlUtil.setToIdentity(aspectRatioMatrix); + GlUtil.setToIdentity(videoFrameAnchorMatrix); + GlUtil.setToIdentity(videoFrameAnchorMatrixInv); + GlUtil.setToIdentity(overlayAnchorMatrix); + GlUtil.setToIdentity(overlayAnchorMatrixInv); + GlUtil.setToIdentity(scaleMatrix); + GlUtil.setToIdentity(scaleMatrixInv); + GlUtil.setToIdentity(rotateMatrix); + GlUtil.setToIdentity(overlayAspectRatioMatrix); + GlUtil.setToIdentity(overlayAspectRatioMatrixInv); + GlUtil.setToIdentity(transformationMatrix); + + // Anchor point of overlay within output frame. + Pair videoFrameAnchor = + overlay.getOverlaySettings(presentationTimeUs).videoFrameAnchor; + Matrix.translateM( + videoFrameAnchorMatrix, + MATRIX_OFFSET, + videoFrameAnchor.first, + videoFrameAnchor.second, + /* z= */ 0f); + Matrix.invertM( + videoFrameAnchorMatrixInv, MATRIX_OFFSET, videoFrameAnchorMatrix, MATRIX_OFFSET); + Matrix.scaleM( aspectRatioMatrix, MATRIX_OFFSET, videoWidth / (float) overlay.getTextureSize(presentationTimeUs).getWidth(), videoHeight / (float) overlay.getTextureSize(presentationTimeUs).getHeight(), - /* z= */ 1); - Matrix.invertM( - overlayMatrix, + /* z= */ 1f); + + // Scale the image. + Pair scale = overlay.getOverlaySettings(presentationTimeUs).scale; + Matrix.setIdentityM(scaleMatrix, MATRIX_OFFSET); + Matrix.scaleM( + scaleMatrix, MATRIX_OFFSET, - overlay.getOverlaySettings(presentationTimeUs).matrix, - MATRIX_OFFSET); - Pair overlayAnchor = overlay.getOverlaySettings(presentationTimeUs).anchor; - GlUtil.setToIdentity(anchorMatrix); + scaleMatrix, + MATRIX_OFFSET, + scale.first, + scale.second, + /* z= */ 1f); + Matrix.invertM(scaleMatrixInv, MATRIX_OFFSET, scaleMatrix, MATRIX_OFFSET); + + // Translate the overlay within its frame. + Pair overlayAnchor = + overlay.getOverlaySettings(presentationTimeUs).overlayAnchor; + Matrix.setIdentityM(overlayAnchorMatrix, MATRIX_OFFSET); Matrix.translateM( - anchorMatrix, - /* mOffset= */ 0, - overlayAnchor.first - * overlay.getTextureSize(presentationTimeUs).getWidth() - / videoWidth, - overlayAnchor.second - * overlay.getTextureSize(presentationTimeUs).getHeight() - / videoHeight, - /* z= */ 1); + overlayAnchorMatrix, + MATRIX_OFFSET, + overlayAnchor.first, + overlayAnchor.second, + /* z= */ 0f); + Matrix.invertM(overlayAnchorMatrixInv, MATRIX_OFFSET, overlayAnchorMatrix, MATRIX_OFFSET); + + // Rotate the image. + Matrix.setIdentityM(rotateMatrix, MATRIX_OFFSET); + Matrix.rotateM( + rotateMatrix, + MATRIX_OFFSET, + rotateMatrix, + MATRIX_OFFSET, + overlay.getOverlaySettings(presentationTimeUs).rotationDegrees, + /* x= */ 0f, + /* y= */ 0f, + /* z= */ 1f); + Matrix.invertM(rotateMatrix, MATRIX_OFFSET, rotateMatrix, MATRIX_OFFSET); + + // Rotation matrix needs to account for overlay aspect ratio to prevent stretching. + Matrix.scaleM( + overlayAspectRatioMatrix, + MATRIX_OFFSET, + (float) overlay.getTextureSize(presentationTimeUs).getHeight() + / (float) overlay.getTextureSize(presentationTimeUs).getWidth(), + /* y= */ 1f, + /* z= */ 1f); + Matrix.invertM( + overlayAspectRatioMatrixInv, MATRIX_OFFSET, overlayAspectRatioMatrix, MATRIX_OFFSET); + + // Rotation needs to be agnostic of the scaling matrix and the aspect ratios. Matrix.multiplyMM( transformationMatrix, MATRIX_OFFSET, - overlayMatrix, + transformationMatrix, MATRIX_OFFSET, - anchorMatrix, + scaleMatrixInv, MATRIX_OFFSET); + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAspectRatioMatrix, + MATRIX_OFFSET); + + // Rotation matrix. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + rotateMatrix, + MATRIX_OFFSET); + + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAspectRatioMatrixInv, + MATRIX_OFFSET); + + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + scaleMatrix, + MATRIX_OFFSET); + + // Translate image. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAnchorMatrixInv, + MATRIX_OFFSET); + + // Scale image. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + scaleMatrixInv, + MATRIX_OFFSET); + + // Correct for aspect ratio of image in output frame. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, transformationMatrix, MATRIX_OFFSET, aspectRatioMatrix, + MATRIX_OFFSET); + + // Anchor position in output frame. + Matrix.multiplyMM( + transformationMatrix, MATRIX_OFFSET, transformationMatrix, + MATRIX_OFFSET, + videoFrameAnchorMatrixInv, MATRIX_OFFSET); + glProgram.setFloatsUniform( Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), transformationMatrix); diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_overlayAnchored.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_overlayAnchored.png new file mode 100644 index 0000000000..2a1898db8e Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_overlayAnchored.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_rotated90.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_rotated90.png new file mode 100644 index 0000000000..9789de8bfc Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_rotated90.png differ