loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update OverlaySettings to explicitly state overlay transformations

Browse Source 
Alters the OverlayShaderProgram implementation to support rotations, however we need to apply the transformations separately in order for them to work as expected so the matrix is removed from the interface in favour of explicit methods.

Adds a rotation test to ensure this ability doesn't regress

PiperOrigin-RevId: 549890847
This commit is contained in:
tofunmi 2023-07-21 10:43:55 +01:00 committed by Rohit Singh
parent d658de5944
commit 7eee15ecb4
8 changed files with 351 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
RELEASENOTES.md
View File

@ -43,6 +43,7 @@
* Remove `TransformationRequest.HdrMode` annotation type and its
associated constants. Use `Composition.HdrMode` and its associated
constants instead.
* Simplify the `OverlaySettings` to fix rotation issues.
* Track Selection:
* Extractors:
* MPEG-TS: Ensure the last frame is rendered by passing the last access

11
demos/transformer/src/main/java/androidx/media3/demo/transformer/TimerOverlay.java
View File

@ -16,12 +16,10 @@
package androidx.media3.demo.transformer;
import android.graphics.Color;
import android.opengl.Matrix;
import android.text.Spannable;
import android.text.SpannableString;
import android.text.style.ForegroundColorSpan;
import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.TextOverlay;
import androidx.media3.effect.TextureOverlay;
@ -36,13 +34,12 @@ import java.util.Locale;
private final OverlaySettings overlaySettings;
public TimerOverlay() {
float[] positioningMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(
positioningMatrix, /* mOffset= */ 0, /* x= */ -0.7f, /* y= */ -0.95f, /* z= */ 1);
overlaySettings =
new OverlaySettings.Builder()
.setAnchor(/* x= */ -1f, /* y= */ -1f)
.setMatrix(positioningMatrix)
// Place the timer in the bottom left corner of the screen with some padding from the
// edges.
.setOverlayAnchor(/* x= */ 1f, /* y= */ 1f)
.setVideoFrameAnchor(/* x= */ -0.7f, /* y= */ -0.95f)
.build();
}

11
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -31,7 +31,6 @@ import android.graphics.Bitmap;
import android.graphics.Color;
import android.graphics.drawable.Drawable;
import android.net.Uri;
import android.opengl.Matrix;
import android.os.Bundle;
import android.os.Handler;
import android.text.Spannable;
@ -58,7 +57,6 @@ import androidx.media3.common.audio.ChannelMixingAudioProcessor;
import androidx.media3.common.audio.ChannelMixingMatrix;
import androidx.media3.common.audio.SonicAudioProcessor;
import androidx.media3.common.util.BitmapLoader;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.datasource.DataSourceBitmapLoader;
import androidx.media3.effect.BitmapOverlay;
@ -580,13 +578,12 @@ public final class TransformerActivity extends AppCompatActivity {
private OverlayEffect createOverlayEffectFromBundle(Bundle bundle, boolean[] selectedEffects) {
ImmutableList.Builder<TextureOverlay> overlaysBuilder = new ImmutableList.Builder<>();
if (selectedEffects[ConfigurationActivity.OVERLAY_LOGO_AND_TIMER_INDEX]) {
float[] logoPositioningMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(
logoPositioningMatrix, /* mOffset= */ 0, /* x= */ -0.95f, /* y= */ -0.95f, /* z= */ 1);
OverlaySettings logoSettings =
new OverlaySettings.Builder()
.setMatrix(logoPositioningMatrix)
.setAnchor(/* x= */ -1f, /* y= */ -1f)
// Place the logo in the bottom left corner of the screen with some padding from the
// edges.
.setOverlayAnchor(/* x= */ 1f, /* y= */ 1f)
.setVideoFrameAnchor(/* x= */ -0.95f, /* y= */ -0.95f)
.build();
Drawable logo;
try {

152
libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayShaderProgramPixelTest.java
View File

@ -31,7 +31,6 @@ import android.graphics.Color;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLSurface;
import android.opengl.Matrix;
import android.text.Spannable;
import android.text.SpannableString;
import android.text.style.ForegroundColorSpan;
@ -64,10 +63,15 @@ public class OverlayShaderProgramPixelTest {
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
private static final String OVERLAY_BITMAP_DEFAULT =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png";
private static final String OVERLAY_BITMAP_ANCHORED =
private static final String OVERLAY_BITMAP_DOUBLY_ANCHORED =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_anchored.png";
private static final String OVERLAY_BITMAP_OVERLAY_ANCHORED =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_overlayAnchored.png";
private static final String OVERLAY_BITMAP_SCALED =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png";
private static final String OVERLAY_BITMAP_ROTATED90 =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_rotated90.png";
private static final String OVERLAY_BITMAP_TRANSLUCENT =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_translucent.png";
private static final String OVERLAY_TEXT_DEFAULT =
@ -154,39 +158,23 @@ public class OverlayShaderProgramPixelTest {
}
@Test
public void drawFrame_scaledBitmapOverlay_letterboxStretchesOverlay() throws Exception {
String testId = "drawFrame_scaledBitmapOverlay";
public void drawFrame_anchoredAndTranslatedBitmapOverlay_blendsBitmapIntoTopLeftOfFrame()
throws Exception {
String testId = "drawFrame_anchoredAndTranslatedBitmapOverlay";
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
float[] scaleMatrix = GlUtil.create4x4IdentityMatrix();
OverlaySettings overlaySettings = new OverlaySettings.Builder().setMatrix(scaleMatrix).build();
OverlaySettings overlaySettings =
new OverlaySettings.Builder()
.setOverlayAnchor(/* x= */ 1f, /* y= */ -1f)
.setVideoFrameAnchor(/* x= */ -1f, /* y= */ 1f)
.build();
BitmapOverlay staticBitmapOverlay =
new BitmapOverlay() {
@Override
public Bitmap getBitmap(long presentationTimeUs) {
return overlayBitmap;
}
@Override
public void configure(Size videoSize) {
Matrix.scaleM(
scaleMatrix,
/* mOffset= */ 0,
/* x= */ videoSize.getWidth() / (float) overlayBitmap.getWidth(),
/* y= */ 1,
/* z= */ 1);
}
@Override
public OverlaySettings getOverlaySettings(long presentationTimeUs) {
return overlaySettings;
}
};
BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings);
overlayShaderProgram =
new OverlayEffect(ImmutableList.of(staticBitmapOverlay))
.toGlShaderProgram(context, /* useHdr= */ false);
Size outputSize = overlayShaderProgram.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_SCALED);
Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_DOUBLY_ANCHORED);
overlayShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
@ -199,13 +187,12 @@ public class OverlayShaderProgramPixelTest {
}
@Test
public void drawFrame_anchoredBitmapOverlay_blendsBitmapIntoTopLeftOfFrame() throws Exception {
String testId = "drawFrame_anchoredBitmapOverlay";
public void drawFrame_overlayAnchoredOnlyBitmapOverlay_anchorsOverlayFromTopLeftCornerOfFrame()
throws Exception {
String testId = "drawFrame_anchoredAndTranslatedBitmapOverlay";
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
float[] translateMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(translateMatrix, /* mOffset= */ 0, /* x= */ -1f, /* y= */ 1f, /* z= */ 1);
OverlaySettings overlaySettings =
new OverlaySettings.Builder().setMatrix(translateMatrix).setAnchor(-1f, 1f).build();
new OverlaySettings.Builder().setOverlayAnchor(/* x= */ 1f, /* y= */ -1f).build();
BitmapOverlay staticBitmapOverlay =
BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings);
overlayShaderProgram =
@ -213,7 +200,31 @@ public class OverlayShaderProgramPixelTest {
.toGlShaderProgram(context, /* useHdr= */ false);
Size outputSize = overlayShaderProgram.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_ANCHORED);
Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_OVERLAY_ANCHORED);
overlayShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void drawFrame_rotatedBitmapOverlay_blendsBitmapRotated90degrees() throws Exception {
String testId = "drawFrame_rotatedBitmapOverlay";
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings = new OverlaySettings.Builder().setRotationDegrees(90f).build();
BitmapOverlay staticBitmapOverlay =
BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings);
overlayShaderProgram =
new OverlayEffect(ImmutableList.of(staticBitmapOverlay))
.toGlShaderProgram(context, /* useHdr= */ false);
Size outputSize = overlayShaderProgram.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_ROTATED90);
overlayShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
@ -333,10 +344,9 @@ public class OverlayShaderProgramPixelTest {
}
@Test
public void drawFrame_translatedTextOverlay_blendsTextIntoFrame() throws Exception {
String testId = "drawFrame_translatedTextOverlay";
float[] translateMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(translateMatrix, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1);
public void drawFrame_anchoredTextOverlay_blendsTextIntoTheTopRightQuadrantOfFrame()
throws Exception {
String testId = "drawFrame_anchoredTextOverlay";
SpannableString overlayText = new SpannableString(/* source= */ "Text styling");
overlayText.setSpan(
new ForegroundColorSpan(Color.GRAY),
@ -344,7 +354,7 @@ public class OverlayShaderProgramPixelTest {
/* end= */ 4,
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
OverlaySettings overlaySettings =
new OverlaySettings.Builder().setMatrix(translateMatrix).build();
new OverlaySettings.Builder().setVideoFrameAnchor(0.5f, 0.5f).build();
TextOverlay staticTextOverlay =
TextOverlay.createStaticTextOverlay(overlayText, overlaySettings);
overlayShaderProgram =
@ -367,8 +377,6 @@ public class OverlayShaderProgramPixelTest {
@Test
public void drawFrame_multipleOverlays_blendsBothIntoFrame() throws Exception {
String testId = "drawFrame_multipleOverlays";
float[] translateMatrix1 = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(translateMatrix1, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1);
SpannableString overlayText = new SpannableString(/* source= */ "Overlay 1");
overlayText.setSpan(
new ForegroundColorSpan(Color.GRAY),
@ -376,7 +384,7 @@ public class OverlayShaderProgramPixelTest {
/* end= */ 4,
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
OverlaySettings overlaySettings1 =
new OverlaySettings.Builder().setMatrix(translateMatrix1).build();
new OverlaySettings.Builder().setVideoFrameAnchor(0.5f, 0.5f).build();
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1);
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setAlpha(0.5f).build();
@ -407,15 +415,12 @@ public class OverlayShaderProgramPixelTest {
/* start= */ 0,
/* end= */ overlayText.length(),
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
float[] scaleTextMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.scaleM(scaleTextMatrix, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1);
OverlaySettings overlaySettings1 =
new OverlaySettings.Builder().setMatrix(scaleTextMatrix).build();
new OverlaySettings.Builder().setScale(/* x= */ 0.5f, /* y= */ 0.5f).build();
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1);
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
float[] scaleMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.scaleM(scaleMatrix, /* mOffset= */ 0, /* x= */ 3, /* y= */ 3, /* z= */ 1);
OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setMatrix(scaleMatrix).build();
OverlaySettings overlaySettings2 =
new OverlaySettings.Builder().setScale(/* x= */ 3, /* y= */ 3).build();
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2);
overlayShaderProgram =
@ -435,6 +440,57 @@ public class OverlayShaderProgramPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void drawFrame_scaledBitmapOverlay_letterboxStretchesOverlay() throws Exception {
String testId = "drawFrame_scaledBitmapOverlay";
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
overlayShaderProgram =
new OverlayEffect(ImmutableList.of(new LetterBoxStretchedBitmapOverlay(overlayBitmap)))
.toGlShaderProgram(context, /* useHdr= */ false);
Size outputSize = overlayShaderProgram.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_SCALED);
overlayShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
private static final class LetterBoxStretchedBitmapOverlay extends BitmapOverlay {
private final OverlaySettings.Builder overlaySettingsBuilder;
private final Bitmap overlayBitmap;
private @MonotonicNonNull OverlaySettings overlaySettings;
public LetterBoxStretchedBitmapOverlay(Bitmap overlayBitmap) {
this.overlayBitmap = overlayBitmap;
overlaySettingsBuilder = new OverlaySettings.Builder();
}
@Override
public Bitmap getBitmap(long presentationTimeUs) {
return overlayBitmap;
}
@Override
public void configure(Size videoSize) {
overlaySettingsBuilder.setScale(
/* x= */ videoSize.getWidth() / (float) overlayBitmap.getWidth(), /* y= */ 1);
overlaySettings = overlaySettingsBuilder.build();
}
@Override
public OverlaySettings getOverlaySettings(long presentationTimeUs) {
return checkNotNull(overlaySettings);
}
}
private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
int outputTexId =
GlUtil.createTexture(

112
libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java
View File

@ -19,37 +19,50 @@ import static androidx.media3.common.util.Assertions.checkArgument;
import android.util.Pair;
import androidx.annotation.FloatRange;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
/** Contains information to control how an {@link TextureOverlay} is displayed on the screen. */
/** Contains information to control how a {@link TextureOverlay} is displayed on the screen. */
@UnstableApi
public final class OverlaySettings {
public final boolean useHdr;
public final float alpha;
public final float[] matrix;
public final Pair<Float, Float> anchor;
public final Pair<Float, Float> videoFrameAnchor;
public final Pair<Float, Float> overlayAnchor;
public final Pair<Float, Float> scale;
public final float rotationDegrees;
private OverlaySettings(boolean useHdr, float alpha, float[] matrix, Pair<Float, Float> anchor) {
private OverlaySettings(
boolean useHdr,
float alpha,
Pair<Float, Float> videoFrameAnchor,
Pair<Float, Float> overlayAnchor,
Pair<Float, Float> scale,
float rotationDegrees) {
this.useHdr = useHdr;
this.alpha = alpha;
this.matrix = matrix;
this.anchor = anchor;
this.videoFrameAnchor = videoFrameAnchor;
this.overlayAnchor = overlayAnchor;
this.scale = scale;
this.rotationDegrees = rotationDegrees;
}
/** A builder for {@link OverlaySettings} instances. */
public static final class Builder {
private boolean useHdr;
private float alpha;
private float[] matrix;
private Pair<Float, Float> anchor;
private Pair<Float, Float> videoFrameAnchor;
private Pair<Float, Float> overlayAnchor;
private Pair<Float, Float> scale;
private float rotationDegrees;
/** Creates a new {@link Builder}. */
public Builder() {
alpha = 1f;
matrix = GlUtil.create4x4IdentityMatrix();
anchor = Pair.create(0f, 0f);
videoFrameAnchor = Pair.create(0f, 0f);
overlayAnchor = Pair.create(0f, 0f);
scale = Pair.create(1f, 1f);
rotationDegrees = 0f;
}
/**
@ -64,18 +77,6 @@ public final class OverlaySettings {
return this;
}
/**
* Sets the {@link android.opengl.Matrix} used to transform the overlay before applying it to a
* frame.
*
* <p>Set to always return the identity matrix by default.
*/
@CanIgnoreReturnValue
public Builder setMatrix(float[] matrix) {
this.matrix = matrix;
return this;
}
/**
* Sets the alpha value of the overlay, altering its transparency.
*
@ -91,28 +92,79 @@ public final class OverlaySettings {
}
/**
* Sets the coordinates for the anchor point of the overlay.
* Sets the coordinates for the anchor point of the overlay within the video frame.
*
* <p>The anchor point is the point inside the overlay that the overlay is positioned from.
* <p>The coordinates are specified in Normalised Device Coordinates (NDCs) relative to the
* video frame. Set to always return {@code (0,0)} (the center of the video frame) by default.
*
* <p>The coordinates are specified in Normalised Device Coordinates (NDCs). Set to always
* return {@code (0,0)} (the center) by default.
* <p>For example, a value of {@code (+1,+1)} will move the overlay's {@linkplain
* #setOverlayAnchor anchor point} to the top right corner of the video frame.
*
* @param x The NDC x-coordinate in the range [-1, 1].
* @param y The NDC y-coordinate in the range [-1, 1].
*/
@CanIgnoreReturnValue
public Builder setAnchor(
public Builder setVideoFrameAnchor(
@FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) {
checkArgument(-1 <= x && x <= 1);
checkArgument(-1 <= y && y <= 1);
this.anchor = Pair.create(x, y);
this.videoFrameAnchor = Pair.create(x, y);
return this;
}
/**
* Sets the coordinates for the anchor point of the overlay.
*
* <p>The anchor point is the point inside the overlay that is placed on the {@linkplain
* #setVideoFrameAnchor video frame anchor}
*
* <p>The coordinates are specified in Normalised Device Coordinates (NDCs) relative to the
* overlay frame. Set to return {@code (0,0)} (the center of the overlay) by default.
*
* <p>For example, a value of {@code (+1,-1)} will result in the overlay being positioned from
* the bottom right corner of its frame.
*
* @param x The NDC x-coordinate in the range [-1, 1].
* @param y The NDC y-coordinate in the range [-1, 1].
*/
@CanIgnoreReturnValue
public Builder setOverlayAnchor(
@FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) {
checkArgument(-1 <= x && x <= 1);
checkArgument(-1 <= y && y <= 1);
this.overlayAnchor = Pair.create(x, y);
return this;
}
/**
* Sets the scaling of the overlay.
*
* @param x The desired scaling in the x axis of the overlay.
* @param y The desired scaling in the y axis of the overlay.
*/
@CanIgnoreReturnValue
public Builder setScale(float x, float y) {
this.scale = Pair.create(x, y);
return this;
}
/**
* Sets the rotation of the overlay, counter-clockwise.
*
* <p>The overlay is rotated at the center of its frame.
*
* @param rotationDegree The desired degrees of rotation, counter-clockwise.
*/
@CanIgnoreReturnValue
public Builder setRotationDegrees(float rotationDegree) {
this.rotationDegrees = rotationDegree;
return this;
}
/** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */
public OverlaySettings build() {
return new OverlaySettings(useHdr, alpha, matrix, anchor);
return new OverlaySettings(
useHdr, alpha, videoFrameAnchor, overlayAnchor, scale, rotationDegrees);
}
}
}

178
libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java
View File

@ -35,9 +35,16 @@ import com.google.common.collect.ImmutableList;
private final GlProgram glProgram;
private final ImmutableList<TextureOverlay> overlays;
private final float[] videoFrameAnchorMatrix;
private final float[] videoFrameAnchorMatrixInv;
private final float[] aspectRatioMatrix;
private final float[] overlayMatrix;
private final float[] anchorMatrix;
private final float[] scaleMatrix;
private final float[] scaleMatrixInv;
private final float[] overlayAnchorMatrix;
private final float[] overlayAnchorMatrixInv;
private final float[] rotateMatrix;
private final float[] overlayAspectRatioMatrix;
private final float[] overlayAspectRatioMatrixInv;
private final float[] transformationMatrix;
private int videoWidth;
@ -63,8 +70,15 @@ import com.google.common.collect.ImmutableList;
"OverlayShaderProgram does not support more than 15 overlays in the same instance.");
this.overlays = overlays;
aspectRatioMatrix = GlUtil.create4x4IdentityMatrix();
overlayMatrix = GlUtil.create4x4IdentityMatrix();
anchorMatrix = GlUtil.create4x4IdentityMatrix();
videoFrameAnchorMatrix = GlUtil.create4x4IdentityMatrix();
videoFrameAnchorMatrixInv = GlUtil.create4x4IdentityMatrix();
overlayAnchorMatrix = GlUtil.create4x4IdentityMatrix();
overlayAnchorMatrixInv = GlUtil.create4x4IdentityMatrix();
rotateMatrix = GlUtil.create4x4IdentityMatrix();
scaleMatrix = GlUtil.create4x4IdentityMatrix();
scaleMatrixInv = GlUtil.create4x4IdentityMatrix();
overlayAspectRatioMatrix = GlUtil.create4x4IdentityMatrix();
overlayAspectRatioMatrixInv = GlUtil.create4x4IdentityMatrix();
transformationMatrix = GlUtil.create4x4IdentityMatrix();
try {
glProgram =
@ -105,43 +119,163 @@ import com.google.common.collect.ImmutableList;
texUnitIndex);
GlUtil.setToIdentity(aspectRatioMatrix);
GlUtil.setToIdentity(videoFrameAnchorMatrix);
GlUtil.setToIdentity(videoFrameAnchorMatrixInv);
GlUtil.setToIdentity(overlayAnchorMatrix);
GlUtil.setToIdentity(overlayAnchorMatrixInv);
GlUtil.setToIdentity(scaleMatrix);
GlUtil.setToIdentity(scaleMatrixInv);
GlUtil.setToIdentity(rotateMatrix);
GlUtil.setToIdentity(overlayAspectRatioMatrix);
GlUtil.setToIdentity(overlayAspectRatioMatrixInv);
GlUtil.setToIdentity(transformationMatrix);
// Anchor point of overlay within output frame.
Pair<Float, Float> videoFrameAnchor =
overlay.getOverlaySettings(presentationTimeUs).videoFrameAnchor;
Matrix.translateM(
videoFrameAnchorMatrix,
MATRIX_OFFSET,
videoFrameAnchor.first,
videoFrameAnchor.second,
/* z= */ 0f);
Matrix.invertM(
videoFrameAnchorMatrixInv, MATRIX_OFFSET, videoFrameAnchorMatrix, MATRIX_OFFSET);
Matrix.scaleM(
aspectRatioMatrix,
MATRIX_OFFSET,
videoWidth / (float) overlay.getTextureSize(presentationTimeUs).getWidth(),
videoHeight / (float) overlay.getTextureSize(presentationTimeUs).getHeight(),
/* z= */ 1);
Matrix.invertM(
overlayMatrix,
/* z= */ 1f);
// Scale the image.
Pair<Float, Float> scale = overlay.getOverlaySettings(presentationTimeUs).scale;
Matrix.setIdentityM(scaleMatrix, MATRIX_OFFSET);
Matrix.scaleM(
scaleMatrix,
MATRIX_OFFSET,
overlay.getOverlaySettings(presentationTimeUs).matrix,
MATRIX_OFFSET);
Pair<Float, Float> overlayAnchor = overlay.getOverlaySettings(presentationTimeUs).anchor;
GlUtil.setToIdentity(anchorMatrix);
scaleMatrix,
MATRIX_OFFSET,
scale.first,
scale.second,
/* z= */ 1f);
Matrix.invertM(scaleMatrixInv, MATRIX_OFFSET, scaleMatrix, MATRIX_OFFSET);
// Translate the overlay within its frame.
Pair<Float, Float> overlayAnchor =
overlay.getOverlaySettings(presentationTimeUs).overlayAnchor;
Matrix.setIdentityM(overlayAnchorMatrix, MATRIX_OFFSET);
Matrix.translateM(
anchorMatrix,
/* mOffset= */ 0,
overlayAnchor.first
* overlay.getTextureSize(presentationTimeUs).getWidth()
/ videoWidth,
overlayAnchor.second
* overlay.getTextureSize(presentationTimeUs).getHeight()
/ videoHeight,
/* z= */ 1);
overlayAnchorMatrix,
MATRIX_OFFSET,
overlayAnchor.first,
overlayAnchor.second,
/* z= */ 0f);
Matrix.invertM(overlayAnchorMatrixInv, MATRIX_OFFSET, overlayAnchorMatrix, MATRIX_OFFSET);
// Rotate the image.
Matrix.setIdentityM(rotateMatrix, MATRIX_OFFSET);
Matrix.rotateM(
rotateMatrix,
MATRIX_OFFSET,
rotateMatrix,
MATRIX_OFFSET,
overlay.getOverlaySettings(presentationTimeUs).rotationDegrees,
/* x= */ 0f,
/* y= */ 0f,
/* z= */ 1f);
Matrix.invertM(rotateMatrix, MATRIX_OFFSET, rotateMatrix, MATRIX_OFFSET);
// Rotation matrix needs to account for overlay aspect ratio to prevent stretching.
Matrix.scaleM(
overlayAspectRatioMatrix,
MATRIX_OFFSET,
(float) overlay.getTextureSize(presentationTimeUs).getHeight()
/ (float) overlay.getTextureSize(presentationTimeUs).getWidth(),
/* y= */ 1f,
/* z= */ 1f);
Matrix.invertM(
overlayAspectRatioMatrixInv, MATRIX_OFFSET, overlayAspectRatioMatrix, MATRIX_OFFSET);
// Rotation needs to be agnostic of the scaling matrix and the aspect ratios.
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
overlayMatrix,
transformationMatrix,
MATRIX_OFFSET,
anchorMatrix,
scaleMatrixInv,
MATRIX_OFFSET);
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET,
overlayAspectRatioMatrix,
MATRIX_OFFSET);
// Rotation matrix.
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET,
rotateMatrix,
MATRIX_OFFSET);
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET,
overlayAspectRatioMatrixInv,
MATRIX_OFFSET);
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET,
scaleMatrix,
MATRIX_OFFSET);
// Translate image.
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET,
overlayAnchorMatrixInv,
MATRIX_OFFSET);
// Scale image.
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET,
scaleMatrixInv,
MATRIX_OFFSET);
// Correct for aspect ratio of image in output frame.
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET,
aspectRatioMatrix,
MATRIX_OFFSET);
// Anchor position in output frame.
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET,
videoFrameAnchorMatrixInv,
MATRIX_OFFSET);
glProgram.setFloatsUniform(
Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), transformationMatrix);

BIN
libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_overlayAnchored.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 509 KiB

BIN
libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_rotated90.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 504 KiB