Add support for changing translucency of overlays through alpha values

Implements milestone 1.3 of the [overlays implementation plan](https://docs.google.com/document/d/1EcP2GN8k8N74hHZyD0KTqm9oQo5-W1dZMqIVyqVGtlo/edit#bookmark=id.76uzcie1dg9d)

PiperOrigin-RevId: 493290147
This commit is contained in:
tofunmi 2022-12-06 14:40:38 +00:00 committed by Ian Baker
parent e6cb502bc6
commit feb3b0b919
5 changed files with 88 additions and 6 deletions

View File

@ -64,6 +64,8 @@ public class OverlayTextureProcessorPixelTest {
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png";
public static final String OVERLAY_BITMAP_SCALED =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png";
public static final String OVERLAY_BITMAP_TRANSLUCENT =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_translucent.png";
public static final String OVERLAY_TEXT_DEFAULT =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png";
public static final String OVERLAY_TEXT_TRANSLATE =
@ -167,6 +169,59 @@ public class OverlayTextureProcessorPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void drawFrame_translucentBitmapOverlay_blendsBitmapIntoFrame() throws Exception {
String testId = "drawFrame_translucentBitmapOverlay";
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings = new OverlaySettings.Builder().setAlpha(0.5f).build();
BitmapOverlay translucentBitmapOverlay =
BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings);
overlayTextureProcessor =
new OverlayEffect(ImmutableList.of(translucentBitmapOverlay))
.toGlTextureProcessor(context, false);
Pair<Integer, Integer> outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.first, outputSize.second);
Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_TRANSLUCENT);
overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second);
maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void drawFrame_transparentTextOverlay_blendsBitmapIntoFrame() throws Exception {
String testId = "drawFrame_transparentTextOverlay";
SpannableString overlayText = new SpannableString(/* source= */ "Text styling");
OverlaySettings overlaySettings = new OverlaySettings.Builder().setAlpha(0f).build();
overlayText.setSpan(
new ForegroundColorSpan(Color.GRAY),
/* start= */ 0,
/* end= */ 4,
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
TextOverlay staticTextOverlay =
TextOverlay.createStaticTextOverlay(overlayText, overlaySettings);
overlayTextureProcessor =
new OverlayEffect(ImmutableList.of(staticTextOverlay))
.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.first, outputSize.second);
Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second);
maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void drawFrame_textOverlay_blendsTextIntoFrame() throws Exception {
String testId = "drawFrame_textOverlay";

View File

@ -20,6 +20,9 @@ precision mediump float;
uniform sampler2D uVideoTexSampler0;
// Texture containing the overlay bitmap.
uniform sampler2D uOverlayTexSampler1;
// The alpha values for the texture.
uniform float uOverlayAlpha1;
varying vec2 vVideoTexSamplingCoord;
varying vec2 vOverlayTexSamplingCoord1;
@ -27,15 +30,18 @@ varying vec2 vOverlayTexSamplingCoord1;
// (https://open.gl/textures) since it's not implemented until OpenGL ES 3.2.
vec4 getClampToBorderOverlayColor() {
if (vOverlayTexSamplingCoord1.x > 1.0 || vOverlayTexSamplingCoord1.x < 0.0
|| vOverlayTexSamplingCoord1.y > 1.0 || vOverlayTexSamplingCoord1.y < 0.0){
|| vOverlayTexSamplingCoord1.y > 1.0 || vOverlayTexSamplingCoord1.y < 0.0) {
return vec4(0.0, 0.0, 0.0, 0.0);
} else {
return vec4(texture2D(uOverlayTexSampler1, vOverlayTexSamplingCoord1));
vec4 overlayColor = vec4(
texture2D(uOverlayTexSampler1, vOverlayTexSamplingCoord1));
overlayColor.a = uOverlayAlpha1 * overlayColor.a;
return overlayColor;
}
}
float getMixAlpha(float videoAlpha, float overlayAlpha) {
if (videoAlpha == 0.0){
if (videoAlpha == 0.0) {
return 1.0;
} else {
return clamp(overlayAlpha/videoAlpha, 0.0, 1.0);

View File

@ -14,6 +14,9 @@ package androidx.media3.effect;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static androidx.media3.common.util.Assertions.checkArgument;
import androidx.annotation.FloatRange;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
@ -22,16 +25,19 @@ import com.google.errorprone.annotations.CanIgnoreReturnValue;
@UnstableApi
public final class OverlaySettings {
public final boolean useHdr;
public final float alpha;
public final float[] matrix;
private OverlaySettings(boolean useHdr, float[] matrix) {
private OverlaySettings(boolean useHdr, float alpha, float[] matrix) {
this.useHdr = useHdr;
this.alpha = alpha;
this.matrix = matrix;
}
/** A builder for {@link OverlaySettings} instances. */
public static final class Builder {
private boolean useHdr;
private float alpha = 1;
private float[] matrix;
/** Creates a new {@link Builder}. */
@ -63,9 +69,23 @@ public final class OverlaySettings {
return this;
}
/**
* Sets the alpha value of the overlay, altering its transparency.
*
* <p>Alpha values range from 0 (all transparent) to 1 (completely opaque).
*
* <p>Set to always return {@code 1} by default.
*/
@CanIgnoreReturnValue
public Builder setAlpha(@FloatRange(from = 0, to = 1) float alpha) {
checkArgument(0 <= alpha && alpha <= 1, "Alpha needs to be in the interval [0, 1].");
this.alpha = alpha;
return this;
}
/** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */
public OverlaySettings build() {
return new OverlaySettings(useHdr, matrix);
return new OverlaySettings(useHdr, alpha, matrix);
}
}
}

View File

@ -99,13 +99,14 @@ import java.io.IOException;
videoHeight / (float) overlayTextureSize.second,
/* z= */ 1);
glProgram.setFloatsUniform("uAspectRatioMatrix", aspectRatioMatrix);
Matrix.invertM(
overlayMatrix,
MATRIX_OFFSET,
overlay.getOverlaySettings(presentationTimeUs).matrix,
MATRIX_OFFSET);
glProgram.setFloatsUniform("uOverlayMatrix", overlayMatrix);
glProgram.setFloatUniform(
"uOverlayAlpha1", overlay.getOverlaySettings(presentationTimeUs).alpha);
} else {
glProgram.setSamplerTexIdUniform(