loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create dynamically created shaders for multiple overlays.

Browse Source 
PiperOrigin-RevId: 496379904
This commit is contained in:
tofunmi 2022-12-19 14:02:00 +00:00 committed by Tianyi Feng
parent aae6941981
commit 3708a4f13e
6 changed files with 215 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java
View File

@ -70,6 +70,10 @@ public class OverlayTextureProcessorPixelTest {
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png";
public static final String OVERLAY_TEXT_TRANSLATE =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_translate.png";
public static final String OVERLAY_MULTIPLE =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_multiple.png";
public static final String OVERLAY_OVERLAP =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_overlap.png";
private final Context context = getApplicationContext();
@ -103,7 +107,7 @@ public class OverlayTextureProcessorPixelTest {
@Test
public void drawFrame_noOverlay_leavesFrameUnchanged() throws Exception {
String testId = "drawFrame_noOverlays";
String testId = "drawFrame_noOverlay";
overlayTextureProcessor =
new OverlayEffect(/* textureOverlays= */ ImmutableList.of())
.toGlTextureProcessor(context, /* useHdr= */ false);
@ -281,6 +285,77 @@ public class OverlayTextureProcessorPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void drawFrame_multipleOverlays_blendsBothIntoFrame() throws Exception {
String testId = "drawFrame_multipleOverlays";
float[] translateMatrix1 = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(translateMatrix1, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1);
SpannableString overlayText = new SpannableString(/* source= */ "Overlay 1");
overlayText.setSpan(
new ForegroundColorSpan(Color.GRAY),
/* start= */ 0,
/* end= */ 4,
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
OverlaySettings overlaySettings1 =
new OverlaySettings.Builder().setMatrix(translateMatrix1).build();
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1);
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setAlpha(0.5f).build();
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2);
overlayTextureProcessor =
new OverlayEffect(ImmutableList.of(textOverlay, bitmapOverlay))
.toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_MULTIPLE);
overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void drawFrame_overlappingOverlays_blendsOnFifoOrder() throws Exception {
String testId = "drawFrame_overlappingOverlays";
SpannableString overlayText = new SpannableString(/* source= */ "Overlapping text");
overlayText.setSpan(
new ForegroundColorSpan(Color.WHITE),
/* start= */ 0,
/* end= */ overlayText.length(),
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
float[] scaleTextMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.scaleM(scaleTextMatrix, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1);
OverlaySettings overlaySettings1 =
new OverlaySettings.Builder().setMatrix(scaleTextMatrix).build();
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1);
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
float[] scaleMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.scaleM(scaleMatrix, /* mOffset= */ 0, /* x= */ 3, /* y= */ 3, /* z= */ 1);
OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setMatrix(scaleMatrix).build();
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2);
overlayTextureProcessor =
new OverlayEffect(ImmutableList.of(bitmapOverlay, textOverlay))
.toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_OVERLAP);
overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
int outputTexId =
GlUtil.createTexture(

58
libraries/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl
View File

@ -1,58 +0,0 @@
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that overlays a bitmap over a video frame.
precision mediump float;
// Texture containing an input video frame.
uniform sampler2D uVideoTexSampler0;
// Texture containing the overlay bitmap.
uniform sampler2D uOverlayTexSampler1;
// The alpha values for the texture.
uniform float uOverlayAlpha1;
varying vec2 vVideoTexSamplingCoord;
varying vec2 vOverlayTexSamplingCoord1;
// Manually implementing the CLAMP_TO_BORDER texture wrapping option
// (https://open.gl/textures) since it's not implemented until OpenGL ES 3.2.
vec4 getClampToBorderOverlayColor() {
if (vOverlayTexSamplingCoord1.x > 1.0 || vOverlayTexSamplingCoord1.x < 0.0
|| vOverlayTexSamplingCoord1.y > 1.0 || vOverlayTexSamplingCoord1.y < 0.0) {
return vec4(0.0, 0.0, 0.0, 0.0);
} else {
vec4 overlayColor = vec4(
texture2D(uOverlayTexSampler1, vOverlayTexSamplingCoord1));
overlayColor.a = uOverlayAlpha1 * overlayColor.a;
return overlayColor;
}
}
float getMixAlpha(float videoAlpha, float overlayAlpha) {
if (videoAlpha == 0.0) {
return 1.0;
} else {
return clamp(overlayAlpha/videoAlpha, 0.0, 1.0);
}
}
void main() {
vec4 videoColor = vec4(texture2D(uVideoTexSampler0, vVideoTexSamplingCoord));
vec4 overlayColor = getClampToBorderOverlayColor();
// Blend the video decoder output and the overlay bitmap.
gl_FragColor = mix(
videoColor, overlayColor, getMixAlpha(videoColor.a, overlayColor.a));
}

37
libraries/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl
View File

@ -1,37 +0,0 @@
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 vertex shader that leaves the frame coordinates unchanged
// and applies matrix transformations to the texture coordinates.
uniform mat4 uAspectRatioMatrix;
uniform mat4 uOverlayMatrix;
attribute vec4 aFramePosition;
varying vec2 vVideoTexSamplingCoord;
varying vec2 vOverlayTexSamplingCoord1;
vec2 getTexSamplingCoord(vec2 ndcPosition) {
return vec2(ndcPosition.x * 0.5 + 0.5, ndcPosition.y * 0.5 + 0.5);
}
void main() {
gl_Position = aFramePosition;
vec4 aOverlayPosition = uAspectRatioMatrix * uOverlayMatrix * aFramePosition;
vOverlayTexSamplingCoord1 = getTexSamplingCoord(aOverlayPosition.xy);
vVideoTexSamplingCoord = getTexSamplingCoord(aFramePosition.xy);
}

186
libraries/effect/src/main/java/androidx/media3/effect/OverlayTextureProcessor.java
View File

@ -24,16 +24,13 @@ import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
/** Applies one or more {@link TextureOverlay}s onto each frame. */
/** Applies zero or more {@link TextureOverlay}s onto each frame. */
/* package */ final class OverlayTextureProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_overlay_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_overlay_es2.glsl";
private static final int MATRIX_OFFSET = 0;
private static final int TRANSPARENT_TEXTURE_WIDTH_HEIGHT = 1;
private final GlProgram glProgram;
private final ImmutableList<TextureOverlay> overlays;
@ -56,16 +53,19 @@ import java.io.IOException;
throws FrameProcessingException {
super(useHdr);
checkArgument(!useHdr, "OverlayTextureProcessor does not support HDR colors yet.");
// TODO: If the limit on the number of overlays ever becomes and issue, investigate expanding it
// in relation to common device limits.
checkArgument(
overlays.size() <= 1,
"OverlayTextureProcessor does not support multiple overlays in the same processor yet.");
overlays.size() <= 8,
"OverlayTextureProcessor does not more than 8 overlays in the same processor yet.");
this.overlays = overlays;
aspectRatioMatrix = GlUtil.create4x4IdentityMatrix();
overlayMatrix = GlUtil.create4x4IdentityMatrix();
try {
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
} catch (GlUtil.GlException | IOException e) {
glProgram =
new GlProgram(createVertexShader(overlays.size()), createFragmentShader(overlays.size()));
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
@ -87,30 +87,32 @@ import java.io.IOException;
try {
glProgram.use();
if (!overlays.isEmpty()) {
TextureOverlay overlay = overlays.get(0);
glProgram.setSamplerTexIdUniform(
"uOverlayTexSampler1", overlay.getTextureId(presentationTimeUs), /* texUnitIndex= */ 1);
Size overlayTextureSize = overlay.getTextureSize(presentationTimeUs);
GlUtil.setToIdentity(aspectRatioMatrix);
Matrix.scaleM(
aspectRatioMatrix,
MATRIX_OFFSET,
videoWidth / (float) overlayTextureSize.getWidth(),
videoHeight / (float) overlayTextureSize.getHeight(),
/* z= */ 1);
glProgram.setFloatsUniform("uAspectRatioMatrix", aspectRatioMatrix);
Matrix.invertM(
overlayMatrix,
MATRIX_OFFSET,
overlay.getOverlaySettings(presentationTimeUs).matrix,
MATRIX_OFFSET);
glProgram.setFloatsUniform("uOverlayMatrix", overlayMatrix);
glProgram.setFloatUniform(
"uOverlayAlpha1", overlay.getOverlaySettings(presentationTimeUs).alpha);
} else {
glProgram.setSamplerTexIdUniform(
"uOverlayTexSampler1", createTransparentTexture(), /* texUnitIndex= */ 1);
for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) {
TextureOverlay overlay = overlays.get(texUnitIndex - 1);
glProgram.setSamplerTexIdUniform(
Util.formatInvariant("uOverlayTexSampler%d", texUnitIndex),
overlay.getTextureId(presentationTimeUs),
texUnitIndex);
GlUtil.setToIdentity(aspectRatioMatrix);
Matrix.scaleM(
aspectRatioMatrix,
MATRIX_OFFSET,
videoWidth / (float) overlay.getTextureSize(presentationTimeUs).getWidth(),
videoHeight / (float) overlay.getTextureSize(presentationTimeUs).getHeight(),
/* z= */ 1);
glProgram.setFloatsUniform(
Util.formatInvariant("uAspectRatioMatrix%d", texUnitIndex), aspectRatioMatrix);
Matrix.invertM(
overlayMatrix,
MATRIX_OFFSET,
overlay.getOverlaySettings(presentationTimeUs).matrix,
MATRIX_OFFSET);
glProgram.setFloatsUniform(
Util.formatInvariant("uOverlayMatrix%d", texUnitIndex), overlayMatrix);
glProgram.setFloatUniform(
Util.formatInvariant("uOverlayAlpha%d", texUnitIndex),
overlay.getOverlaySettings(presentationTimeUs).alpha);
}
}
glProgram.setSamplerTexIdUniform("uVideoTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms();
@ -122,20 +124,6 @@ import java.io.IOException;
}
}
private int createTransparentTexture() throws FrameProcessingException {
try {
int textureId =
GlUtil.createTexture(
TRANSPARENT_TEXTURE_WIDTH_HEIGHT,
TRANSPARENT_TEXTURE_WIDTH_HEIGHT,
/* useHighPrecisionColorComponents= */ false);
GlUtil.checkGlError();
return textureId;
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
@Override
public void release() throws FrameProcessingException {
super.release();
@ -145,4 +133,108 @@ import java.io.IOException;
throw new FrameProcessingException(e);
}
}
private static String createVertexShader(int numOverlays) {
StringBuilder shader =
new StringBuilder()
.append("#version 100\n")
.append("attribute vec4 aFramePosition;\n")
.append("varying vec2 vVideoTexSamplingCoord0;\n");
for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
shader
.append(Util.formatInvariant("uniform mat4 uAspectRatioMatrix%d;\n", texUnitIndex))
.append(Util.formatInvariant("uniform mat4 uOverlayMatrix%d;\n", texUnitIndex))
.append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%d;\n", texUnitIndex));
}
shader
.append("vec2 getTexSamplingCoord(vec2 ndcPosition){\n")
.append(" return vec2(ndcPosition.x * 0.5 + 0.5, ndcPosition.y * 0.5 + 0.5);\n")
.append("}\n")
.append("void main() {\n")
.append(" gl_Position = aFramePosition;\n")
.append(" vVideoTexSamplingCoord0 = getTexSamplingCoord(aFramePosition.xy);\n");
for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
shader
.append(Util.formatInvariant(" vec4 aOverlayPosition%d = \n", texUnitIndex))
.append(
Util.formatInvariant(
" uAspectRatioMatrix%d * uOverlayMatrix%d * aFramePosition;\n",
texUnitIndex, texUnitIndex))
.append(
Util.formatInvariant(
" vOverlayTexSamplingCoord%d = getTexSamplingCoord(aOverlayPosition%d.xy);\n",
texUnitIndex, texUnitIndex));
}
shader.append("}\n");
return shader.toString();
}
private static String createFragmentShader(int numOverlays) {
StringBuilder shader =
new StringBuilder()
.append("#version 100\n")
.append("precision mediump float;\n")
.append("uniform sampler2D uVideoTexSampler0;\n")
.append("varying vec2 vVideoTexSamplingCoord0;\n")
.append("// Manually implementing the CLAMP_TO_BORDER texture wrapping option\n")
.append(
"// (https://open.gl/textures) since it's not implemented until OpenGL ES 3.2.\n")
.append("vec4 getClampToBorderOverlayColor(\n")
.append(" sampler2D texSampler, vec2 texSamplingCoord, float alpha){\n")
.append(" if (texSamplingCoord.x > 1.0 || texSamplingCoord.x < 0.0\n")
.append(" || texSamplingCoord.y > 1.0 || texSamplingCoord.y < 0.0) {\n")
.append(" return vec4(0.0, 0.0, 0.0, 0.0);\n")
.append(" } else {\n")
.append(" vec4 overlayColor = vec4(texture2D(texSampler, texSamplingCoord));\n")
.append(" overlayColor.a = alpha * overlayColor.a;\n")
.append(" return overlayColor;\n")
.append(" }\n")
.append("}\n")
.append("\n")
.append("float getMixAlpha(float videoAlpha, float overlayAlpha) {\n")
.append(" if (videoAlpha == 0.0) {\n")
.append(" return 1.0;\n")
.append(" } else {\n")
.append(" return clamp(overlayAlpha/videoAlpha, 0.0, 1.0);\n")
.append(" }\n")
.append("}\n");
for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
shader
.append(Util.formatInvariant("uniform sampler2D uOverlayTexSampler%d;\n", texUnitIndex))
.append(Util.formatInvariant("uniform float uOverlayAlpha%d;\n", texUnitIndex))
.append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%d;\n", texUnitIndex));
}
shader
.append("void main() {\n")
.append(
" vec4 videoColor = vec4(texture2D(uVideoTexSampler0, vVideoTexSamplingCoord0));\n")
.append(" vec4 fragColor = videoColor;\n");
for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
shader
.append(
Util.formatInvariant(
" vec4 overlayColor%d = getClampToBorderOverlayColor(\n", texUnitIndex))
.append(
Util.formatInvariant(
" uOverlayTexSampler%d, vOverlayTexSamplingCoord%d, uOverlayAlpha%d);\n",
texUnitIndex, texUnitIndex, texUnitIndex))
.append(" fragColor = mix(\n")
.append(
Util.formatInvariant(
" fragColor, overlayColor%d, getMixAlpha(videoColor.a, overlayColor%d.a));\n",
texUnitIndex, texUnitIndex));
}
shader.append(" gl_FragColor = fragColor;\n").append("}\n");
return shader.toString();
}
}

BIN
libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_multiple.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 546 KiB

BIN
libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_overlap.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 546 KiB