diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java index 572b1a2fe0..e9eb761d07 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java @@ -70,6 +70,10 @@ public class OverlayTextureProcessorPixelTest { "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png"; public static final String OVERLAY_TEXT_TRANSLATE = "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_translate.png"; + public static final String OVERLAY_MULTIPLE = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_multiple.png"; + public static final String OVERLAY_OVERLAP = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_overlap.png"; private final Context context = getApplicationContext(); @@ -103,7 +107,7 @@ public class OverlayTextureProcessorPixelTest { @Test public void drawFrame_noOverlay_leavesFrameUnchanged() throws Exception { - String testId = "drawFrame_noOverlays"; + String testId = "drawFrame_noOverlay"; overlayTextureProcessor = new OverlayEffect(/* textureOverlays= */ ImmutableList.of()) .toGlTextureProcessor(context, /* useHdr= */ false); @@ -281,6 +285,77 @@ public class OverlayTextureProcessorPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void drawFrame_multipleOverlays_blendsBothIntoFrame() throws Exception { + String testId = "drawFrame_multipleOverlays"; + float[] translateMatrix1 = GlUtil.create4x4IdentityMatrix(); + Matrix.translateM(translateMatrix1, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1); + SpannableString overlayText = new SpannableString(/* source= */ "Overlay 1"); + overlayText.setSpan( + new ForegroundColorSpan(Color.GRAY), + /* start= */ 0, + /* end= */ 4, + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + OverlaySettings overlaySettings1 = + new OverlaySettings.Builder().setMatrix(translateMatrix1).build(); + TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1); + Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); + OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setAlpha(0.5f).build(); + BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2); + overlayTextureProcessor = + new OverlayEffect(ImmutableList.of(textOverlay, bitmapOverlay)) + .toGlTextureProcessor(context, /* useHdr= */ false); + Size outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = readBitmap(OVERLAY_MULTIPLE); + + overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_overlappingOverlays_blendsOnFifoOrder() throws Exception { + String testId = "drawFrame_overlappingOverlays"; + SpannableString overlayText = new SpannableString(/* source= */ "Overlapping text"); + overlayText.setSpan( + new ForegroundColorSpan(Color.WHITE), + /* start= */ 0, + /* end= */ overlayText.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + float[] scaleTextMatrix = GlUtil.create4x4IdentityMatrix(); + Matrix.scaleM(scaleTextMatrix, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1); + OverlaySettings overlaySettings1 = + new OverlaySettings.Builder().setMatrix(scaleTextMatrix).build(); + TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1); + Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); + float[] scaleMatrix = GlUtil.create4x4IdentityMatrix(); + Matrix.scaleM(scaleMatrix, /* mOffset= */ 0, /* x= */ 3, /* y= */ 3, /* z= */ 1); + OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setMatrix(scaleMatrix).build(); + BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2); + + overlayTextureProcessor = + new OverlayEffect(ImmutableList.of(bitmapOverlay, textOverlay)) + .toGlTextureProcessor(context, /* useHdr= */ false); + Size outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = readBitmap(OVERLAY_OVERLAP); + + overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { int outputTexId = GlUtil.createTexture( diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl deleted file mode 100644 index 9723f70e83..0000000000 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl +++ /dev/null @@ -1,58 +0,0 @@ -#version 100 -// Copyright 2022 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// ES 2 fragment shader that overlays a bitmap over a video frame. - -precision mediump float; -// Texture containing an input video frame. -uniform sampler2D uVideoTexSampler0; -// Texture containing the overlay bitmap. -uniform sampler2D uOverlayTexSampler1; -// The alpha values for the texture. -uniform float uOverlayAlpha1; - -varying vec2 vVideoTexSamplingCoord; -varying vec2 vOverlayTexSamplingCoord1; - -// Manually implementing the CLAMP_TO_BORDER texture wrapping option -// (https://open.gl/textures) since it's not implemented until OpenGL ES 3.2. -vec4 getClampToBorderOverlayColor() { - if (vOverlayTexSamplingCoord1.x > 1.0 || vOverlayTexSamplingCoord1.x < 0.0 - || vOverlayTexSamplingCoord1.y > 1.0 || vOverlayTexSamplingCoord1.y < 0.0) { - return vec4(0.0, 0.0, 0.0, 0.0); - } else { - vec4 overlayColor = vec4( - texture2D(uOverlayTexSampler1, vOverlayTexSamplingCoord1)); - overlayColor.a = uOverlayAlpha1 * overlayColor.a; - return overlayColor; - } -} - -float getMixAlpha(float videoAlpha, float overlayAlpha) { - if (videoAlpha == 0.0) { - return 1.0; - } else { - return clamp(overlayAlpha/videoAlpha, 0.0, 1.0); - } -} - -void main() { - vec4 videoColor = vec4(texture2D(uVideoTexSampler0, vVideoTexSamplingCoord)); - vec4 overlayColor = getClampToBorderOverlayColor(); - - // Blend the video decoder output and the overlay bitmap. - gl_FragColor = mix( - videoColor, overlayColor, getMixAlpha(videoColor.a, overlayColor.a)); -} diff --git a/libraries/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl b/libraries/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl deleted file mode 100644 index 2aa40bfe9a..0000000000 --- a/libraries/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl +++ /dev/null @@ -1,37 +0,0 @@ -#version 100 -// Copyright 2022 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// ES 2 vertex shader that leaves the frame coordinates unchanged -// and applies matrix transformations to the texture coordinates. - -uniform mat4 uAspectRatioMatrix; -uniform mat4 uOverlayMatrix; -attribute vec4 aFramePosition; -varying vec2 vVideoTexSamplingCoord; -varying vec2 vOverlayTexSamplingCoord1; - - -vec2 getTexSamplingCoord(vec2 ndcPosition) { - return vec2(ndcPosition.x * 0.5 + 0.5, ndcPosition.y * 0.5 + 0.5); -} - -void main() { - gl_Position = aFramePosition; - vec4 aOverlayPosition = uAspectRatioMatrix * uOverlayMatrix * aFramePosition; - vOverlayTexSamplingCoord1 = getTexSamplingCoord(aOverlayPosition.xy); - vVideoTexSamplingCoord = getTexSamplingCoord(aFramePosition.xy); -} - - diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayTextureProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayTextureProcessor.java index d818aeaed9..70b6faa939 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlayTextureProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlayTextureProcessor.java @@ -24,16 +24,13 @@ import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Size; +import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; -import java.io.IOException; -/** Applies one or more {@link TextureOverlay}s onto each frame. */ +/** Applies zero or more {@link TextureOverlay}s onto each frame. */ /* package */ final class OverlayTextureProcessor extends SingleFrameGlTextureProcessor { - private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_overlay_es2.glsl"; - private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_overlay_es2.glsl"; private static final int MATRIX_OFFSET = 0; - private static final int TRANSPARENT_TEXTURE_WIDTH_HEIGHT = 1; private final GlProgram glProgram; private final ImmutableList overlays; @@ -56,16 +53,19 @@ import java.io.IOException; throws FrameProcessingException { super(useHdr); checkArgument(!useHdr, "OverlayTextureProcessor does not support HDR colors yet."); + // TODO: If the limit on the number of overlays ever becomes and issue, investigate expanding it + // in relation to common device limits. checkArgument( - overlays.size() <= 1, - "OverlayTextureProcessor does not support multiple overlays in the same processor yet."); + overlays.size() <= 8, + "OverlayTextureProcessor does not more than 8 overlays in the same processor yet."); this.overlays = overlays; aspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); overlayMatrix = GlUtil.create4x4IdentityMatrix(); try { - glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); - } catch (GlUtil.GlException | IOException e) { + glProgram = + new GlProgram(createVertexShader(overlays.size()), createFragmentShader(overlays.size())); + } catch (GlUtil.GlException e) { throw new FrameProcessingException(e); } @@ -87,30 +87,32 @@ import java.io.IOException; try { glProgram.use(); if (!overlays.isEmpty()) { - TextureOverlay overlay = overlays.get(0); - glProgram.setSamplerTexIdUniform( - "uOverlayTexSampler1", overlay.getTextureId(presentationTimeUs), /* texUnitIndex= */ 1); - Size overlayTextureSize = overlay.getTextureSize(presentationTimeUs); - GlUtil.setToIdentity(aspectRatioMatrix); - Matrix.scaleM( - aspectRatioMatrix, - MATRIX_OFFSET, - videoWidth / (float) overlayTextureSize.getWidth(), - videoHeight / (float) overlayTextureSize.getHeight(), - /* z= */ 1); - glProgram.setFloatsUniform("uAspectRatioMatrix", aspectRatioMatrix); - Matrix.invertM( - overlayMatrix, - MATRIX_OFFSET, - overlay.getOverlaySettings(presentationTimeUs).matrix, - MATRIX_OFFSET); - glProgram.setFloatsUniform("uOverlayMatrix", overlayMatrix); - glProgram.setFloatUniform( - "uOverlayAlpha1", overlay.getOverlaySettings(presentationTimeUs).alpha); - - } else { - glProgram.setSamplerTexIdUniform( - "uOverlayTexSampler1", createTransparentTexture(), /* texUnitIndex= */ 1); + for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) { + TextureOverlay overlay = overlays.get(texUnitIndex - 1); + glProgram.setSamplerTexIdUniform( + Util.formatInvariant("uOverlayTexSampler%d", texUnitIndex), + overlay.getTextureId(presentationTimeUs), + texUnitIndex); + GlUtil.setToIdentity(aspectRatioMatrix); + Matrix.scaleM( + aspectRatioMatrix, + MATRIX_OFFSET, + videoWidth / (float) overlay.getTextureSize(presentationTimeUs).getWidth(), + videoHeight / (float) overlay.getTextureSize(presentationTimeUs).getHeight(), + /* z= */ 1); + glProgram.setFloatsUniform( + Util.formatInvariant("uAspectRatioMatrix%d", texUnitIndex), aspectRatioMatrix); + Matrix.invertM( + overlayMatrix, + MATRIX_OFFSET, + overlay.getOverlaySettings(presentationTimeUs).matrix, + MATRIX_OFFSET); + glProgram.setFloatsUniform( + Util.formatInvariant("uOverlayMatrix%d", texUnitIndex), overlayMatrix); + glProgram.setFloatUniform( + Util.formatInvariant("uOverlayAlpha%d", texUnitIndex), + overlay.getOverlaySettings(presentationTimeUs).alpha); + } } glProgram.setSamplerTexIdUniform("uVideoTexSampler0", inputTexId, /* texUnitIndex= */ 0); glProgram.bindAttributesAndUniforms(); @@ -122,20 +124,6 @@ import java.io.IOException; } } - private int createTransparentTexture() throws FrameProcessingException { - try { - int textureId = - GlUtil.createTexture( - TRANSPARENT_TEXTURE_WIDTH_HEIGHT, - TRANSPARENT_TEXTURE_WIDTH_HEIGHT, - /* useHighPrecisionColorComponents= */ false); - GlUtil.checkGlError(); - return textureId; - } catch (GlUtil.GlException e) { - throw new FrameProcessingException(e); - } - } - @Override public void release() throws FrameProcessingException { super.release(); @@ -145,4 +133,108 @@ import java.io.IOException; throw new FrameProcessingException(e); } } + + private static String createVertexShader(int numOverlays) { + StringBuilder shader = + new StringBuilder() + .append("#version 100\n") + .append("attribute vec4 aFramePosition;\n") + .append("varying vec2 vVideoTexSamplingCoord0;\n"); + + for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) { + shader + .append(Util.formatInvariant("uniform mat4 uAspectRatioMatrix%d;\n", texUnitIndex)) + .append(Util.formatInvariant("uniform mat4 uOverlayMatrix%d;\n", texUnitIndex)) + .append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%d;\n", texUnitIndex)); + } + + shader + .append("vec2 getTexSamplingCoord(vec2 ndcPosition){\n") + .append(" return vec2(ndcPosition.x * 0.5 + 0.5, ndcPosition.y * 0.5 + 0.5);\n") + .append("}\n") + .append("void main() {\n") + .append(" gl_Position = aFramePosition;\n") + .append(" vVideoTexSamplingCoord0 = getTexSamplingCoord(aFramePosition.xy);\n"); + + for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) { + shader + .append(Util.formatInvariant(" vec4 aOverlayPosition%d = \n", texUnitIndex)) + .append( + Util.formatInvariant( + " uAspectRatioMatrix%d * uOverlayMatrix%d * aFramePosition;\n", + texUnitIndex, texUnitIndex)) + .append( + Util.formatInvariant( + " vOverlayTexSamplingCoord%d = getTexSamplingCoord(aOverlayPosition%d.xy);\n", + texUnitIndex, texUnitIndex)); + } + + shader.append("}\n"); + + return shader.toString(); + } + + private static String createFragmentShader(int numOverlays) { + StringBuilder shader = + new StringBuilder() + .append("#version 100\n") + .append("precision mediump float;\n") + .append("uniform sampler2D uVideoTexSampler0;\n") + .append("varying vec2 vVideoTexSamplingCoord0;\n") + .append("// Manually implementing the CLAMP_TO_BORDER texture wrapping option\n") + .append( + "// (https://open.gl/textures) since it's not implemented until OpenGL ES 3.2.\n") + .append("vec4 getClampToBorderOverlayColor(\n") + .append(" sampler2D texSampler, vec2 texSamplingCoord, float alpha){\n") + .append(" if (texSamplingCoord.x > 1.0 || texSamplingCoord.x < 0.0\n") + .append(" || texSamplingCoord.y > 1.0 || texSamplingCoord.y < 0.0) {\n") + .append(" return vec4(0.0, 0.0, 0.0, 0.0);\n") + .append(" } else {\n") + .append(" vec4 overlayColor = vec4(texture2D(texSampler, texSamplingCoord));\n") + .append(" overlayColor.a = alpha * overlayColor.a;\n") + .append(" return overlayColor;\n") + .append(" }\n") + .append("}\n") + .append("\n") + .append("float getMixAlpha(float videoAlpha, float overlayAlpha) {\n") + .append(" if (videoAlpha == 0.0) {\n") + .append(" return 1.0;\n") + .append(" } else {\n") + .append(" return clamp(overlayAlpha/videoAlpha, 0.0, 1.0);\n") + .append(" }\n") + .append("}\n"); + + for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) { + shader + .append(Util.formatInvariant("uniform sampler2D uOverlayTexSampler%d;\n", texUnitIndex)) + .append(Util.formatInvariant("uniform float uOverlayAlpha%d;\n", texUnitIndex)) + .append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%d;\n", texUnitIndex)); + } + + shader + .append("void main() {\n") + .append( + " vec4 videoColor = vec4(texture2D(uVideoTexSampler0, vVideoTexSamplingCoord0));\n") + .append(" vec4 fragColor = videoColor;\n"); + + for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) { + shader + .append( + Util.formatInvariant( + " vec4 overlayColor%d = getClampToBorderOverlayColor(\n", texUnitIndex)) + .append( + Util.formatInvariant( + " uOverlayTexSampler%d, vOverlayTexSamplingCoord%d, uOverlayAlpha%d);\n", + texUnitIndex, texUnitIndex, texUnitIndex)) + .append(" fragColor = mix(\n") + .append( + Util.formatInvariant( + " fragColor, overlayColor%d, getMixAlpha(videoColor.a, overlayColor%d.a));\n", + texUnitIndex, texUnitIndex)); + } + + shader.append(" gl_FragColor = fragColor;\n").append("}\n"); + + return shader.toString(); + } } diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_multiple.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_multiple.png new file mode 100644 index 0000000000..78f43b1925 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_multiple.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_overlap.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_overlap.png new file mode 100644 index 0000000000..b2446aac98 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_overlap.png differ