Expected images are taken from an emulator, so tests on different emulators or physical + * devices may fail. To test on other devices, please increase the {@link + * BitmapPixelTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output + * bitmaps as recommended in {@link DefaultVideoFrameProcessorPixelTest}. + */ +@RunWith(AndroidJUnit4.class) +public final class AlphaScaleShaderProgramPixelTest { + @Rule public final TestName testName = new TestName(); + // TODO: b/262694346 - Use media3test_srgb instead of media3test, throughout our tests. This is + // this test image is intended to be interpreted as sRGB, but media3test is stored in a niche + // color transfer, which can make alpha tests more difficult to debug. + private static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/input_images/media3test_srgb.png"; + private static final String DECREASE_ALPHA_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/decrease_alpha.png"; + private static final String INCREASE_ALPHA_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/increase_alpha.png"; + private static final String ZERO_ALPHA_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/zero_alpha.png"; + + private final Context context = getApplicationContext(); + + private @MonotonicNonNull String testId; + private @MonotonicNonNull EGLDisplay eglDisplay; + private @MonotonicNonNull EGLContext eglContext; + private @MonotonicNonNull SingleFrameGlShaderProgram defaultShaderProgram; + private @MonotonicNonNull EGLSurface placeholderEglSurface; + private int inputTexId; + private int inputWidth; + private int inputHeight; + + @Before + public void createGlObjects() throws IOException, GlUtil.GlException { + eglDisplay = GlUtil.getDefaultEglDisplay(); + eglContext = GlUtil.createEglContext(eglDisplay); + placeholderEglSurface = GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay); + + Bitmap inputBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); + inputWidth = inputBitmap.getWidth(); + inputHeight = inputBitmap.getHeight(); + inputTexId = createGlTextureFromBitmap(inputBitmap); + + int outputTexId = + GlUtil.createTexture(inputWidth, inputHeight, /* useHighPrecisionColorComponents= */ false); + int frameBuffer = GlUtil.createFboForTexture(outputTexId); + GlUtil.focusFramebuffer( + checkNotNull(eglDisplay), + checkNotNull(eglContext), + checkNotNull(placeholderEglSurface), + frameBuffer, + inputWidth, + inputHeight); + GlUtil.clearFocusedBuffers(); + } + + @Before + @EnsuresNonNull("testId") + public void setUpTestId() { + testId = testName.getMethodName(); + } + + @After + public void release() throws GlUtil.GlException, VideoFrameProcessingException { + if (defaultShaderProgram != null) { + defaultShaderProgram.release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + } + + @Test + @RequiresNonNull("testId") + public void noOpAlpha_matchesGoldenFile() throws Exception { + defaultShaderProgram = new AlphaScale(1.0f).toGlShaderProgram(context, /* useHdr= */ false); + Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "input", expectedBitmap, /* path= */ null); + + defaultShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + @RequiresNonNull("testId") + public void zeroAlpha_matchesGoldenFile() throws Exception { + defaultShaderProgram = new AlphaScale(0.0f).toGlShaderProgram(context, /* useHdr= */ false); + Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = readBitmap(ZERO_ALPHA_PNG_ASSET_PATH); + + defaultShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + @RequiresNonNull("testId") + public void decreaseAlpha_matchesGoldenFile() throws Exception { + defaultShaderProgram = new AlphaScale(0.5f).toGlShaderProgram(context, /* useHdr= */ false); + Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = readBitmap(DECREASE_ALPHA_PNG_ASSET_PATH); + + defaultShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + @RequiresNonNull("testId") + public void increaseAlpha_matchesGoldenFile() throws Exception { + defaultShaderProgram = new AlphaScale(1.5f).toGlShaderProgram(context, /* useHdr= */ false); + Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = readBitmap(INCREASE_ALPHA_PNG_ASSET_PATH); + + defaultShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } +} diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_alpha_scale_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_alpha_scale_es2.glsl new file mode 100644 index 0000000000..aa15d4c588 --- /dev/null +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_alpha_scale_es2.glsl @@ -0,0 +1,27 @@ +#version 100 +// Copyright 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 fragment shader that samples from a (non-external) texture with +// uTexSampler, and multiplies its alpha value by uAlphaScale. + +precision mediump float; +uniform sampler2D uTexSampler; +uniform float uAlphaScale; +varying vec2 vTexSamplingCoord; + +void main() { + vec4 src = texture2D(uTexSampler, vTexSamplingCoord); + gl_FragColor = vec4(src.rgb, src.a * uAlphaScale); +} diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl index 74ce8153cf..7ddee94d7a 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl @@ -14,13 +14,12 @@ // limitations under the License. // ES 2 fragment shader that samples from a (non-external) texture with -// uTexSampler. +// uTexSampler and copies this to the output. precision mediump float; uniform sampler2D uTexSampler; varying vec2 vTexSamplingCoord; void main() { - vec3 src = texture2D(uTexSampler, vTexSamplingCoord).xyz; - gl_FragColor = vec4(src, 1.0); + gl_FragColor = texture2D(uTexSampler, vTexSamplingCoord); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/AlphaScale.java b/libraries/effect/src/main/java/androidx/media3/effect/AlphaScale.java new file mode 100644 index 0000000000..ad3f1bdb59 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/AlphaScale.java @@ -0,0 +1,52 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkArgument; + +import android.content.Context; +import androidx.annotation.FloatRange; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.UnstableApi; + +/** Scales the alpha value (i.e. the translucency) of a frame. */ +@UnstableApi +public final class AlphaScale implements GlEffect { + private final float alphaScale; + + /** + * Creates a new instance to scale the entire frame's alpha values by {@code alphaScale}, to + * modify translucency. + * + *
An {@code alphaScale} value of {@code 1} means no change is applied. A value below {@code 1} + * reduces translucency, and a value above {@code 1} increases translucency. + */ + public AlphaScale(@FloatRange(from = 0) float alphaScale) { + checkArgument(0 <= alphaScale); + this.alphaScale = alphaScale; + } + + @Override + public SingleFrameGlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return new AlphaScaleShaderProgram(context, useHdr, alphaScale); + } + + @Override + public boolean isNoOp(int inputWidth, int inputHeight) { + return alphaScale == 1f; + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/AlphaScaleShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/AlphaScaleShaderProgram.java new file mode 100644 index 0000000000..b9d1667181 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/AlphaScaleShaderProgram.java @@ -0,0 +1,85 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import android.content.Context; +import android.opengl.GLES20; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Size; +import java.io.IOException; + +/** Scales the alpha value for each pixel in the fragment shader. */ +/* package */ final class AlphaScaleShaderProgram extends SingleFrameGlShaderProgram { + private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_alpha_scale_es2.glsl"; + + private final GlProgram glProgram; + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. + * @param alphaScale The alpha scale factor. + * @throws VideoFrameProcessingException If a problem occurs while reading shader files. + */ + public AlphaScaleShaderProgram(Context context, boolean useHdr, float alphaScale) + throws VideoFrameProcessingException { + super(/* useHighPrecisionColorComponents= */ useHdr); + + try { + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + } catch (IOException | GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + + float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); + glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix); + glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + + glProgram.setFloatUniform("uAlphaScale", alphaScale); + } + + @Override + public Size configure(int inputWidth, int inputHeight) { + return new Size(inputWidth, inputHeight); + } + + @Override + public void drawFrame(int inputTexId, long presentationTimeUs) + throws VideoFrameProcessingException { + try { + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + glProgram.bindAttributesAndUniforms(); + + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e, presentationTimeUs); + } + } +} diff --git a/libraries/test_data/src/test/assets/media/bitmap/input_images/media3test_srgb.png b/libraries/test_data/src/test/assets/media/bitmap/input_images/media3test_srgb.png new file mode 100644 index 0000000000..5914bb0fe2 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/input_images/media3test_srgb.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/decrease_alpha.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/decrease_alpha.png new file mode 100644 index 0000000000..38a5ca5ca1 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/decrease_alpha.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/increase_alpha.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/increase_alpha.png new file mode 100644 index 0000000000..399d2d5cdb Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/increase_alpha.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/partial_alpha.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/partial_alpha.png new file mode 100644 index 0000000000..5b54a8206c Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/partial_alpha.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/zero_alpha.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/zero_alpha.png new file mode 100644 index 0000000000..af807ef8de Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/zero_alpha.png differ diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java index 09369e5300..8be01ff37a 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java @@ -430,7 +430,8 @@ public class BitmapPixelTestUtil { bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false); // Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down // while OpenGL's positive y-axis points up. - GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, flipBitmapVertically(bitmap), 0); + GLUtils.texImage2D( + GLES20.GL_TEXTURE_2D, /* level= */ 0, flipBitmapVertically(bitmap), /* border= */ 0); GlUtil.checkGlError(); return texId; }