Effect: Add alpha effect.

This effect can scale Alpha values.

This is needed for testing Compositor, and coincidentally was requested
by a partner in the past. This also will generally be more useful in
full multi-asset, where we may want to have one input have different alpha
values than another

Also, update fragment_shader_copy_es2.glsl to not throw away alpha values.
This means ThumbnailStripShaderProgram (the only place this glsl is used) will
consider input alpha and output alpha appropriately

PiperOrigin-RevId: 555915092

libraries/effect/src/androidTest/java/androidx/media3/effect/AlphaScaleShaderProgramPixelTest.java

@@ -0,0 +1,193 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapFromFocusedGlFramebuffer;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.createGlTextureFromBitmap;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
+import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.opengl.EGLContext;
+import android.opengl.EGLDisplay;
+import android.opengl.EGLSurface;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.Size;
+import androidx.media3.test.utils.BitmapPixelTestUtil;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import java.io.IOException;
+import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.RequiresNonNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+/**
+ * Pixel tests for {@link AlphaScale}.
+ *
+ * <p>Expected images are taken from an emulator, so tests on different emulators or physical
+ * devices may fail. To test on other devices, please increase the {@link
+ * BitmapPixelTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output
+ * bitmaps as recommended in {@link DefaultVideoFrameProcessorPixelTest}.
+ */
+@RunWith(AndroidJUnit4.class)
+public final class AlphaScaleShaderProgramPixelTest {
+  @Rule public final TestName testName = new TestName();
+  // TODO: b/262694346 - Use media3test_srgb instead of media3test, throughout our tests. This is
+  //  this test image is intended to be interpreted as sRGB, but media3test is stored in a niche
+  //  color transfer, which can make alpha tests more difficult to debug.
+  private static final String ORIGINAL_PNG_ASSET_PATH =
+      "media/bitmap/input_images/media3test_srgb.png";
+  private static final String DECREASE_ALPHA_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/decrease_alpha.png";
+  private static final String INCREASE_ALPHA_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/increase_alpha.png";
+  private static final String ZERO_ALPHA_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/zero_alpha.png";
+
+  private final Context context = getApplicationContext();
+
+  private @MonotonicNonNull String testId;
+  private @MonotonicNonNull EGLDisplay eglDisplay;
+  private @MonotonicNonNull EGLContext eglContext;
+  private @MonotonicNonNull SingleFrameGlShaderProgram defaultShaderProgram;
+  private @MonotonicNonNull EGLSurface placeholderEglSurface;
+  private int inputTexId;
+  private int inputWidth;
+  private int inputHeight;
+
+  @Before
+  public void createGlObjects() throws IOException, GlUtil.GlException {
+    eglDisplay = GlUtil.getDefaultEglDisplay();
+    eglContext = GlUtil.createEglContext(eglDisplay);
+    placeholderEglSurface = GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay);
+
+    Bitmap inputBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
+    inputWidth = inputBitmap.getWidth();
+    inputHeight = inputBitmap.getHeight();
+    inputTexId = createGlTextureFromBitmap(inputBitmap);
+
+    int outputTexId =
+        GlUtil.createTexture(inputWidth, inputHeight, /* useHighPrecisionColorComponents= */ false);
+    int frameBuffer = GlUtil.createFboForTexture(outputTexId);
+    GlUtil.focusFramebuffer(
+        checkNotNull(eglDisplay),
+        checkNotNull(eglContext),
+        checkNotNull(placeholderEglSurface),
+        frameBuffer,
+        inputWidth,
+        inputHeight);
+    GlUtil.clearFocusedBuffers();
+  }
+
+  @Before
+  @EnsuresNonNull("testId")
+  public void setUpTestId() {
+    testId = testName.getMethodName();
+  }
+
+  @After
+  public void release() throws GlUtil.GlException, VideoFrameProcessingException {
+    if (defaultShaderProgram != null) {
+      defaultShaderProgram.release();
+    }
+    GlUtil.destroyEglContext(eglDisplay, eglContext);
+  }
+
+  @Test
+  @RequiresNonNull("testId")
+  public void noOpAlpha_matchesGoldenFile() throws Exception {
+    defaultShaderProgram = new AlphaScale(1.0f).toGlShaderProgram(context, /* useHdr= */ false);
+    Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
+    Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "input", expectedBitmap, /* path= */ null);
+
+    defaultShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
+
+    // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  @RequiresNonNull("testId")
+  public void zeroAlpha_matchesGoldenFile() throws Exception {
+    defaultShaderProgram = new AlphaScale(0.0f).toGlShaderProgram(context, /* useHdr= */ false);
+    Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
+    Bitmap expectedBitmap = readBitmap(ZERO_ALPHA_PNG_ASSET_PATH);
+
+    defaultShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
+
+    // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  @RequiresNonNull("testId")
+  public void decreaseAlpha_matchesGoldenFile() throws Exception {
+    defaultShaderProgram = new AlphaScale(0.5f).toGlShaderProgram(context, /* useHdr= */ false);
+    Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
+    Bitmap expectedBitmap = readBitmap(DECREASE_ALPHA_PNG_ASSET_PATH);
+
+    defaultShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
+
+    // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  @RequiresNonNull("testId")
+  public void increaseAlpha_matchesGoldenFile() throws Exception {
+    defaultShaderProgram = new AlphaScale(1.5f).toGlShaderProgram(context, /* useHdr= */ false);
+    Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
+    Bitmap expectedBitmap = readBitmap(INCREASE_ALPHA_PNG_ASSET_PATH);
+
+    defaultShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
+
+    // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+}

libraries/effect/src/main/assets/shaders/fragment_shader_alpha_scale_es2.glsl

@@ -0,0 +1,27 @@
+#version 100
+// Copyright 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ES 2 fragment shader that samples from a (non-external) texture with
+// uTexSampler, and multiplies its alpha value by uAlphaScale.
+
+precision mediump float;
+uniform sampler2D uTexSampler;
+uniform float uAlphaScale;
+varying vec2 vTexSamplingCoord;
+
+void main() {
+    vec4 src = texture2D(uTexSampler, vTexSamplingCoord);
+    gl_FragColor = vec4(src.rgb, src.a * uAlphaScale);
+}

libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl

@@ -14,13 +14,12 @@
 // limitations under the License.
 
 // ES 2 fragment shader that samples from a (non-external) texture with
-// uTexSampler.
+// uTexSampler and copies this to the output.
 
 precision mediump float;
 uniform sampler2D uTexSampler;
 varying vec2 vTexSamplingCoord;
 
 void main() {
-    vec3 src = texture2D(uTexSampler, vTexSamplingCoord).xyz;
-    gl_FragColor = vec4(src, 1.0);
+    gl_FragColor = texture2D(uTexSampler, vTexSamplingCoord);
 }

libraries/effect/src/main/java/androidx/media3/effect/AlphaScale.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+
+import android.content.Context;
+import androidx.annotation.FloatRange;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.UnstableApi;
+
+/** Scales the alpha value (i.e. the translucency) of a frame. */
+@UnstableApi
+public final class AlphaScale implements GlEffect {
+  private final float alphaScale;
+
+  /**
+   * Creates a new instance to scale the entire frame's alpha values by {@code alphaScale}, to
+   * modify translucency.
+   *
+   * <p>An {@code alphaScale} value of {@code 1} means no change is applied. A value below {@code 1}
+   * reduces translucency, and a value above {@code 1} increases translucency.
+   */
+  public AlphaScale(@FloatRange(from = 0) float alphaScale) {
+    checkArgument(0 <= alphaScale);
+    this.alphaScale = alphaScale;
+  }
+
+  @Override
+  public SingleFrameGlShaderProgram toGlShaderProgram(Context context, boolean useHdr)
+      throws VideoFrameProcessingException {
+    return new AlphaScaleShaderProgram(context, useHdr, alphaScale);
+  }
+
+  @Override
+  public boolean isNoOp(int inputWidth, int inputHeight) {
+    return alphaScale == 1f;
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/AlphaScaleShaderProgram.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.effect;
+
+import android.content.Context;
+import android.opengl.GLES20;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlProgram;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.Size;
+import java.io.IOException;
+
+/** Scales the alpha value for each pixel in the fragment shader. */
+/* package */ final class AlphaScaleShaderProgram extends SingleFrameGlShaderProgram {
+  private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
+  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_alpha_scale_es2.glsl";
+
+  private final GlProgram glProgram;
+
+  /**
+   * Creates a new instance.
+   *
+   * @param context The {@link Context}.
+   * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
+   *     in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
+   * @param alphaScale The alpha scale factor.
+   * @throws VideoFrameProcessingException If a problem occurs while reading shader files.
+   */
+  public AlphaScaleShaderProgram(Context context, boolean useHdr, float alphaScale)
+      throws VideoFrameProcessingException {
+    super(/* useHighPrecisionColorComponents= */ useHdr);
+
+    try {
+      glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
+    } catch (IOException | GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e);
+    }
+
+    // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
+    glProgram.setBufferAttribute(
+        "aFramePosition",
+        GlUtil.getNormalizedCoordinateBounds(),
+        GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
+
+    float[] identityMatrix = GlUtil.create4x4IdentityMatrix();
+    glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix);
+    glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
+
+    glProgram.setFloatUniform("uAlphaScale", alphaScale);
+  }
+
+  @Override
+  public Size configure(int inputWidth, int inputHeight) {
+    return new Size(inputWidth, inputHeight);
+  }
+
+  @Override
+  public void drawFrame(int inputTexId, long presentationTimeUs)
+      throws VideoFrameProcessingException {
+    try {
+      glProgram.use();
+      glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
+      glProgram.bindAttributesAndUniforms();
+
+      // The four-vertex triangle strip forms a quad.
+      GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
+    } catch (GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e, presentationTimeUs);
+    }
+  }
+}

libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java

@@ -430,7 +430,8 @@ public class BitmapPixelTestUtil {
             bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false);
     // Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down
     // while OpenGL's positive y-axis points up.
-    GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, flipBitmapVertically(bitmap), 0);
+    GLUtils.texImage2D(
+        GLES20.GL_TEXTURE_2D, /* level= */ 0, flipBitmapVertically(bitmap), /* border= */ 0);
     GlUtil.checkGlError();
     return texId;
   }