Compositor: Use multi-stage rendering and occlude background.

PiperOrigin-RevId: 558110739

libraries/effect/src/main/assets/shaders/fragment_shader_compositor_es2.glsl

@@ -1,32 +0,0 @@
-#version 100
-// Copyright 2023 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Basic ES2 compositor shader that samples from a (non-external) textures
-// with uTexSampler1 and uTexSampler2, copying each with alpha = .5 to the
-// output.
-// TODO: b/262694346 - Allow alpha to be customized for each input.
-// TODO: b/262694346 - Allow for an arbitrary amount of inputs.
-
-precision mediump float;
-uniform sampler2D uTexSampler1;
-uniform sampler2D uTexSampler2;
-varying vec2 vTexSamplingCoord;
-
-void main() {
-  vec4 inputColor1 = texture2D(uTexSampler1, vTexSamplingCoord);
-  vec4 inputColor2 = texture2D(uTexSampler2, vTexSamplingCoord);
-  gl_FragColor = vec4(inputColor1.rgb * 0.5 + inputColor2.rgb * 0.5, 1.0);
-  gl_FragColor.a = 1.0;
-}

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java

@@ -67,7 +67,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
   private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread";
   private static final String TAG = "DefaultVideoCompositor";
   private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
-  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_compositor_es2.glsl";
+  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl";
   private static final int PRIMARY_INPUT_ID = 0;
 
   private final Context context;
@@ -390,6 +390,8 @@ public final class DefaultVideoCompositor implements VideoCompositor {
           "aFramePosition",
           GlUtil.getNormalizedCoordinateBounds(),
           GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
+      glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix());
+      glProgram.setFloatsUniform("uTransformationMatrix", GlUtil.create4x4IdentityMatrix());
     } catch (IOException e) {
       throw new VideoFrameProcessingException(e);
     }
@@ -404,16 +406,33 @@ public final class DefaultVideoCompositor implements VideoCompositor {
 
     GlProgram glProgram = checkNotNull(this.glProgram);
     glProgram.use();
-    glProgram.setSamplerTexIdUniform("uTexSampler1", inputTexture1.texId, /* texUnitIndex= */ 0);
-    glProgram.setSamplerTexIdUniform("uTexSampler2", inputTexture2.texId, /* texUnitIndex= */ 1);
 
-    glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix());
-    glProgram.setFloatsUniform("uTransformationMatrix", GlUtil.create4x4IdentityMatrix());
-    glProgram.setBufferAttribute(
-        "aFramePosition",
-        GlUtil.getNormalizedCoordinateBounds(),
-        GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
+    // Setup for blending.
+    GLES20.glEnable(GLES20.GL_BLEND);
+    // Similar to:
+    // dst.rgb = src.rgb * src.a + dst.rgb * (1 - src.a)
+    // dst.a   = src.a           + dst.a   * (1 - src.a)
+    GLES20.glBlendFuncSeparate(
+        /* srcRGB= */ GLES20.GL_SRC_ALPHA,
+        /* dstRGB= */ GLES20.GL_ONE_MINUS_SRC_ALPHA,
+        /* srcAlpha= */ GLES20.GL_ONE,
+        /* dstAlpha= */ GLES20.GL_ONE_MINUS_SRC_ALPHA);
+    GlUtil.checkGlError();
+
+    // Draw textures from back to front.
+    blendOntoFocusedTexture(inputTexture2.texId);
+    blendOntoFocusedTexture(inputTexture1.texId);
+
+    GLES20.glDisable(GLES20.GL_BLEND);
+
+    GlUtil.checkGlError();
+  }
+
+  private void blendOntoFocusedTexture(int texId) throws GlUtil.GlException {
+    GlProgram glProgram = checkNotNull(this.glProgram);
+    glProgram.setSamplerTexIdUniform("uTexSampler", texId, /* texUnitIndex= */ 0);
     glProgram.bindAttributesAndUniforms();
+
     // The four-vertex triangle strip forms a quad.
     GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
     GlUtil.checkGlError();

libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java

@@ -257,6 +257,11 @@ public class BitmapPixelTestUtil {
         int actualColor = actual.getPixel(x, y);
         int expectedColor = expected.getPixel(x, y);
 
+        if (Color.alpha(actualColor) == 0 && Color.alpha(expectedColor) == 0) {
+          // If both colors are transparent, ignore RGB pixel differences for this pixel.
+          differencesBitmap.setPixel(x, y, Color.TRANSPARENT);
+          continue;
+        }
         int alphaDifference = abs(Color.alpha(actualColor) - Color.alpha(expectedColor));
         int redDifference = abs(Color.red(actualColor) - Color.red(expectedColor));
         int blueDifference = abs(Color.blue(actualColor) - Color.blue(expectedColor));
@@ -303,6 +308,10 @@ public class BitmapPixelTestUtil {
         Color actualColor = actual.getColor(x, y);
         Color expectedColor = expected.getColor(x, y);
 
+        if (actualColor.alpha() == 0 && expectedColor.alpha() == 0) {
+          // If both colors are transparent, ignore RGB pixel differences for this pixel.
+          continue;
+        }
         float alphaDifference = abs(actualColor.alpha() - expectedColor.alpha());
         float redDifference = abs(actualColor.red() - expectedColor.red());
         float blueDifference = abs(actualColor.blue() - expectedColor.blue());

libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java

@@ -44,6 +44,7 @@ import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.Util;
+import androidx.media3.effect.AlphaScale;
 import androidx.media3.effect.DefaultGlObjectsProvider;
 import androidx.media3.effect.DefaultVideoCompositor;
 import androidx.media3.effect.DefaultVideoFrameProcessor;
@@ -98,15 +99,17 @@ public final class DefaultVideoCompositorPixelTest {
   @Parameterized.Parameter public boolean useSharedExecutor;
   @Rule public final TestName testName = new TestName();
 
-  private static final String ORIGINAL_PNG_ASSET_PATH = "media/bitmap/input_images/media3test.png";
+  private static final String ORIGINAL_PNG_ASSET_PATH =
+      "media/bitmap/input_images/media3test_srgb.png";
   private static final String TEST_DIRECTORY = "media/bitmap/CompositorTestTimestamps/";
 
   private @MonotonicNonNull String testId;
   private @MonotonicNonNull VideoCompositorTestRunner compositorTestRunner;
-  private static final ImmutableList<Effect> TWO_INPUT_COMPOSITOR_EFFECTS =
+  private static final ImmutableList<ImmutableList<Effect>> TWO_INPUT_COMPOSITOR_EFFECT_LISTS =
       ImmutableList.of(
-          RgbFilter.createGrayscaleFilter(),
-          new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build());
+          ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)),
+          ImmutableList.of(
+              new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
 
   @Before
   @EnsuresNonNull("testId")
@@ -126,7 +129,7 @@ public final class DefaultVideoCompositorPixelTest {
   public void compositeTwoInputs_withOneFrameFromEach_differentTimestamp_matchesExpectedBitmap()
       throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* durationSec= */ 1, /* offsetToAddSec= */ 0, /* frameRate= */ 1);
@@ -147,12 +150,104 @@ public final class DefaultVideoCompositorPixelTest {
     compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(ImmutableList.of("0s_1s"));
   }
 
+  @Test
+  @RequiresNonNull("testId")
+  public void compositeTwoInputs_withPrimaryTransparent_differentTimestamp_matchesExpectedBitmap()
+      throws Exception {
+    ImmutableList<ImmutableList<Effect>> inputEffects =
+        ImmutableList.of(
+            ImmutableList.of(new AlphaScale(0f)),
+            ImmutableList.of(
+                new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
+    compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
+
+    compositorTestRunner.queueBitmapToInput(
+        /* inputId= */ 0, /* durationSec= */ 1, /* offsetToAddSec= */ 0, /* frameRate= */ 1);
+    compositorTestRunner.queueBitmapToInput(
+        /* inputId= */ 1, /* durationSec= */ 1, /* offsetToAddSec= */ 1, /* frameRate= */ 1);
+    compositorTestRunner.endCompositing();
+
+    saveAndAssertBitmapMatchesExpected(
+        testId,
+        compositorTestRunner.inputBitmapReaders.get(0).getBitmap(),
+        /* actualBitmapLabel= */ "actual_input_transparent",
+        TEST_DIRECTORY + "input_transparent.png");
+    saveAndAssertBitmapMatchesExpected(
+        testId,
+        compositorTestRunner.inputBitmapReaders.get(1).getBitmap(),
+        /* actualBitmapLabel= */ "actual_input_rotate180",
+        TEST_DIRECTORY + "input_rotate180_1s.png");
+    compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
+        ImmutableList.of("0s_transparent_1s"));
+  }
+
+  @Test
+  @RequiresNonNull("testId")
+  public void compositeTwoInputs_withPrimaryOpaque_differentTimestamp_matchesExpectedBitmap()
+      throws Exception {
+    ImmutableList<ImmutableList<Effect>> inputEffects =
+        ImmutableList.of(
+            ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(100f)),
+            ImmutableList.of(
+                new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
+    compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
+
+    compositorTestRunner.queueBitmapToInput(
+        /* inputId= */ 0, /* durationSec= */ 1, /* offsetToAddSec= */ 0, /* frameRate= */ 1);
+    compositorTestRunner.queueBitmapToInput(
+        /* inputId= */ 1, /* durationSec= */ 1, /* offsetToAddSec= */ 1, /* frameRate= */ 1);
+    compositorTestRunner.endCompositing();
+
+    saveAndAssertBitmapMatchesExpected(
+        testId,
+        compositorTestRunner.inputBitmapReaders.get(0).getBitmap(),
+        /* actualBitmapLabel= */ "actual_input_grayscale_opaque",
+        TEST_DIRECTORY + "output_grayscale_opaque_0s.png");
+    saveAndAssertBitmapMatchesExpected(
+        testId,
+        compositorTestRunner.inputBitmapReaders.get(1).getBitmap(),
+        /* actualBitmapLabel= */ "actual_input_rotate180",
+        TEST_DIRECTORY + "input_rotate180_1s.png");
+    compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
+        ImmutableList.of("grayscale_opaque_0s"));
+  }
+
+  @Test
+  @RequiresNonNull("testId")
+  public void compositeTwoInputs_withSecondaryAlphaZero_differentTimestamp_matchesExpectedBitmap()
+      throws Exception {
+    ImmutableList<ImmutableList<Effect>> inputEffects =
+        ImmutableList.of(
+            ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)),
+            ImmutableList.of(new AlphaScale(0f)));
+    compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
+
+    compositorTestRunner.queueBitmapToInput(
+        /* inputId= */ 0, /* durationSec= */ 1, /* offsetToAddSec= */ 0, /* frameRate= */ 1);
+    compositorTestRunner.queueBitmapToInput(
+        /* inputId= */ 1, /* durationSec= */ 1, /* offsetToAddSec= */ 1, /* frameRate= */ 1);
+    compositorTestRunner.endCompositing();
+
+    saveAndAssertBitmapMatchesExpected(
+        testId,
+        compositorTestRunner.inputBitmapReaders.get(0).getBitmap(),
+        /* actualBitmapLabel= */ "actual_input_grayscale",
+        TEST_DIRECTORY + "input_grayscale_0s.png");
+    saveAndAssertBitmapMatchesExpected(
+        testId,
+        compositorTestRunner.inputBitmapReaders.get(1).getBitmap(),
+        /* actualBitmapLabel= */ "actual_input_transparent",
+        TEST_DIRECTORY + "input_transparent.png");
+    compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
+        ImmutableList.of("0s_1s_transparent"));
+  }
+
   @Test
   @RequiresNonNull("testId")
   public void compositeTwoInputs_withFiveFramesFromEach_matchesExpectedTimestamps()
       throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
 
     compositorTestRunner.queueBitmapToAllInputs(/* durationSec= */ 5);
     compositorTestRunner.endCompositing();
@@ -182,7 +277,7 @@ public final class DefaultVideoCompositorPixelTest {
   public void composite_onePrimaryAndFiveSecondaryFrames_matchesExpectedTimestamps()
       throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* durationSec= */ 5, /* offsetToAddSec= */ 0, /* frameRate= */ 0.2f);
@@ -215,7 +310,7 @@ public final class DefaultVideoCompositorPixelTest {
   public void composite_fivePrimaryAndOneSecondaryFrames_matchesExpectedTimestamps()
       throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* durationSec= */ 5, /* offsetToAddSec= */ 0, /* frameRate= */ 1f);
@@ -249,7 +344,7 @@ public final class DefaultVideoCompositorPixelTest {
   public void composite_primaryDoubleSecondaryFrameRate_matchesExpectedTimestamps()
       throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* durationSec= */ 4, /* offsetToAddSec= */ 0, /* frameRate= */ 1f);
@@ -282,7 +377,7 @@ public final class DefaultVideoCompositorPixelTest {
   @RequiresNonNull("testId")
   public void composite_primaryHalfSecondaryFrameRate_matchesExpectedTimestamps() throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* durationSec= */ 4, /* offsetToAddSec= */ 0, /* frameRate= */ 0.5f);
@@ -316,7 +411,7 @@ public final class DefaultVideoCompositorPixelTest {
   public void composite_primaryVariableFrameRateWithOffset_matchesExpectedTimestampsAndBitmaps()
       throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* durationSec= */ 2, /* offsetToAddSec= */ 1, /* frameRate= */ 0.5f);
@@ -353,7 +448,7 @@ public final class DefaultVideoCompositorPixelTest {
   public void composite_secondaryVariableFrameRateWithOffset_matchesExpectedTimestampsAndBitmaps()
       throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* durationSec= */ 5, /* offsetToAddSec= */ 0, /* frameRate= */ 1f);
@@ -390,7 +485,7 @@ public final class DefaultVideoCompositorPixelTest {
   public void compositeTwoInputs_withTenFramesFromEach_matchesExpectedFrameCount()
       throws Exception {
     compositorTestRunner =
-        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECTS);
+        new VideoCompositorTestRunner(testId, useSharedExecutor, TWO_INPUT_COMPOSITOR_EFFECT_LISTS);
     int numberOfFramesToQueue = 10;
 
     compositorTestRunner.queueBitmapToAllInputs(/* durationSec= */ numberOfFramesToQueue);
@@ -428,13 +523,15 @@ public final class DefaultVideoCompositorPixelTest {
      * @param testId The {@link String} identifier for the test, used to name output files.
      * @param useSharedExecutor Whether to use a shared executor for {@link
      *     VideoFrameProcessorTestRunner} and {@link VideoCompositor} instances.
-     * @param inputEffects {@link Effect}s to apply for {@link VideoCompositor} input sources. The
-     *     size of this {@link List} is the amount of inputs. One {@link Effect} is used for each
-     *     input. For each input, the frame timestamp and {@code inputId} are overlaid via {@link
-     *     TextOverlay} prior to any {@code inputEffects} being applied.
+     * @param inputEffectLists {@link Effect}s to apply for {@link VideoCompositor} input sources.
+     *     The size of this outer {@link List} is the amount of inputs. One inner list of {@link
+     *     Effect}s is used for each input. For each input, the frame timestamp and {@code inputId}
+     *     are overlaid via {@link TextOverlay} prior to its effects being applied.
      */
     public VideoCompositorTestRunner(
-        String testId, boolean useSharedExecutor, List<Effect> inputEffects)
+        String testId,
+        boolean useSharedExecutor,
+        ImmutableList<ImmutableList<Effect>> inputEffectLists)
         throws GlUtil.GlException, VideoFrameProcessingException {
       this.testId = testId;
       sharedExecutorService =
@@ -481,10 +578,12 @@ public final class DefaultVideoCompositorPixelTest {
               /* textureOutputCapacity= */ 1);
       inputBitmapReaders = new ArrayList<>();
       inputVideoFrameProcessorTestRunners = new ArrayList<>();
-      assertThat(inputEffects).hasSize(COMPOSITOR_INPUT_SIZE);
-      for (int i = 0; i < inputEffects.size(); i++) {
+      assertThat(inputEffectLists).hasSize(COMPOSITOR_INPUT_SIZE);
+      for (int i = 0; i < inputEffectLists.size(); i++) {
         TextureBitmapReader textureBitmapReader = new TextureBitmapReader();
         inputBitmapReaders.add(textureBitmapReader);
+        ImmutableList.Builder<Effect> effectsToApply = new ImmutableList.Builder<>();
+        effectsToApply.add(createTimestampOverlayEffect(i)).addAll(inputEffectLists.get(i));
         VideoFrameProcessorTestRunner vfpTestRunner =
             createVideoFrameProcessorTestRunnerBuilder(
                     testId,
@@ -492,7 +591,7 @@ public final class DefaultVideoCompositorPixelTest {
                     videoCompositor,
                     sharedExecutorService,
                     glObjectsProvider)
-                .setEffects(createTimestampOverlayEffect(i), inputEffects.get(i))
+                .setEffects(effectsToApply.build())
                 .build();
         inputVideoFrameProcessorTestRunners.add(vfpTestRunner);
       }