Compact a RGB Matrix chain into a singular RGB Matrix.

PiperOrigin-RevId: 468013019

libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java

@@ -80,6 +80,10 @@ public final class GlEffectsFrameProcessorPixelTest {
       "media/bitmap/sample_mp4_first_frame/crop_then_aspect_ratio.png";
   public static final String ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH =
       "media/bitmap/sample_mp4_first_frame/rotate_45_scale_to_fit.png";
+  public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/increase_brightness.png";
+  public static final String GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/grayscale_then_increase_red_channel.png";
 
   /** Input video of which we only use the first frame. */
   private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
@@ -327,6 +331,101 @@ public final class GlEffectsFrameProcessorPixelTest {
     assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
   }
 
+  @Test
+  public void processData_increaseBrightness_producesExpectedOutput() throws Exception {
+    String testId = "processData_increaseBrightness";
+    ImmutableList<Effect> increaseBrightness =
+        ImmutableList.of(
+            new RgbAdjustment.Builder().setRedScale(5).build(),
+            new RgbAdjustment.Builder().setGreenScale(5).build(),
+            new RgbAdjustment.Builder().setBlueScale(5).build());
+    setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, increaseBrightness);
+    Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH);
+
+    Bitmap actualBitmap = processFirstFrameAndEnd();
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "actual", actualBitmap);
+    // TODO(b/207848601): switch to using proper tooling for testing against golden data.
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void processData_fullRotationIncreaseBrightnessAndCenterCrop_producesExpectedOutput()
+      throws Exception {
+    String testId = "drawFrame_fullRotationIncreaseBrightnessAndCenterCrop";
+    Crop centerCrop =
+        new Crop(/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f);
+    ImmutableList<Effect> increaseBrightnessFullRotationCenterCrop =
+        ImmutableList.of(
+            new Rotation(/* degrees= */ 90),
+            new RgbAdjustment.Builder().setRedScale(5).build(),
+            new RgbAdjustment.Builder().setGreenScale(5).build(),
+            new Rotation(/* degrees= */ 90),
+            new Rotation(/* degrees= */ 90),
+            new RgbAdjustment.Builder().setBlueScale(5).build(),
+            new Rotation(/* degrees= */ 90),
+            centerCrop);
+    setUpAndPrepareFirstFrame(
+        DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO,
+        ImmutableList.of(
+            new RgbAdjustment.Builder().setRedScale(5).setBlueScale(5).setGreenScale(5).build(),
+            centerCrop));
+    Bitmap centerCropAndBrightnessIncreaseResultBitmap = processFirstFrameAndEnd();
+    setUpAndPrepareFirstFrame(
+        DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, increaseBrightnessFullRotationCenterCrop);
+
+    Bitmap fullRotationBrightnessIncreaseAndCenterCropResultBitmap = processFirstFrameAndEnd();
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "centerCrop", centerCropAndBrightnessIncreaseResultBitmap);
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
+        testId,
+        /* bitmapLabel= */ "full4StepRotationBrightnessIncreaseAndCenterCrop",
+        fullRotationBrightnessIncreaseAndCenterCropResultBitmap);
+    // TODO(b/207848601): switch to using proper tooling for testing against golden data.
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            centerCropAndBrightnessIncreaseResultBitmap,
+            fullRotationBrightnessIncreaseAndCenterCropResultBitmap,
+            testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  // TODO(b/239757183): Consider moving RgbMatrix composition tests to a new file.
+  public void drawFrame_grayscaleAndIncreaseRedChannel_producesGrayscaleAndRedImage()
+      throws Exception {
+    String testId = "drawFrame_grayscale";
+    // Grayscale transformation matrix with the BT.709 standard from
+    // https://en.wikipedia.org/wiki/Grayscale#Converting_colour_to_grayscale
+    // TODO(b/241240659): Use static grayscale filter from RgbFilter once it exists.
+    float[] grayscaleMatrix = {
+      0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0,
+      0, 0, 1
+    };
+    ImmutableList<Effect> grayscaleThenIncreaseRed =
+        ImmutableList.of(
+            (RgbMatrix) presentationTimeUs -> grayscaleMatrix,
+            new RgbAdjustment.Builder().setRedScale(3).build());
+    setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, grayscaleThenIncreaseRed);
+    Bitmap expectedBitmap =
+        BitmapTestUtil.readBitmap(GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH);
+
+    Bitmap actualBitmap = processFirstFrameAndEnd();
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "actual", actualBitmap);
+    // TODO(b/207848601): switch to using proper tooling for testing against golden data.
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
   // TODO(b/227624622): Add a test for HDR input after BitmapTestUtil can read HDR bitmaps, using
   //  GlEffectWrapper to ensure usage of intermediate textures.
 

libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java

@@ -31,6 +31,7 @@ import android.util.Pair;
 import androidx.media3.common.FrameProcessingException;
 import androidx.media3.common.util.GlUtil;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.junit.After;
@@ -153,7 +154,7 @@ public final class RgbAdjustmentPixelTest {
   }
 
   @Test
-  public void drawFrame_redOnlyFilter_setsBlueAndGreenValuesToZero() throws Exception {
+  public void drawFrame_redOnlyFilter_removeBlueAndGreenValues() throws Exception {
     String testId = "drawFrame_redOnlyFilter";
     RgbMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build();
     rgbMatrixProcessor = new RgbMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false);
@@ -243,4 +244,82 @@ public final class RgbAdjustmentPixelTest {
             expectedBitmap, actualBitmap, testId);
     assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
   }
+
+  @Test
+  public void drawFrame_removeRedGreenAndBlueValuesInAChain_producesBlackImage() throws Exception {
+    String testId = "drawFrame_removeRedGreenBlueValuesInAChain";
+    RgbMatrix noRed = new RgbAdjustment.Builder().setRedScale(0).build();
+    RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build();
+    RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build();
+    rgbMatrixProcessor =
+        new RgbMatrixProcessor(
+            context, ImmutableList.of(noRed, noGreen, noBlue), /* useHdr= */ false);
+    Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
+    Bitmap expectedBitmap =
+        BitmapTestUtil.createArgb8888BitmapWithSolidColor(
+            outputSize.first, outputSize.second, Color.BLACK);
+
+    rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+            outputSize.first, outputSize.second);
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_removeBlueAndGreenValuesInAChain_producesOnlyRedImage() throws Exception {
+    String testId = "drawFrame_removeBlueAndGreenValuesInAChain";
+    RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build();
+    RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build();
+    rgbMatrixProcessor =
+        new RgbMatrixProcessor(context, ImmutableList.of(noGreen, noBlue), /* useHdr= */ false);
+    Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
+    Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH);
+
+    rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+            outputSize.first, outputSize.second);
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_increasesAndDecreasesRed_producesNoChange() throws Exception {
+    String testId = "drawFrame_increaseAndDecreaseRed";
+    float redScale = 4;
+    RgbMatrix scaleRedMatrix = new RgbAdjustment.Builder().setRedScale(redScale).build();
+    RgbMatrix scaleRedByInverseMatrix =
+        new RgbAdjustment.Builder().setRedScale(1 / redScale).build();
+    rgbMatrixProcessor =
+        new RgbMatrixProcessor(
+            context,
+            ImmutableList.of(scaleRedMatrix, scaleRedByInverseMatrix),
+            /* useHdr= */ false);
+    Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
+    Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
+
+    rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+            outputSize.first, outputSize.second);
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
+        testId, /* bitmapLabel= */ "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
 }

libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java

@@ -155,6 +155,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
    *     The first is an {@link ExternalTextureProcessor} and the last is a {@link
    *     FinalMatrixTransformationProcessorWrapper}.
    */
+  // TODO(b/239757183): Squash GlMatrixTransformation and RgbMatrix together.
   private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffects(
       Context context,
       List<Effect> effects,
@@ -168,15 +169,25 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
         new ImmutableList.Builder<>();
     ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
         new ImmutableList.Builder<>();
+    ImmutableList.Builder<RgbMatrix> rgbaMatrixTransformationListBuilder =
+        new ImmutableList.Builder<>();
     boolean sampleFromExternalTexture = true;
     for (int i = 0; i < effects.size(); i++) {
       Effect effect = effects.get(i);
       checkArgument(effect instanceof GlEffect, "GlEffectsFrameProcessor only supports GlEffects");
       GlEffect glEffect = (GlEffect) effect;
+      // The following logic may change the order of the RgbMatrix and GlMatrixTransformation
+      // effects. This does not influence the output since RgbMatrix only changes the individual
+      // pixels and does not take any location in account, which the GlMatrixTransformation
+      // may change.
       if (glEffect instanceof GlMatrixTransformation) {
         matrixTransformationListBuilder.add((GlMatrixTransformation) glEffect);
         continue;
       }
+      if (glEffect instanceof RgbMatrix) {
+        rgbaMatrixTransformationListBuilder.add((RgbMatrix) glEffect);
+        continue;
+      }
       ImmutableList<GlMatrixTransformation> matrixTransformations =
           matrixTransformationListBuilder.build();
       if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) {
@@ -190,9 +201,40 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
         matrixTransformationListBuilder = new ImmutableList.Builder<>();
         sampleFromExternalTexture = false;
       }
+      ImmutableList<RgbMatrix> rgbaMatrixTransformations =
+          rgbaMatrixTransformationListBuilder.build();
+      if (!rgbaMatrixTransformations.isEmpty()) {
+        textureProcessorListBuilder.add(
+            new RgbMatrixProcessor(
+                context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo)));
+        rgbaMatrixTransformationListBuilder = new ImmutableList.Builder<>();
+      }
       textureProcessorListBuilder.add(
           glEffect.toGlTextureProcessor(context, ColorInfo.isTransferHdr(colorInfo)));
     }
+
+    ImmutableList<RgbMatrix> rgbaMatrixTransformations =
+        rgbaMatrixTransformationListBuilder.build();
+    if (!rgbaMatrixTransformations.isEmpty()) {
+      // Add a MatrixTransformationProcessor if none yet exists for sampling from an external
+      // texture.
+      if (sampleFromExternalTexture) {
+        // TODO(b/239757183): Remove the unnecessary MatrixTransformationProcessor after it got
+        // merged with RgbMatrixProcessor.
+        textureProcessorListBuilder.add(
+            new MatrixTransformationProcessor(
+                context,
+                ImmutableList.of(),
+                sampleFromExternalTexture,
+                colorInfo,
+                /* outputOpticalColors= */ false));
+        sampleFromExternalTexture = false;
+      }
+      textureProcessorListBuilder.add(
+          new RgbMatrixProcessor(
+              context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo)));
+    }
+
     textureProcessorListBuilder.add(
         new FinalMatrixTransformationProcessorWrapper(
             context,

libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java

@@ -23,23 +23,53 @@ import android.util.Pair;
 import androidx.media3.common.FrameProcessingException;
 import androidx.media3.common.util.GlProgram;
 import androidx.media3.common.util.GlUtil;
+import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 
-/** Applies an {@link RgbMatrix} to each frame. */
+/**
+ * Applies a sequence of {@link RgbMatrix} to each frame.
+ *
+ * <p>After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the
+ * color space. Intermediate reults are not clamped.
+ */
 /* package */ final class RgbMatrixProcessor extends SingleFrameGlTextureProcessor {
   private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
   private static final String FRAGMENT_SHADER_PATH =
       "shaders/fragment_shader_transformation_es2.glsl";
 
   private final GlProgram glProgram;
-  private final RgbMatrix rgbMatrix;
+  private final ImmutableList<RgbMatrix> rgbMatrices;
 
-  // TODO(b/239431666): Support chaining multiple RgbMatrix instances in RgbMatrixProcessor.
   // TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor.
+  /**
+   * Creates a new instance.
+   *
+   * @param context The {@link Context}.
+   * @param rgbMatrix The {@link RgbMatrix} to apply to each frame.
+   * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
+   *     operation fails or is unsupported.
+   */
   public RgbMatrixProcessor(Context context, RgbMatrix rgbMatrix, boolean useHdr)
       throws FrameProcessingException {
+    this(context, ImmutableList.of(rgbMatrix), useHdr);
+  }
+
+  /**
+   * Creates a new instance.
+   *
+   * @param context The {@link Context}.
+   * @param rgbMatrices The {@link RgbMatrix} to apply to each frame.
+   * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
+   *     operation fails or is unsupported.
+   */
+  public RgbMatrixProcessor(Context context, ImmutableList<RgbMatrix> rgbMatrices, boolean useHdr)
+      throws FrameProcessingException {
     super(useHdr);
-    this.rgbMatrix = rgbMatrix;
+    this.rgbMatrices = rgbMatrices;
 
     try {
       glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
@@ -64,9 +94,35 @@ import java.io.IOException;
     return Pair.create(inputWidth, inputHeight);
   }
 
+  private static float[] createCompositeRgbaMatrixArray(
+      ImmutableList<RgbMatrix> rgbMatrices, long presentationTimeUs) {
+    float[] tempResultMatrix = new float[16];
+    float[] compositeRgbaMatrix = new float[16];
+    Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0);
+
+    for (int i = 0; i < rgbMatrices.size(); i++) {
+      Matrix.multiplyMM(
+          /* result= */ tempResultMatrix,
+          /* resultOffset= */ 0,
+          /* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs),
+          /* lhsOffset= */ 0,
+          /* rhs= */ compositeRgbaMatrix,
+          /* rhsOffset= */ 0);
+      System.arraycopy(
+          /* src= */ tempResultMatrix,
+          /* srcPos= */ 0,
+          /* dest= */ compositeRgbaMatrix,
+          /* destPost= */ 0,
+          /* length= */ tempResultMatrix.length);
+    }
+
+    return compositeRgbaMatrix;
+  }
+
   @Override
   public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
-    float[] rgbMatrixArray = rgbMatrix.getMatrix(presentationTimeUs);
+    // TODO(b/239431666): Add caching for compacting Matrices.
+    float[] rgbMatrixArray = createCompositeRgbaMatrixArray(rgbMatrices, presentationTimeUs);
     try {
       glProgram.use();
       glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);