diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java index 5ccb2491fb..0945738b8e 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java @@ -80,6 +80,10 @@ public final class GlEffectsFrameProcessorPixelTest { "media/bitmap/sample_mp4_first_frame/crop_then_aspect_ratio.png"; public static final String ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH = "media/bitmap/sample_mp4_first_frame/rotate_45_scale_to_fit.png"; + public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/increase_brightness.png"; + public static final String GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/grayscale_then_increase_red_channel.png"; /** Input video of which we only use the first frame. */ private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; @@ -327,6 +331,101 @@ public final class GlEffectsFrameProcessorPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void processData_increaseBrightness_producesExpectedOutput() throws Exception { + String testId = "processData_increaseBrightness"; + ImmutableList increaseBrightness = + ImmutableList.of( + new RgbAdjustment.Builder().setRedScale(5).build(), + new RgbAdjustment.Builder().setGreenScale(5).build(), + new RgbAdjustment.Builder().setBlueScale(5).build()); + setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, increaseBrightness); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void processData_fullRotationIncreaseBrightnessAndCenterCrop_producesExpectedOutput() + throws Exception { + String testId = "drawFrame_fullRotationIncreaseBrightnessAndCenterCrop"; + Crop centerCrop = + new Crop(/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f); + ImmutableList increaseBrightnessFullRotationCenterCrop = + ImmutableList.of( + new Rotation(/* degrees= */ 90), + new RgbAdjustment.Builder().setRedScale(5).build(), + new RgbAdjustment.Builder().setGreenScale(5).build(), + new Rotation(/* degrees= */ 90), + new Rotation(/* degrees= */ 90), + new RgbAdjustment.Builder().setBlueScale(5).build(), + new Rotation(/* degrees= */ 90), + centerCrop); + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, + ImmutableList.of( + new RgbAdjustment.Builder().setRedScale(5).setBlueScale(5).setGreenScale(5).build(), + centerCrop)); + Bitmap centerCropAndBrightnessIncreaseResultBitmap = processFirstFrameAndEnd(); + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, increaseBrightnessFullRotationCenterCrop); + + Bitmap fullRotationBrightnessIncreaseAndCenterCropResultBitmap = processFirstFrameAndEnd(); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "centerCrop", centerCropAndBrightnessIncreaseResultBitmap); + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, + /* bitmapLabel= */ "full4StepRotationBrightnessIncreaseAndCenterCrop", + fullRotationBrightnessIncreaseAndCenterCropResultBitmap); + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + centerCropAndBrightnessIncreaseResultBitmap, + fullRotationBrightnessIncreaseAndCenterCropResultBitmap, + testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + // TODO(b/239757183): Consider moving RgbMatrix composition tests to a new file. + public void drawFrame_grayscaleAndIncreaseRedChannel_producesGrayscaleAndRedImage() + throws Exception { + String testId = "drawFrame_grayscale"; + // Grayscale transformation matrix with the BT.709 standard from + // https://en.wikipedia.org/wiki/Grayscale#Converting_colour_to_grayscale + // TODO(b/241240659): Use static grayscale filter from RgbFilter once it exists. + float[] grayscaleMatrix = { + 0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0, + 0, 0, 1 + }; + ImmutableList grayscaleThenIncreaseRed = + ImmutableList.of( + (RgbMatrix) presentationTimeUs -> grayscaleMatrix, + new RgbAdjustment.Builder().setRedScale(3).build()); + setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, grayscaleThenIncreaseRed); + Bitmap expectedBitmap = + BitmapTestUtil.readBitmap(GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + // TODO(b/227624622): Add a test for HDR input after BitmapTestUtil can read HDR bitmaps, using // GlEffectWrapper to ensure usage of intermediate textures. diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java index e4bc54c8e1..de621a7b36 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java @@ -31,6 +31,7 @@ import android.util.Pair; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.GlUtil; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; @@ -153,7 +154,7 @@ public final class RgbAdjustmentPixelTest { } @Test - public void drawFrame_redOnlyFilter_setsBlueAndGreenValuesToZero() throws Exception { + public void drawFrame_redOnlyFilter_removeBlueAndGreenValues() throws Exception { String testId = "drawFrame_redOnlyFilter"; RgbMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build(); rgbMatrixProcessor = new RgbMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false); @@ -243,4 +244,82 @@ public final class RgbAdjustmentPixelTest { expectedBitmap, actualBitmap, testId); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + + @Test + public void drawFrame_removeRedGreenAndBlueValuesInAChain_producesBlackImage() throws Exception { + String testId = "drawFrame_removeRedGreenBlueValuesInAChain"; + RgbMatrix noRed = new RgbAdjustment.Builder().setRedScale(0).build(); + RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build(); + RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build(); + rgbMatrixProcessor = + new RgbMatrixProcessor( + context, ImmutableList.of(noRed, noGreen, noBlue), /* useHdr= */ false); + Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = + BitmapTestUtil.createArgb8888BitmapWithSolidColor( + outputSize.first, outputSize.second, Color.BLACK); + + rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_removeBlueAndGreenValuesInAChain_producesOnlyRedImage() throws Exception { + String testId = "drawFrame_removeBlueAndGreenValuesInAChain"; + RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build(); + RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build(); + rgbMatrixProcessor = + new RgbMatrixProcessor(context, ImmutableList.of(noGreen, noBlue), /* useHdr= */ false); + Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH); + + rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_increasesAndDecreasesRed_producesNoChange() throws Exception { + String testId = "drawFrame_increaseAndDecreaseRed"; + float redScale = 4; + RgbMatrix scaleRedMatrix = new RgbAdjustment.Builder().setRedScale(redScale).build(); + RgbMatrix scaleRedByInverseMatrix = + new RgbAdjustment.Builder().setRedScale(1 / redScale).build(); + rgbMatrixProcessor = + new RgbMatrixProcessor( + context, + ImmutableList.of(scaleRedMatrix, scaleRedByInverseMatrix), + /* useHdr= */ false); + Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + + rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java index a74033d1c8..f0656749cc 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java @@ -155,6 +155,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { * The first is an {@link ExternalTextureProcessor} and the last is a {@link * FinalMatrixTransformationProcessorWrapper}. */ + // TODO(b/239757183): Squash GlMatrixTransformation and RgbMatrix together. private static ImmutableList getGlTextureProcessorsForGlEffects( Context context, List effects, @@ -168,15 +169,25 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { new ImmutableList.Builder<>(); ImmutableList.Builder matrixTransformationListBuilder = new ImmutableList.Builder<>(); + ImmutableList.Builder rgbaMatrixTransformationListBuilder = + new ImmutableList.Builder<>(); boolean sampleFromExternalTexture = true; for (int i = 0; i < effects.size(); i++) { Effect effect = effects.get(i); checkArgument(effect instanceof GlEffect, "GlEffectsFrameProcessor only supports GlEffects"); GlEffect glEffect = (GlEffect) effect; + // The following logic may change the order of the RgbMatrix and GlMatrixTransformation + // effects. This does not influence the output since RgbMatrix only changes the individual + // pixels and does not take any location in account, which the GlMatrixTransformation + // may change. if (glEffect instanceof GlMatrixTransformation) { matrixTransformationListBuilder.add((GlMatrixTransformation) glEffect); continue; } + if (glEffect instanceof RgbMatrix) { + rgbaMatrixTransformationListBuilder.add((RgbMatrix) glEffect); + continue; + } ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) { @@ -190,9 +201,40 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { matrixTransformationListBuilder = new ImmutableList.Builder<>(); sampleFromExternalTexture = false; } + ImmutableList rgbaMatrixTransformations = + rgbaMatrixTransformationListBuilder.build(); + if (!rgbaMatrixTransformations.isEmpty()) { + textureProcessorListBuilder.add( + new RgbMatrixProcessor( + context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo))); + rgbaMatrixTransformationListBuilder = new ImmutableList.Builder<>(); + } textureProcessorListBuilder.add( glEffect.toGlTextureProcessor(context, ColorInfo.isTransferHdr(colorInfo))); } + + ImmutableList rgbaMatrixTransformations = + rgbaMatrixTransformationListBuilder.build(); + if (!rgbaMatrixTransformations.isEmpty()) { + // Add a MatrixTransformationProcessor if none yet exists for sampling from an external + // texture. + if (sampleFromExternalTexture) { + // TODO(b/239757183): Remove the unnecessary MatrixTransformationProcessor after it got + // merged with RgbMatrixProcessor. + textureProcessorListBuilder.add( + new MatrixTransformationProcessor( + context, + ImmutableList.of(), + sampleFromExternalTexture, + colorInfo, + /* outputOpticalColors= */ false)); + sampleFromExternalTexture = false; + } + textureProcessorListBuilder.add( + new RgbMatrixProcessor( + context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo))); + } + textureProcessorListBuilder.add( new FinalMatrixTransformationProcessorWrapper( context, diff --git a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java index 4075ea21fd..1643cee566 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java @@ -23,23 +23,53 @@ import android.util.Pair; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; +import com.google.common.collect.ImmutableList; import java.io.IOException; -/** Applies an {@link RgbMatrix} to each frame. */ +/** + * Applies a sequence of {@link RgbMatrix} to each frame. + * + *

After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the + * color space. Intermediate reults are not clamped. + */ /* package */ final class RgbMatrixProcessor extends SingleFrameGlTextureProcessor { private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_transformation_es2.glsl"; private final GlProgram glProgram; - private final RgbMatrix rgbMatrix; + private final ImmutableList rgbMatrices; - // TODO(b/239431666): Support chaining multiple RgbMatrix instances in RgbMatrixProcessor. // TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor. + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param rgbMatrix The {@link RgbMatrix} to apply to each frame. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL + * operation fails or is unsupported. + */ public RgbMatrixProcessor(Context context, RgbMatrix rgbMatrix, boolean useHdr) throws FrameProcessingException { + this(context, ImmutableList.of(rgbMatrix), useHdr); + } + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param rgbMatrices The {@link RgbMatrix} to apply to each frame. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL + * operation fails or is unsupported. + */ + public RgbMatrixProcessor(Context context, ImmutableList rgbMatrices, boolean useHdr) + throws FrameProcessingException { super(useHdr); - this.rgbMatrix = rgbMatrix; + this.rgbMatrices = rgbMatrices; try { glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); @@ -64,9 +94,35 @@ import java.io.IOException; return Pair.create(inputWidth, inputHeight); } + private static float[] createCompositeRgbaMatrixArray( + ImmutableList rgbMatrices, long presentationTimeUs) { + float[] tempResultMatrix = new float[16]; + float[] compositeRgbaMatrix = new float[16]; + Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0); + + for (int i = 0; i < rgbMatrices.size(); i++) { + Matrix.multiplyMM( + /* result= */ tempResultMatrix, + /* resultOffset= */ 0, + /* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs), + /* lhsOffset= */ 0, + /* rhs= */ compositeRgbaMatrix, + /* rhsOffset= */ 0); + System.arraycopy( + /* src= */ tempResultMatrix, + /* srcPos= */ 0, + /* dest= */ compositeRgbaMatrix, + /* destPost= */ 0, + /* length= */ tempResultMatrix.length); + } + + return compositeRgbaMatrix; + } + @Override public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { - float[] rgbMatrixArray = rgbMatrix.getMatrix(presentationTimeUs); + // TODO(b/239431666): Add caching for compacting Matrices. + float[] rgbMatrixArray = createCompositeRgbaMatrixArray(rgbMatrices, presentationTimeUs); try { glProgram.use(); glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/grayscale_then_increase_red_channel.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/grayscale_then_increase_red_channel.png new file mode 100644 index 0000000000..07dc4f149c Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/grayscale_then_increase_red_channel.png differ