diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java index 92de9b1828..8e2ffbc92d 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java @@ -400,17 +400,9 @@ public final class GlEffectsFrameProcessorPixelTest { public void drawFrame_grayscaleAndIncreaseRedChannel_producesGrayscaleAndRedImage() throws Exception { String testId = "drawFrame_grayscale"; - // Grayscale transformation matrix with the BT.709 standard from - // https://en.wikipedia.org/wiki/Grayscale#Converting_colour_to_grayscale - // TODO(b/241240659): Use static grayscale filter from RgbFilter once it exists. - float[] grayscaleMatrix = { - 0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0, - 0, 0, 1 - }; ImmutableList grayscaleThenIncreaseRed = ImmutableList.of( - (RgbMatrix) presentationTimeUs -> grayscaleMatrix, - new RgbAdjustment.Builder().setRedScale(3).build()); + RgbFilter.createGrayscaleFilter(), new RgbAdjustment.Builder().setRedScale(3).build()); setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, grayscaleThenIncreaseRed); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java index de621a7b36..a6f23bc024 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java @@ -57,8 +57,6 @@ public final class RgbAdjustmentPixelTest { "media/bitmap/sample_mp4_first_frame/increase_red_channel.png"; public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH = "media/bitmap/sample_mp4_first_frame/increase_brightness.png"; - public static final String GRAYSCALE_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/grayscale.png"; private final Context context = getApplicationContext(); @@ -102,12 +100,6 @@ public final class RgbAdjustmentPixelTest { GlUtil.destroyEglContext(eglDisplay, eglContext); } - private static RgbMatrixProcessor createRgbMatrixProcessor(Context context, float[] rgbMatrix) - throws FrameProcessingException { - return ((RgbMatrix) presentationTimeUs -> rgbMatrix) - .toGlTextureProcessor(context, /* useHdr= */ false); - } - @Test public void drawFrame_identityMatrix_leavesFrameUnchanged() throws Exception { String testId = "drawFrame_identityMatrix"; @@ -218,33 +210,6 @@ public final class RgbAdjustmentPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } - @Test - // TODO(b/239430283): Move test to RgbFilterPixelTest once it exists. - public void drawFrame_grayscale_producesGrayscaleImage() throws Exception { - String testId = "drawFrame_grayscale"; - // Grayscale transformation matrix with the BT.709 standard from - // https://en.wikipedia.org/wiki/Grayscale#Converting_colour_to_grayscale - float[] grayscaleMatrix = { - 0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0, - 0, 0, 1 - }; - rgbMatrixProcessor = createRgbMatrixProcessor(/* context= */ context, grayscaleMatrix); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); - Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH); - - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); - Bitmap actualBitmap = - BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( - outputSize.first, outputSize.second); - - BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( - testId, /* bitmapLabel= */ "actual", actualBitmap); - float averagePixelAbsoluteDifference = - BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( - expectedBitmap, actualBitmap, testId); - assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); - } - @Test public void drawFrame_removeRedGreenAndBlueValuesInAChain_producesBlackImage() throws Exception { String testId = "drawFrame_removeRedGreenBlueValuesInAChain"; diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java new file mode 100644 index 0000000000..a0ed700a75 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java @@ -0,0 +1,140 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.effect.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Bitmap; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLSurface; +import android.util.Pair; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.util.GlUtil; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Pixel tests for {@link RgbFilter}. + * + *

Expected images are taken from an emulator, so tests on different emulators or physical + * devices may fail. To test on other devices, please increase the {@link + * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps + * as recommended in {@link GlEffectsFrameProcessorPixelTest}. + */ +@RunWith(AndroidJUnit4.class) +public final class RgbFilterPixelTest { + public static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/original.png"; + public static final String GRAYSCALE_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/grayscale.png"; + public static final String INVERT_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/invert.png"; + + private final Context context = getApplicationContext(); + + private @MonotonicNonNull EGLDisplay eglDisplay; + private @MonotonicNonNull EGLContext eglContext; + private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor; + private @MonotonicNonNull EGLSurface placeholderEglSurface; + private int inputTexId; + private int outputTexId; + private int inputWidth; + private int inputHeight; + + @Before + public void createGlObjects() throws IOException, GlUtil.GlException { + eglDisplay = GlUtil.createEglDisplay(); + eglContext = GlUtil.createEglContext(eglDisplay); + Bitmap inputBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + inputWidth = inputBitmap.getWidth(); + inputHeight = inputBitmap.getHeight(); + placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay); + GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, inputWidth, inputHeight); + inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap); + + outputTexId = + GlUtil.createTexture(inputWidth, inputHeight, /* useHighPrecisionColorComponents= */ false); + int frameBuffer = GlUtil.createFboForTexture(outputTexId); + GlUtil.focusFramebuffer( + checkNotNull(eglDisplay), + checkNotNull(eglContext), + checkNotNull(placeholderEglSurface), + frameBuffer, + inputWidth, + inputHeight); + } + + @After + public void release() throws GlUtil.GlException, FrameProcessingException { + if (rgbMatrixProcessor != null) { + rgbMatrixProcessor.release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + } + + @Test + public void drawFrame_grayscale_producesGrayscaleImage() throws Exception { + String testId = "drawFrame_grayscale"; + RgbMatrix grayscaleMatrix = RgbFilter.createGrayscaleFilter(); + rgbMatrixProcessor = new RgbMatrixProcessor(context, grayscaleMatrix, /* useHdr= */ false); + Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH); + + rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_inverted_producesInvertedFrame() throws Exception { + String testId = "drawFrame_inverted"; + RgbMatrix invertedMatrix = RgbFilter.createInvertedFilter(); + rgbMatrixProcessor = new RgbMatrixProcessor(context, invertedMatrix, /* useHdr= */ false); + Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INVERT_PNG_ASSET_PATH); + + rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/RgbAdjustment.java b/libraries/effect/src/main/java/androidx/media3/effect/RgbAdjustment.java index e5e8769f75..4d4604b589 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/RgbAdjustment.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/RgbAdjustment.java @@ -96,7 +96,7 @@ public final class RgbAdjustment implements RgbMatrix { } @Override - public float[] getMatrix(long presentationTimeUs) { + public float[] getMatrix(long presentationTimeUs, boolean useHdr) { return rgbMatrix; } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java b/libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java new file mode 100644 index 0000000000..2e96368182 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java @@ -0,0 +1,99 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkState; + +import android.content.Context; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.util.UnstableApi; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Provides common color filters. */ +@UnstableApi +public class RgbFilter implements RgbMatrix { + private static final int COLOR_FILTER_GRAYSCALE_INDEX = 1; + private static final int COLOR_FILTER_INVERTED_INDEX = 2; + + // Grayscale transformation matrix using the BT.709 luminance coefficients from + // https://en.wikipedia.org/wiki/Grayscale#Converting_colour_to_grayscale + private static final float[] FILTER_MATRIX_GRAYSCALE_SDR = { + 0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0, 0, + 0, 1 + }; + // Grayscale transformation using the BT.2020 primary colors from + // https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.2020-2-201510-I!!PDF-E.pdf + // TODO(b/241240659): Add HDR tests once infrastructure supports it. + private static final float[] FILTER_MATRIX_GRAYSCALE_HDR = { + 0.2627f, 0.2627f, 0.2627f, 0, 0.6780f, 0.6780f, 0.6780f, 0, 0.0593f, 0.0593f, 0.0593f, 0, 0, 0, + 0, 1 + }; + // Inverted filter uses the transformation R' = -R + 1 = 1 - R. + private static final float[] FILTER_MATRIX_INVERTED = { + -1, 0, 0, 0, 0, -1, 0, 0, 0, 0, -1, 0, 1, 1, 1, 1 + }; + + private final int colorFilter; + /** + * Ensures that the usage of HDR is consistent. {@code null} indicates that HDR has not yet been + * set. + */ + private @MonotonicNonNull Boolean useHdr; + + /** Creates a new grayscale {@code RgbFilter} instance. */ + public static RgbFilter createGrayscaleFilter() { + return new RgbFilter(COLOR_FILTER_GRAYSCALE_INDEX); + } + + /** Creates a new inverted {@code RgbFilter} instance. */ + public static RgbFilter createInvertedFilter() { + return new RgbFilter(COLOR_FILTER_INVERTED_INDEX); + } + + private RgbFilter(int colorFilter) { + this.colorFilter = colorFilter; + } + + private void checkForConsistentHdrSetting(boolean useHdr) { + if (this.useHdr == null) { + this.useHdr = useHdr; + } else { + checkState(this.useHdr == useHdr, "Changing HDR setting is not supported."); + } + } + + @Override + public float[] getMatrix(long presentationTimeUs, boolean useHdr) { + checkForConsistentHdrSetting(useHdr); + switch (colorFilter) { + case COLOR_FILTER_GRAYSCALE_INDEX: + return useHdr ? FILTER_MATRIX_GRAYSCALE_HDR : FILTER_MATRIX_GRAYSCALE_SDR; + case COLOR_FILTER_INVERTED_INDEX: + return FILTER_MATRIX_INVERTED; + default: + // Should never happen. + throw new IllegalStateException("Invalid color filter " + colorFilter); + } + } + + @Override + public RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr) + throws FrameProcessingException { + checkForConsistentHdrSetting(useHdr); + return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java index 18a8891045..82c2a96bc0 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java @@ -29,8 +29,14 @@ public interface RgbMatrix extends GlEffect { /** * Returns the 4x4 RGB transformation {@linkplain android.opengl.Matrix matrix} to apply to the * color values of each pixel in the frame with the given timestamp. + * + * @param presentationTimeUs The timestamp of the frame to apply the matrix on. + * @param useHdr If {@code true}, colors will be in linear RGB BT.2020. If {@code false}, colors + * will be in gamma RGB BT.709. Must be consistent with {@code useHdr} in {@link + * #toGlTextureProcessor(Context, boolean)}. + * @return The {@code RgbMatrix} to apply to the frame. */ - float[] getMatrix(long presentationTimeUs); + float[] getMatrix(long presentationTimeUs, boolean useHdr); @Override default RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java index f43d27643c..6cdf76844c 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java @@ -39,6 +39,7 @@ import java.io.IOException; private final GlProgram glProgram; private final ImmutableList rgbMatrices; + private final boolean useHdr; // TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor. /** @@ -70,6 +71,7 @@ import java.io.IOException; throws FrameProcessingException { super(useHdr); this.rgbMatrices = rgbMatrices; + this.useHdr = useHdr; try { glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); @@ -95,7 +97,7 @@ import java.io.IOException; } private static float[] createCompositeRgbaMatrixArray( - ImmutableList rgbMatrices, long presentationTimeUs) { + ImmutableList rgbMatrices, boolean useHdr, long presentationTimeUs) { float[] tempResultMatrix = new float[16]; float[] compositeRgbaMatrix = new float[16]; Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0); @@ -104,7 +106,7 @@ import java.io.IOException; Matrix.multiplyMM( /* result= */ tempResultMatrix, /* resultOffset= */ 0, - /* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs), + /* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs, useHdr), /* lhsOffset= */ 0, /* rhs= */ compositeRgbaMatrix, /* rhsOffset= */ 0); @@ -122,7 +124,8 @@ import java.io.IOException; @Override public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { // TODO(b/239431666): Add caching for compacting Matrices. - float[] rgbMatrixArray = createCompositeRgbaMatrixArray(rgbMatrices, presentationTimeUs); + float[] rgbMatrixArray = + createCompositeRgbaMatrixArray(rgbMatrices, useHdr, presentationTimeUs); try { glProgram.use(); glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/invert.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/invert.png new file mode 100644 index 0000000000..e909fe6084 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/invert.png differ