diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/RgbaMatrixPixelTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/RgbaMatrixPixelTest.java new file mode 100644 index 0000000000..3d4419fddd --- /dev/null +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/RgbaMatrixPixelTest.java @@ -0,0 +1,234 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Color; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLSurface; +import android.opengl.Matrix; +import android.util.Pair; +import androidx.media3.common.FrameProcessingException; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.util.GlUtil; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Pixel tests for {@link RgbaMatrix}. + * + *

Expected images are taken from an emulator, so tests on different emulators or physical + * devices may fail. To test on other devices, please increase the {@link + * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps + * as recommended in {@link GlEffectsFrameProcessorPixelTest}. + */ +@RunWith(AndroidJUnit4.class) +public final class RgbaMatrixPixelTest { + public static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/original.png"; + public static final String ONLY_RED_CHANNEL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/only_red_channel.png"; + public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/increase_brightness.png"; + public static final String GRAYSCALE_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/grayscale.png"; + public static final int COLOR_MATRIX_RED_INDEX = 0; + public static final int COLOR_MATRIX_GREEN_INDEX = 5; + public static final int COLOR_MATRIX_BLUE_INDEX = 10; + public static final int COLOR_MATRIX_ALPHA_INDEX = 15; + + private final Context context = getApplicationContext(); + + private @MonotonicNonNull EGLDisplay eglDisplay; + private @MonotonicNonNull EGLContext eglContext; + private @MonotonicNonNull SingleFrameGlTextureProcessor rgbaMatrixProcessor; + private @MonotonicNonNull EGLSurface placeholderEglSurface; + private int inputTexId; + private int outputTexId; + private int inputWidth; + private int inputHeight; + + @Before + public void createGlObjects() throws IOException, GlUtil.GlException { + eglDisplay = GlUtil.createEglDisplay(); + eglContext = GlUtil.createEglContext(eglDisplay); + Bitmap inputBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + inputWidth = inputBitmap.getWidth(); + inputHeight = inputBitmap.getHeight(); + placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay); + GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, inputWidth, inputHeight); + inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap); + + outputTexId = + GlUtil.createTexture(inputWidth, inputHeight, /* useHighPrecisionColorComponents= */ false); + int frameBuffer = GlUtil.createFboForTexture(outputTexId); + GlUtil.focusFramebuffer( + checkNotNull(eglDisplay), + checkNotNull(eglContext), + checkNotNull(placeholderEglSurface), + frameBuffer, + inputWidth, + inputHeight); + } + + @After + public void release() throws GlUtil.GlException, FrameProcessingException { + if (rgbaMatrixProcessor != null) { + rgbaMatrixProcessor.release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + } + + private static RgbaMatrixProcessor createRgbaMatrixProcessor(Context context, float[] rgbaMatrix) + throws FrameProcessingException { + return ((RgbaMatrix) presentationTimeUs -> rgbaMatrix) + .toGlTextureProcessor(context, /* useHdr= */ false); + } + + @Test + public void drawFrame_identityMatrix_leavesFrameUnchanged() throws Exception { + String testId = "drawFrame_identityMatrix"; + float[] identityMatrix = new float[16]; + Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0); + rgbaMatrixProcessor = createRgbaMatrixProcessor(context, identityMatrix); + Pair outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + + rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_removeColors_producesBlackFrame() throws Exception { + String testId = "drawFrame_removeColors"; + float[] removeColorFilter = new float[16]; + Matrix.setIdentityM(removeColorFilter, /* smOffset= */ 0); + removeColorFilter[COLOR_MATRIX_RED_INDEX] = 0; + removeColorFilter[COLOR_MATRIX_GREEN_INDEX] = 0; + removeColorFilter[COLOR_MATRIX_BLUE_INDEX] = 0; + rgbaMatrixProcessor = createRgbaMatrixProcessor(context, removeColorFilter); + Pair outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = + BitmapTestUtil.createArgb8888BitmapWithSolidColor( + outputSize.first, outputSize.second, Color.BLACK); + + rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_redOnlyFilter_setsBlueAndGreenValuesToZero() throws Exception { + String testId = "drawFrame_redOnlyFilter"; + float[] redOnlyFilter = new float[16]; + Matrix.setIdentityM(redOnlyFilter, /* smOffset= */ 0); + redOnlyFilter[COLOR_MATRIX_GREEN_INDEX] = 0; + redOnlyFilter[COLOR_MATRIX_BLUE_INDEX] = 0; + rgbaMatrixProcessor = createRgbaMatrixProcessor(context, redOnlyFilter); + Pair outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH); + + rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_increaseBrightness_increasesAllValues() throws Exception { + String testId = "drawFrame_increaseBrightness"; + float[] increaseBrightnessMatrix = new float[16]; + Matrix.setIdentityM(increaseBrightnessMatrix, /* smOffset= */ 0); + Matrix.scaleM(increaseBrightnessMatrix, /* mOffset= */ 0, /* x= */ 5, /* y= */ 5, /* z= */ 5); + rgbaMatrixProcessor = createRgbaMatrixProcessor(context, increaseBrightnessMatrix); + Pair outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH); + + rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_grayscale_producesGrayscaleImage() throws Exception { + String testId = "drawFrame_grayscale"; + // Grayscale transformation matrix with the BT.709 standard from + // https://en.wikipedia.org/wiki/Grayscale#Converting_colour_to_grayscale + float[] grayscaleFilter = { + 0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0, + 0, 0, 1 + }; + rgbaMatrixProcessor = createRgbaMatrixProcessor(context, grayscaleFilter); + Pair outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH); + + rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } +} diff --git a/library/transformer/src/main/assets/shaders/fragment_shader_transformation_es2.glsl b/library/transformer/src/main/assets/shaders/fragment_shader_transformation_es2.glsl new file mode 100644 index 0000000000..4b70580e26 --- /dev/null +++ b/library/transformer/src/main/assets/shaders/fragment_shader_transformation_es2.glsl @@ -0,0 +1,28 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 fragment shader that samples from a (non-external) texture with +// uTexSampler, copying from this texture to the current output while +// applying a 4x4 RGBA color matrix to change the pixel colors. + +precision mediump float; +uniform sampler2D uTexSampler; +uniform mat4 uColorMatrix; +varying vec2 vTexSamplingCoord; + +void main() { + vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); + gl_FragColor = uColorMatrix * inputColor; +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/RgbaMatrix.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/RgbaMatrix.java new file mode 100644 index 0000000000..1c2ceb0efe --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/RgbaMatrix.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import android.content.Context; +import androidx.media3.common.FrameProcessingException; + +/** + * Specifies a 4x4 RGBA color transformation matrix to apply to each frame in the fragment shader. + */ +public interface RgbaMatrix extends GlEffect { + + /** + * Returns the 4x4 RGBA transformation {@linkplain android.opengl.Matrix matrix} to apply to the + * color values of each pixel in the frame with the given timestamp. + */ + float[] getMatrix(long presentationTimeUs); + + @Override + default RgbaMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr) + throws FrameProcessingException { + return new RgbaMatrixProcessor(context, /* rgbaMatrix= */ this, useHdr); + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/RgbaMatrixProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/RgbaMatrixProcessor.java new file mode 100644 index 0000000000..9c9b9e0195 --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/RgbaMatrixProcessor.java @@ -0,0 +1,82 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import android.content.Context; +import android.opengl.GLES20; +import android.opengl.Matrix; +import android.util.Pair; +import androidx.media3.common.FrameProcessingException; +import com.google.android.exoplayer2.util.GlProgram; +import com.google.android.exoplayer2.util.GlUtil; +import java.io.IOException; + +/** Applies an {@link RgbaMatrix} to each frame. */ +/* package */ final class RgbaMatrixProcessor extends SingleFrameGlTextureProcessor { + private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = + "shaders/fragment_shader_transformation_es2.glsl"; + + private final GlProgram glProgram; + private final RgbaMatrix rgbaMatrix; + + // TODO(b/239431666): Support chaining multiple RgbaMatrix instances in RgbaMatrixProcessor. + // TODO(b/239757183): Merge RgbaMatrixProcessor with MatrixTransformationProcessor. + public RgbaMatrixProcessor(Context context, RgbaMatrix rgbaMatrix, boolean useHdr) + throws FrameProcessingException { + super(useHdr); + this.rgbaMatrix = rgbaMatrix; + + try { + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + } catch (IOException | GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + + float[] identityMatrix = new float[16]; + Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0); + glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix); + glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + } + + @Override + public Pair configure(int inputWidth, int inputHeight) { + return Pair.create(inputWidth, inputHeight); + } + + @Override + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { + float[] rgbaMatrixArray = rgbaMatrix.getMatrix(presentationTimeUs); + try { + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + glProgram.setFloatsUniform("uColorMatrix", rgbaMatrixArray); + glProgram.bindAttributesAndUniforms(); + + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e, presentationTimeUs); + } + } +} diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/grayscale.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/grayscale.png new file mode 100644 index 0000000000..11b83d2320 Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/grayscale.png differ diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/increase_brightness.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/increase_brightness.png new file mode 100644 index 0000000000..2d65aaa29f Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/increase_brightness.png differ diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/only_red_channel.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/only_red_channel.png new file mode 100644 index 0000000000..56d38b4e21 Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/only_red_channel.png differ