From 672405afdff53ee746a4bcd237170de3b9feb1f7 Mon Sep 17 00:00:00 2001 From: leonwind Date: Mon, 5 Sep 2022 21:51:11 +0000 Subject: [PATCH] Merge RgbProcessor and MatrixTransformation. PiperOrigin-RevId: 472325145 --- ...atrixTransformationProcessorPixelTest.java | 13 +- .../media3/effect/RgbAdjustmentPixelTest.java | 86 ++++++---- .../media3/effect/RgbFilterPixelTest.java | 22 +-- .../shaders/fragment_shader_copy_es2.glsl | 24 --- .../shaders/fragment_shader_oetf_es3.glsl | 7 +- .../fragment_shader_transformation_es2.glsl | 4 +- ...t_shader_transformation_external_es2.glsl} | 9 +- ...ader_transformation_external_yuv_es3.glsl} | 6 +- ...lMatrixTransformationProcessorWrapper.java | 7 +- .../effect/GlEffectsFrameProcessor.java | 40 +---- .../media3/effect/GlMatrixTransformation.java | 5 +- .../effect/MatrixTransformationProcessor.java | 113 ++++++++++--- .../androidx/media3/effect/RgbFilter.java | 9 +- .../androidx/media3/effect/RgbMatrix.java | 9 +- .../media3/effect/RgbMatrixProcessor.java | 151 ------------------ 15 files changed, 207 insertions(+), 298 deletions(-) delete mode 100644 libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl rename libraries/effect/src/main/assets/shaders/{fragment_shader_copy_external_es2.glsl => fragment_shader_transformation_external_es2.glsl} (74%) rename libraries/effect/src/main/assets/shaders/{fragment_shader_copy_external_yuv_es3.glsl => fragment_shader_transformation_external_yuv_es3.glsl} (96%) delete mode 100644 libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java index 479e8f2f87..702390ac83 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java @@ -28,7 +28,6 @@ import android.opengl.EGLSurface; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.GlUtil; import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.google.common.collect.ImmutableList; import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; @@ -97,8 +96,7 @@ public final class MatrixTransformationProcessorPixelTest { Matrix identityMatrix = new Matrix(); MatrixTransformation noEditsTransformation = (long presentationTimeUs) -> identityMatrix; matrixTransformationFrameProcessor = - MatrixTransformationProcessor.create( - context, ImmutableList.of(noEditsTransformation), /* useHdr= */ false); + noEditsTransformation.toGlTextureProcessor(context, /* useHdr= */ false); matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); @@ -123,8 +121,7 @@ public final class MatrixTransformationProcessorPixelTest { MatrixTransformation translateRightTransformation = (long presentationTimeUs) -> translateRightMatrix; matrixTransformationFrameProcessor = - MatrixTransformationProcessor.create( - context, ImmutableList.of(translateRightTransformation), /* useHdr= */ false); + translateRightTransformation.toGlTextureProcessor(context, /* useHdr= */ false); matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); @@ -148,8 +145,7 @@ public final class MatrixTransformationProcessorPixelTest { scaleNarrowMatrix.postScale(.5f, 1.2f); MatrixTransformation scaleNarrowTransformation = (long presentationTimeUs) -> scaleNarrowMatrix; matrixTransformationFrameProcessor = - MatrixTransformationProcessor.create( - context, ImmutableList.of(scaleNarrowTransformation), /* useHdr= */ false); + scaleNarrowTransformation.toGlTextureProcessor(context, /* useHdr= */ false); matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH); @@ -173,8 +169,7 @@ public final class MatrixTransformationProcessorPixelTest { rotate90Matrix.postRotate(/* degrees= */ 90); MatrixTransformation rotate90Transformation = (long presentationTimeUs) -> rotate90Matrix; matrixTransformationFrameProcessor = - MatrixTransformationProcessor.create( - context, ImmutableList.of(rotate90Transformation), /* useHdr= */ false); + rotate90Transformation.toGlTextureProcessor(context, /* useHdr= */ false); matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java index a6f23bc024..3f11589190 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java @@ -62,7 +62,7 @@ public final class RgbAdjustmentPixelTest { private @MonotonicNonNull EGLDisplay eglDisplay; private @MonotonicNonNull EGLContext eglContext; - private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor; + private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor; private @MonotonicNonNull EGLSurface placeholderEglSurface; private int inputTexId; private int outputTexId; @@ -94,8 +94,8 @@ public final class RgbAdjustmentPixelTest { @After public void release() throws GlUtil.GlException, FrameProcessingException { - if (rgbMatrixProcessor != null) { - rgbMatrixProcessor.release(); + if (matrixTransformationProcessor != null) { + matrixTransformationProcessor.release(); } GlUtil.destroyEglContext(eglDisplay, eglContext); } @@ -104,11 +104,13 @@ public final class RgbAdjustmentPixelTest { public void drawFrame_identityMatrix_leavesFrameUnchanged() throws Exception { String testId = "drawFrame_identityMatrix"; RgbMatrix identityMatrix = new RgbAdjustment.Builder().build(); - rgbMatrixProcessor = new RgbMatrixProcessor(context, identityMatrix, /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + identityMatrix.toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); @@ -126,13 +128,15 @@ public final class RgbAdjustmentPixelTest { String testId = "drawFrame_removeColors"; RgbMatrix removeColorMatrix = new RgbAdjustment.Builder().setRedScale(0).setGreenScale(0).setBlueScale(0).build(); - rgbMatrixProcessor = new RgbMatrixProcessor(context, removeColorMatrix, /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + removeColorMatrix.toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.createArgb8888BitmapWithSolidColor( outputSize.first, outputSize.second, Color.BLACK); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); @@ -149,11 +153,13 @@ public final class RgbAdjustmentPixelTest { public void drawFrame_redOnlyFilter_removeBlueAndGreenValues() throws Exception { String testId = "drawFrame_redOnlyFilter"; RgbMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build(); - rgbMatrixProcessor = new RgbMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + redOnlyMatrix.toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); @@ -170,11 +176,13 @@ public final class RgbAdjustmentPixelTest { public void drawFrame_increaseRedChannel_producesBrighterAndRedderFrame() throws Exception { String testId = "drawFrame_increaseRedChannel"; RgbMatrix increaseRedMatrix = new RgbAdjustment.Builder().setRedScale(5).build(); - rgbMatrixProcessor = new RgbMatrixProcessor(context, increaseRedMatrix, /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + increaseRedMatrix.toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_RED_CHANNEL_PNG_ASSET_PATH); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); @@ -192,12 +200,13 @@ public final class RgbAdjustmentPixelTest { String testId = "drawFrame_increaseBrightness"; RgbMatrix increaseBrightnessMatrix = new RgbAdjustment.Builder().setRedScale(5).setGreenScale(5).setBlueScale(5).build(); - rgbMatrixProcessor = - new RgbMatrixProcessor(context, increaseBrightnessMatrix, /* useHdr = */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + increaseBrightnessMatrix.toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); @@ -216,15 +225,19 @@ public final class RgbAdjustmentPixelTest { RgbMatrix noRed = new RgbAdjustment.Builder().setRedScale(0).build(); RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build(); RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build(); - rgbMatrixProcessor = - new RgbMatrixProcessor( - context, ImmutableList.of(noRed, noGreen, noBlue), /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + MatrixTransformationProcessor.create( + context, + /* matrixTransformations= */ ImmutableList.of(), + /* rgbMatrices= */ ImmutableList.of(noRed, noGreen, noBlue), + /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.createArgb8888BitmapWithSolidColor( outputSize.first, outputSize.second, Color.BLACK); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); @@ -242,12 +255,17 @@ public final class RgbAdjustmentPixelTest { String testId = "drawFrame_removeBlueAndGreenValuesInAChain"; RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build(); RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build(); - rgbMatrixProcessor = - new RgbMatrixProcessor(context, ImmutableList.of(noGreen, noBlue), /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + MatrixTransformationProcessor.create( + context, + /* matrixTransformations= */ ImmutableList.of(), + /* rgbMatrices= */ ImmutableList.of(noGreen, noBlue), + /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); @@ -267,15 +285,17 @@ public final class RgbAdjustmentPixelTest { RgbMatrix scaleRedMatrix = new RgbAdjustment.Builder().setRedScale(redScale).build(); RgbMatrix scaleRedByInverseMatrix = new RgbAdjustment.Builder().setRedScale(1 / redScale).build(); - rgbMatrixProcessor = - new RgbMatrixProcessor( + matrixTransformationProcessor = + MatrixTransformationProcessor.create( context, - ImmutableList.of(scaleRedMatrix, scaleRedByInverseMatrix), + /* matrixTransformations= */ ImmutableList.of(), + /* rgbMatrices= */ ImmutableList.of(scaleRedMatrix, scaleRedByInverseMatrix), /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java index a0ed700a75..7468de9d40 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java @@ -58,7 +58,7 @@ public final class RgbFilterPixelTest { private @MonotonicNonNull EGLDisplay eglDisplay; private @MonotonicNonNull EGLContext eglContext; - private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor; + private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor; private @MonotonicNonNull EGLSurface placeholderEglSurface; private int inputTexId; private int outputTexId; @@ -90,8 +90,8 @@ public final class RgbFilterPixelTest { @After public void release() throws GlUtil.GlException, FrameProcessingException { - if (rgbMatrixProcessor != null) { - rgbMatrixProcessor.release(); + if (matrixTransformationProcessor != null) { + matrixTransformationProcessor.release(); } GlUtil.destroyEglContext(eglDisplay, eglContext); } @@ -100,11 +100,13 @@ public final class RgbFilterPixelTest { public void drawFrame_grayscale_producesGrayscaleImage() throws Exception { String testId = "drawFrame_grayscale"; RgbMatrix grayscaleMatrix = RgbFilter.createGrayscaleFilter(); - rgbMatrixProcessor = new RgbMatrixProcessor(context, grayscaleMatrix, /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + grayscaleMatrix.toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); @@ -121,11 +123,13 @@ public final class RgbFilterPixelTest { public void drawFrame_inverted_producesInvertedFrame() throws Exception { String testId = "drawFrame_inverted"; RgbMatrix invertedMatrix = RgbFilter.createInvertedFilter(); - rgbMatrixProcessor = new RgbMatrixProcessor(context, invertedMatrix, /* useHdr= */ false); - Pair outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight); + matrixTransformationProcessor = + invertedMatrix.toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = + matrixTransformationProcessor.configure(inputWidth, inputHeight); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INVERT_PNG_ASSET_PATH); - rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.first, outputSize.second); diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl deleted file mode 100644 index 83dba1af26..0000000000 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl +++ /dev/null @@ -1,24 +0,0 @@ -#version 100 -// Copyright 2022 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// ES 2 fragment shader that samples from a (non-external) texture with uTexSampler, -// copying from this texture to the current output. - -precision mediump float; -uniform sampler2D uTexSampler; -varying vec2 vTexSamplingCoord; -void main() { - gl_FragColor = texture2D(uTexSampler, vTexSamplingCoord); -} diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl index f908bb6e52..42e14e0eb8 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl @@ -15,7 +15,8 @@ // ES 3 fragment shader that: // 1. samples optical linear BT.2020 RGB from a (non-external) texture with -// uTexSampler, +// uTexSampler, and applies a 4x4 RGB color matrix to change the pixel +// colors, // 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB, // and // 3. copies this converted texture color to the current output. @@ -28,6 +29,7 @@ out vec4 outColor; // Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed. uniform int uOetfColorTransfer; uniform mat3 uColorTransform; +uniform mat4 uRgbMatrix; // TODO(b/227624622): Consider using mediump to save precision, if it won't lead // to noticeable quantization. @@ -83,5 +85,6 @@ highp vec3 getElectricalColor(highp vec3 linearColor) { void main() { vec4 inputColor = texture(uTexSampler, vTexSamplingCoord); - outColor = vec4(getElectricalColor(inputColor.rgb), inputColor.a); + vec4 transformedColors = uRgbMatrix * vec4(inputColor.rgb, 1); + outColor = vec4(getElectricalColor(transformedColors.rgb), inputColor.a); } diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl index ec9e645f84..9ebda59526 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl @@ -19,11 +19,11 @@ precision mediump float; uniform sampler2D uTexSampler; -uniform mat4 uColorMatrix; +uniform mat4 uRgbMatrix; varying vec2 vTexSamplingCoord; void main() { vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); - gl_FragColor = uColorMatrix * vec4(inputColor.rgb, 1); + gl_FragColor = uRgbMatrix * vec4(inputColor.rgb, 1); gl_FragColor.a = inputColor.a; } diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_es2.glsl similarity index 74% rename from libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl rename to libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_es2.glsl index ed021e44e3..4948d15c40 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_es2.glsl @@ -14,12 +14,17 @@ // limitations under the License. // ES 2 fragment shader that samples from an external texture with uTexSampler, -// copying from this texture to the current output. +// copying from this texture to the current output while applying a 4x4 RGB +// color matrix to change the pixel colors. #extension GL_OES_EGL_image_external : require precision mediump float; uniform samplerExternalOES uTexSampler; +uniform mat4 uRgbMatrix; varying vec2 vTexSamplingCoord; + void main() { - gl_FragColor = texture2D(uTexSampler, vTexSamplingCoord); + vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); + gl_FragColor = uRgbMatrix * vec4(inputColor.rgb, 1); + gl_FragColor.a = inputColor.a; } diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl similarity index 96% rename from libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl rename to libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl index 7ed9cafc4a..27cbb571bd 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl @@ -23,13 +23,15 @@ // 3. If uEotfColorTransfer is COLOR_TRANSFER_NO_VALUE, outputs electrical // (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for // intermediate shaders by applying the HLG or PQ EOTF. -// 4. Copies this converted texture color to the current output, with alpha = 1. +// 4. Copies this converted texture color to the current output, with alpha = 1, +// while applying a 4x4 RGB color matrix to change the pixel colors. #extension GL_OES_EGL_image_external : require #extension GL_EXT_YUV_target : require precision mediump float; uniform __samplerExternal2DY2YEXT uTexSampler; uniform mat3 uYuvToRgbColorTransform; +uniform mat4 uRgbMatrix; // C.java#ColorTransfer value. uniform int uEotfColorTransfer; in vec2 vTexSamplingCoord; @@ -101,5 +103,5 @@ vec3 yuvToRgb(vec3 yuv) { void main() { vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz; vec3 rgb = yuvToRgb(srcYuv); - outColor = vec4(getOpticalColor(rgb), 1.0); + outColor = uRgbMatrix * vec4(getOpticalColor(rgb), 1.0); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java index cbfc6d8ba0..e8c590df96 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java @@ -66,6 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Context context; private final ImmutableList matrixTransformations; + private final ImmutableList rgbMatrices; private final EGLDisplay eglDisplay; private final EGLContext eglContext; private final DebugViewProvider debugViewProvider; @@ -100,6 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EGLDisplay eglDisplay, EGLContext eglContext, ImmutableList matrixTransformations, + ImmutableList rgbMatrices, FrameProcessor.Listener frameProcessorListener, DebugViewProvider debugViewProvider, boolean sampleFromExternalTexture, @@ -107,6 +109,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; boolean releaseFramesAutomatically) { this.context = context; this.matrixTransformations = matrixTransformations; + this.rgbMatrices = rgbMatrices; this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.debugViewProvider = debugViewProvider; @@ -374,11 +377,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; if (sampleFromExternalTexture) { matrixTransformationProcessor = MatrixTransformationProcessor.createWithExternalSamplerApplyingEotfThenOetf( - context, expandedMatrixTransformations, colorInfo); + context, expandedMatrixTransformations, rgbMatrices, colorInfo); } else { matrixTransformationProcessor = MatrixTransformationProcessor.createApplyingOetf( - context, expandedMatrixTransformations, colorInfo); + context, expandedMatrixTransformations, rgbMatrices, colorInfo); } matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java index 300fd7f066..5f32693040 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java @@ -184,8 +184,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { new ImmutableList.Builder<>(); ImmutableList.Builder matrixTransformationListBuilder = new ImmutableList.Builder<>(); - ImmutableList.Builder rgbaMatrixTransformationListBuilder = - new ImmutableList.Builder<>(); + ImmutableList.Builder rgbMatrixListBuilder = new ImmutableList.Builder<>(); boolean sampleFromExternalTexture = true; for (int i = 0; i < effects.size(); i++) { Effect effect = effects.get(i); @@ -200,62 +199,39 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { continue; } if (glEffect instanceof RgbMatrix) { - rgbaMatrixTransformationListBuilder.add((RgbMatrix) glEffect); + rgbMatrixListBuilder.add((RgbMatrix) glEffect); continue; } ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); - if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) { + ImmutableList rgbMatrices = rgbMatrixListBuilder.build(); + if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromExternalTexture) { MatrixTransformationProcessor matrixTransformationProcessor; if (sampleFromExternalTexture) { matrixTransformationProcessor = MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf( - context, matrixTransformations, colorInfo); + context, matrixTransformations, rgbMatrices, colorInfo); } else { matrixTransformationProcessor = MatrixTransformationProcessor.create( - context, matrixTransformations, ColorInfo.isTransferHdr(colorInfo)); + context, matrixTransformations, rgbMatrices, ColorInfo.isTransferHdr(colorInfo)); } textureProcessorListBuilder.add(matrixTransformationProcessor); matrixTransformationListBuilder = new ImmutableList.Builder<>(); + rgbMatrixListBuilder = new ImmutableList.Builder<>(); sampleFromExternalTexture = false; } - ImmutableList rgbaMatrixTransformations = - rgbaMatrixTransformationListBuilder.build(); - if (!rgbaMatrixTransformations.isEmpty()) { - textureProcessorListBuilder.add( - new RgbMatrixProcessor( - context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo))); - rgbaMatrixTransformationListBuilder = new ImmutableList.Builder<>(); - } textureProcessorListBuilder.add( glEffect.toGlTextureProcessor(context, ColorInfo.isTransferHdr(colorInfo))); } - ImmutableList rgbaMatrixTransformations = - rgbaMatrixTransformationListBuilder.build(); - if (!rgbaMatrixTransformations.isEmpty()) { - // Add a MatrixTransformationProcessor if none yet exists for sampling from an external - // texture. - if (sampleFromExternalTexture) { - // TODO(b/239757183): Remove the unnecessary MatrixTransformationProcessor after it got - // merged with RgbMatrixProcessor. - textureProcessorListBuilder.add( - MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf( - context, /* matrixTransformations= */ ImmutableList.of(), colorInfo)); - sampleFromExternalTexture = false; - } - textureProcessorListBuilder.add( - new RgbMatrixProcessor( - context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo))); - } - textureProcessorListBuilder.add( new FinalMatrixTransformationProcessorWrapper( context, eglDisplay, eglContext, matrixTransformationListBuilder.build(), + rgbMatrixListBuilder.build(), listener, debugViewProvider, sampleFromExternalTexture, diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java b/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java index 9f04b230d5..a4e8803bd7 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java @@ -56,6 +56,9 @@ public interface GlMatrixTransformation extends GlEffect { default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException { return MatrixTransformationProcessor.create( - context, /* matrixTransformations= */ ImmutableList.of(this), useHdr); + context, + /* matrixTransformations= */ ImmutableList.of(this), + /* rgbMatrices= */ ImmutableList.of(), + useHdr); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java index c4fcb54a71..d6ef72bfab 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java @@ -34,17 +34,22 @@ import java.io.IOException; import java.util.Arrays; /** - * Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into - * an output frame based on their locations after applying the sequence of transformation matrices. + * Applies a sequence of {@link MatrixTransformation MatrixTransformations} in the vertex shader and + * a sequence of {@link RgbMatrix RgbMatrices} in the fragment shader. Copies input pixels into an + * output frame based on their locations after applying the sequence of transformation matrices. * - *

Operations are done on normalized device coordinates (-1 to 1 on x, y, and z axes). - * Transformed vertices that are moved outside of this range after any of the transformation - * matrices are clipped to the NDC range. + *

{@link MatrixTransformation} operations are done on normalized device coordinates (-1 to 1 on + * x, y, and z axes). Transformed vertices that are moved outside of this range after any of the + * transformation matrices are clipped to the NDC range. + * + *

After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the + * color space (e.g. BT.709 for SDR). Intermediate results are not clamped. * *

The background color of the output frame will be (r=0, g=0, b=0, a=0). * *

Can copy frames from an external texture and apply color transformations for HDR if needed. */ +// TODO(b/239757183): Rename Matrix to a more generic name. @UnstableApi @SuppressWarnings("FunctionalInterfaceClash") // b/228192298 /* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor @@ -54,13 +59,14 @@ import java.util.Arrays; "shaders/vertex_shader_transformation_es2.glsl"; private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH = "shaders/vertex_shader_transformation_es3.glsl"; - private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl"; + private static final String FRAGMENT_SHADER_TRANSFORMATION_PATH = + "shaders/fragment_shader_transformation_es2.glsl"; private static final String FRAGMENT_SHADER_OETF_ES3_PATH = "shaders/fragment_shader_oetf_es3.glsl"; - private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = - "shaders/fragment_shader_copy_external_es2.glsl"; - private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = - "shaders/fragment_shader_copy_external_yuv_es3.glsl"; + private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH = + "shaders/fragment_shader_transformation_external_es2.glsl"; + private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH = + "shaders/fragment_shader_transformation_external_yuv_es3.glsl"; private static final ImmutableList NDC_SQUARE = ImmutableList.of( new float[] {-1, -1, 0, 1}, @@ -84,6 +90,10 @@ import java.util.Arrays; /** The {@link MatrixTransformation MatrixTransformations} to apply. */ private final ImmutableList matrixTransformations; + /** The {@link RgbMatrix RgbMatrices} to apply. */ + private final ImmutableList rgbMatrices; + /** Whether the frame is in HDR or not. */ + private final boolean useHdr; /** * The transformation matrices provided by the {@link MatrixTransformation MatrixTransformations} * for the most recent frame. @@ -117,19 +127,25 @@ import java.util.Arrays; * * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to - * apply to each frame in order. + * apply to each frame in order. Can be empty to apply no vertex transformations. + * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be + * empty to apply no color transformations. * @param useHdr Whether input and output colors are HDR. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. */ public static MatrixTransformationProcessor create( - Context context, ImmutableList matrixTransformations, boolean useHdr) + Context context, + ImmutableList matrixTransformations, + ImmutableList rgbMatrices, + boolean useHdr) throws FrameProcessingException { GlProgram glProgram = - createGlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_COPY_PATH); + createGlProgram( + context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH); // No transfer functions needed, because input and output are both optical colors. - return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr); + return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr); } /** @@ -148,7 +164,9 @@ import java.util.Arrays; * * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to - * apply to each frame in order. + * apply to each frame in order. Can be empty to apply no vertex transformations. + * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be + * empty to apply no color transformations. * @param electricalColorInfo The electrical {@link ColorInfo} describing input colors. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. @@ -156,13 +174,16 @@ import java.util.Arrays; public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotf( Context context, ImmutableList matrixTransformations, + ImmutableList rgbMatrices, ColorInfo electricalColorInfo) throws FrameProcessingException { boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); String vertexShaderFilePath = useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; String fragmentShaderFilePath = - useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + useHdr + ? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH + : FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH; GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); // TODO(b/241902517): Implement gamma transfer functions. @@ -184,7 +205,7 @@ import java.util.Arrays; glProgram.setIntUniform("uEotfColorTransfer", colorTransfer); } - return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr); + return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr); } /** @@ -199,7 +220,9 @@ import java.util.Arrays; * * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to - * apply to each frame in order. + * apply to each frame in order. Can be empty to apply no vertex transformations. + * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be + * empty to apply no color transformations. * @param electricalColorInfo The electrical {@link ColorInfo} describing output colors. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. @@ -207,13 +230,14 @@ import java.util.Arrays; public static MatrixTransformationProcessor createApplyingOetf( Context context, ImmutableList matrixTransformations, + ImmutableList rgbMatrices, ColorInfo electricalColorInfo) throws FrameProcessingException { boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); String vertexShaderFilePath = useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; String fragmentShaderFilePath = - useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH; + useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_PATH; GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); // TODO(b/241902517): Implement gamma transfer functions. @@ -224,7 +248,7 @@ import java.util.Arrays; glProgram.setIntUniform("uOetfColorTransfer", colorTransfer); } - return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr); + return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr); } /** @@ -239,7 +263,9 @@ import java.util.Arrays; * * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to - * apply to each frame in order. + * apply to each frame in order. Can be empty to apply no vertex transformations. + * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be + * empty to apply no color transformations. * @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. @@ -247,13 +273,16 @@ import java.util.Arrays; public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotfThenOetf( Context context, ImmutableList matrixTransformations, + ImmutableList rgbMatrices, ColorInfo electricalColorInfo) throws FrameProcessingException { boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); String vertexShaderFilePath = useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; String fragmentShaderFilePath = - useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + useHdr + ? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH + : FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH; GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); // TODO(b/241902517): Implement gamma transfer functions. @@ -273,7 +302,7 @@ import java.util.Arrays; glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE); } - return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr); + return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr); } /** @@ -281,17 +310,22 @@ import java.util.Arrays; * * @param glProgram The {@link GlProgram}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to - * apply to each frame in order. + * apply to each frame in order. Can be empty to apply no vertex transformations. + * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be + * empty to apply no color transformations. * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code * EXT_YUV_target} OpenGL extension. */ private MatrixTransformationProcessor( GlProgram glProgram, ImmutableList matrixTransformations, + ImmutableList rgbMatrices, boolean useHdr) { super(useHdr); this.glProgram = glProgram; this.matrixTransformations = matrixTransformations; + this.rgbMatrices = rgbMatrices; + this.useHdr = useHdr; transformationMatrixCache = new float[matrixTransformations.size()][16]; compositeTransformationMatrix = new float[16]; @@ -333,11 +367,13 @@ import java.util.Arrays; if (visiblePolygon.size() < 3) { return; // Need at least three visible vertices for a triangle. } - + float[] compositeRgbMatrix = + createCompositeRgbaMatrixArray(rgbMatrices, useHdr, presentationTimeUs); try { glProgram.use(); glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrix); + glProgram.setFloatsUniform("uRgbMatrix", compositeRgbMatrix); glProgram.setBufferAttribute( "aFramePosition", GlUtil.createVertexBuffer(visiblePolygon), @@ -425,4 +461,31 @@ import java.util.Arrays; } return matrixChanged; } + + // TODO(b/239757183): Add caching for RgbMatrix and refactor RgbMatrix and MatrixTransformation + // composing. + private static float[] createCompositeRgbaMatrixArray( + ImmutableList rgbMatrices, boolean useHdr, long presentationTimeUs) { + float[] tempResultMatrix = new float[16]; + float[] compositeRgbaMatrix = new float[16]; + Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0); + + for (int i = 0; i < rgbMatrices.size(); i++) { + Matrix.multiplyMM( + /* result= */ tempResultMatrix, + /* resultOffset= */ 0, + /* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs, useHdr), + /* lhsOffset= */ 0, + /* rhs= */ compositeRgbaMatrix, + /* rhsOffset= */ 0); + System.arraycopy( + /* src= */ tempResultMatrix, + /* srcPos= */ 0, + /* dest= */ compositeRgbaMatrix, + /* destPost= */ 0, + /* length= */ tempResultMatrix.length); + } + + return compositeRgbaMatrix; + } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java b/libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java index 2e96368182..99e218ed3d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java @@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.UnstableApi; +import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** Provides common color filters. */ @@ -91,9 +92,13 @@ public class RgbFilter implements RgbMatrix { } @Override - public RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr) + public MatrixTransformationProcessor toGlTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException { checkForConsistentHdrSetting(useHdr); - return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr); + return MatrixTransformationProcessor.create( + context, + /* matrixTransformations= */ ImmutableList.of(), + /* rgbMatrices= */ ImmutableList.of(this), + useHdr); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java index 82c2a96bc0..1a5371695d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java @@ -19,6 +19,7 @@ package androidx.media3.effect; import android.content.Context; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.UnstableApi; +import com.google.common.collect.ImmutableList; /** * Specifies a 4x4 RGB color transformation matrix to apply to each frame in the fragment shader. @@ -39,8 +40,12 @@ public interface RgbMatrix extends GlEffect { float[] getMatrix(long presentationTimeUs, boolean useHdr); @Override - default RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr) + default MatrixTransformationProcessor toGlTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException { - return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr); + return MatrixTransformationProcessor.create( + context, + /* matrixTransformations= */ ImmutableList.of(), + /* rgbMatrices= */ ImmutableList.of(this), + useHdr); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java deleted file mode 100644 index 6cdf76844c..0000000000 --- a/libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.effect; - -import android.content.Context; -import android.opengl.GLES20; -import android.opengl.Matrix; -import android.util.Pair; -import androidx.media3.common.FrameProcessingException; -import androidx.media3.common.util.GlProgram; -import androidx.media3.common.util.GlUtil; -import com.google.common.collect.ImmutableList; -import java.io.IOException; - -/** - * Applies a sequence of {@link RgbMatrix} to each frame. - * - *

After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the - * color space. Intermediate results are not clamped. - */ -/* package */ final class RgbMatrixProcessor extends SingleFrameGlTextureProcessor { - private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; - private static final String FRAGMENT_SHADER_PATH = - "shaders/fragment_shader_transformation_es2.glsl"; - - private final GlProgram glProgram; - private final ImmutableList rgbMatrices; - private final boolean useHdr; - - // TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor. - /** - * Creates a new instance. - * - * @param context The {@link Context}. - * @param rgbMatrix The {@link RgbMatrix} to apply to each frame. - * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. - * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL - * operation fails or is unsupported. - */ - public RgbMatrixProcessor(Context context, RgbMatrix rgbMatrix, boolean useHdr) - throws FrameProcessingException { - this(context, ImmutableList.of(rgbMatrix), useHdr); - } - - /** - * Creates a new instance. - * - * @param context The {@link Context}. - * @param rgbMatrices The {@link RgbMatrix} to apply to each frame. - * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. - * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL - * operation fails or is unsupported. - */ - public RgbMatrixProcessor(Context context, ImmutableList rgbMatrices, boolean useHdr) - throws FrameProcessingException { - super(useHdr); - this.rgbMatrices = rgbMatrices; - this.useHdr = useHdr; - - try { - glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); - } catch (IOException | GlUtil.GlException e) { - throw new FrameProcessingException(e); - } - - // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. - glProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - - float[] identityMatrix = new float[16]; - Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0); - glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix); - glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); - } - - @Override - public Pair configure(int inputWidth, int inputHeight) { - return Pair.create(inputWidth, inputHeight); - } - - private static float[] createCompositeRgbaMatrixArray( - ImmutableList rgbMatrices, boolean useHdr, long presentationTimeUs) { - float[] tempResultMatrix = new float[16]; - float[] compositeRgbaMatrix = new float[16]; - Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0); - - for (int i = 0; i < rgbMatrices.size(); i++) { - Matrix.multiplyMM( - /* result= */ tempResultMatrix, - /* resultOffset= */ 0, - /* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs, useHdr), - /* lhsOffset= */ 0, - /* rhs= */ compositeRgbaMatrix, - /* rhsOffset= */ 0); - System.arraycopy( - /* src= */ tempResultMatrix, - /* srcPos= */ 0, - /* dest= */ compositeRgbaMatrix, - /* destPost= */ 0, - /* length= */ tempResultMatrix.length); - } - - return compositeRgbaMatrix; - } - - @Override - public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { - // TODO(b/239431666): Add caching for compacting Matrices. - float[] rgbMatrixArray = - createCompositeRgbaMatrixArray(rgbMatrices, useHdr, presentationTimeUs); - try { - glProgram.use(); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); - glProgram.setFloatsUniform("uColorMatrix", rgbMatrixArray); - glProgram.bindAttributesAndUniforms(); - - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - } catch (GlUtil.GlException e) { - throw new FrameProcessingException(e, presentationTimeUs); - } - } - - @Override - public void release() throws FrameProcessingException { - super.release(); - try { - glProgram.delete(); - } catch (GlUtil.GlException e) { - throw new FrameProcessingException(e); - } - } -}