From 1b6482960e9240a7697c6a13fdde198e3bf375a1 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Tue, 30 Aug 2022 17:50:21 +0000 Subject: [PATCH] HDR: Use factory for MatrixTransformationProcessor. Separate MatrixTransformationProcessor constructors by color input and output. PiperOrigin-RevId: 471034768 --- ...atrixTransformationProcessorPixelTest.java | 30 +-- ...lMatrixTransformationProcessorWrapper.java | 20 +- .../effect/GlEffectsFrameProcessor.java | 26 +- .../media3/effect/GlMatrixTransformation.java | 4 +- .../effect/MatrixTransformationProcessor.java | 247 ++++++++++-------- 5 files changed, 182 insertions(+), 145 deletions(-) diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java index 826e9ed923..479e8f2f87 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java @@ -28,6 +28,7 @@ import android.opengl.EGLSurface; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.GlUtil; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; @@ -94,11 +95,10 @@ public final class MatrixTransformationProcessorPixelTest { public void drawFrame_noEdits_producesExpectedOutput() throws Exception { String testId = "drawFrame_noEdits"; Matrix identityMatrix = new Matrix(); + MatrixTransformation noEditsTransformation = (long presentationTimeUs) -> identityMatrix; matrixTransformationFrameProcessor = - new MatrixTransformationProcessor( - context, - /* useHdr= */ false, - /* matrixTransformation= */ (long presentationTimeUs) -> identityMatrix); + MatrixTransformationProcessor.create( + context, ImmutableList.of(noEditsTransformation), /* useHdr= */ false); matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); @@ -120,11 +120,11 @@ public final class MatrixTransformationProcessorPixelTest { String testId = "drawFrame_translateRight"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); + MatrixTransformation translateRightTransformation = + (long presentationTimeUs) -> translateRightMatrix; matrixTransformationFrameProcessor = - new MatrixTransformationProcessor( - context, - /* useHdr= */ false, - /* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix); + MatrixTransformationProcessor.create( + context, ImmutableList.of(translateRightTransformation), /* useHdr= */ false); matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); @@ -146,11 +146,10 @@ public final class MatrixTransformationProcessorPixelTest { String testId = "drawFrame_scaleNarrow"; Matrix scaleNarrowMatrix = new Matrix(); scaleNarrowMatrix.postScale(.5f, 1.2f); + MatrixTransformation scaleNarrowTransformation = (long presentationTimeUs) -> scaleNarrowMatrix; matrixTransformationFrameProcessor = - new MatrixTransformationProcessor( - context, - /* useHdr= */ false, - /* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix); + MatrixTransformationProcessor.create( + context, ImmutableList.of(scaleNarrowTransformation), /* useHdr= */ false); matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH); @@ -172,11 +171,10 @@ public final class MatrixTransformationProcessorPixelTest { String testId = "drawFrame_rotate90"; Matrix rotate90Matrix = new Matrix(); rotate90Matrix.postRotate(/* degrees= */ 90); + MatrixTransformation rotate90Transformation = (long presentationTimeUs) -> rotate90Matrix; matrixTransformationFrameProcessor = - new MatrixTransformationProcessor( - context, - /* useHdr= */ false, - /* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix); + MatrixTransformationProcessor.create( + context, ImmutableList.of(rotate90Transformation), /* useHdr= */ false); matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java index 24e2a1cf37..cbfc6d8ba0 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java @@ -368,13 +368,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Presentation.createForWidthAndHeight( outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT)); - MatrixTransformationProcessor matrixTransformationProcessor = - new MatrixTransformationProcessor( - context, - matrixTransformationListBuilder.build(), - sampleFromExternalTexture, - colorInfo, - /* outputElectricalColors= */ true); + MatrixTransformationProcessor matrixTransformationProcessor; + ImmutableList expandedMatrixTransformations = + matrixTransformationListBuilder.build(); + if (sampleFromExternalTexture) { + matrixTransformationProcessor = + MatrixTransformationProcessor.createWithExternalSamplerApplyingEotfThenOetf( + context, expandedMatrixTransformations, colorInfo); + } else { + matrixTransformationProcessor = + MatrixTransformationProcessor.createApplyingOetf( + context, expandedMatrixTransformations, colorInfo); + } + matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); Pair outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java index 11bddb4e13..300fd7f066 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java @@ -206,13 +206,17 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) { - textureProcessorListBuilder.add( - new MatrixTransformationProcessor( - context, - matrixTransformations, - sampleFromExternalTexture, - colorInfo, - /* outputElectricalColors= */ false)); + MatrixTransformationProcessor matrixTransformationProcessor; + if (sampleFromExternalTexture) { + matrixTransformationProcessor = + MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf( + context, matrixTransformations, colorInfo); + } else { + matrixTransformationProcessor = + MatrixTransformationProcessor.create( + context, matrixTransformations, ColorInfo.isTransferHdr(colorInfo)); + } + textureProcessorListBuilder.add(matrixTransformationProcessor); matrixTransformationListBuilder = new ImmutableList.Builder<>(); sampleFromExternalTexture = false; } @@ -237,12 +241,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { // TODO(b/239757183): Remove the unnecessary MatrixTransformationProcessor after it got // merged with RgbMatrixProcessor. textureProcessorListBuilder.add( - new MatrixTransformationProcessor( - context, - ImmutableList.of(), - sampleFromExternalTexture, - colorInfo, - /* outputElectricalColors= */ false)); + MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf( + context, /* matrixTransformations= */ ImmutableList.of(), colorInfo)); sampleFromExternalTexture = false; } textureProcessorListBuilder.add( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java b/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java index 4ae5a62646..9f04b230d5 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java @@ -20,6 +20,7 @@ import android.opengl.Matrix; import android.util.Pair; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.UnstableApi; +import com.google.common.collect.ImmutableList; /** * Specifies a 4x4 transformation {@link Matrix} to apply in the vertex shader for each frame. @@ -54,6 +55,7 @@ public interface GlMatrixTransformation extends GlEffect { @Override default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException { - return new MatrixTransformationProcessor(context, useHdr, /* matrixTransformation= */ this); + return MatrixTransformationProcessor.create( + context, /* matrixTransformations= */ ImmutableList.of(this), useHdr); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java index 2686327658..c4fcb54a71 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java @@ -112,97 +112,61 @@ import java.util.Arrays; /** * Creates a new instance. * - * @param context The {@link Context}. - * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. - * @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation - * matrix to use for each frame. - * @throws FrameProcessingException If a problem occurs while reading shader files. - */ - public MatrixTransformationProcessor( - Context context, boolean useHdr, MatrixTransformation matrixTransformation) - throws FrameProcessingException { - this( - createGlProgram( - context, - /* inputElectricalColorsFromExternalTexture= */ false, - useHdr, - /* outputElectricalColors= */ false), - ImmutableList.of(matrixTransformation), - useHdr); - } - - /** - * Creates a new instance. - * - * @param context The {@link Context}. - * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. - * @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation - * matrix to use for each frame. - * @throws FrameProcessingException If a problem occurs while reading shader files. - */ - public MatrixTransformationProcessor( - Context context, boolean useHdr, GlMatrixTransformation matrixTransformation) - throws FrameProcessingException { - this( - createGlProgram( - context, - /* inputElectricalColorsFromExternalTexture= */ false, - useHdr, - /* outputElectricalColors= */ false), - ImmutableList.of(matrixTransformation), - useHdr); - } - - /** - * Creates a new instance. - * - *

Able to convert nonlinear electrical {@link ColorInfo} inputs and outputs to and from the - * intermediate optical {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma - * RGB BT.709 for SDR. + *

Input and output are both intermediate optical colors, which are linear RGB BT.2020 if + * {@code useHdr} is {@code true} and gamma RGB BT.709 if not. * * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * apply to each frame in order. - * @param inputElectricalColorsFromExternalTexture Whether electrical color input will be provided - * using an external texture. If {@code true}, the caller should use {@link - * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with - * the external texture. - * @param electricalColorInfo The electrical {@link ColorInfo}, only used to transform between - * color spaces and transfers, when {@code inputElectricalColorsFromExternalTexture} or {@code - * outputElectricalColors} are {@code true}. If it is {@linkplain - * ColorInfo#isTransferHdr(ColorInfo) HDR}, intermediate {@link GlTextureProcessor} colors - * will be in linear RGB BT.2020. Otherwise, these colors will be in gamma RGB BT.709. - * @param outputElectricalColors If {@code true}, outputs {@code electricalColorInfo}. If {@code - * false}, outputs intermediate colors of linear RGB BT.2020 if {@code electricalColorInfo} is - * {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 otherwise. + * @param useHdr Whether input and output colors are HDR. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. */ - public MatrixTransformationProcessor( + public static MatrixTransformationProcessor create( + Context context, ImmutableList matrixTransformations, boolean useHdr) + throws FrameProcessingException { + GlProgram glProgram = + createGlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_COPY_PATH); + + // No transfer functions needed, because input and output are both optical colors. + return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr); + } + + /** + * Creates a new instance. + * + *

Input will be sampled from an external texture. The caller should use {@link + * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the + * external texture. + * + *

Applies the {@code electricalColorInfo} EOTF to convert from electrical color input, to + * intermediate optical {@link GlTextureProcessor} color output, before {@code + * matrixTransformations} are applied. + * + *

Intermediate optical colors are linear RGB BT.2020 if {@code electricalColorInfo} is + * {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 if not. + * + * @param context The {@link Context}. + * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to + * apply to each frame in order. + * @param electricalColorInfo The electrical {@link ColorInfo} describing input colors. + * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL + * operation fails or is unsupported. + */ + public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotf( Context context, ImmutableList matrixTransformations, - boolean inputElectricalColorsFromExternalTexture, - ColorInfo electricalColorInfo, - boolean outputElectricalColors) + ColorInfo electricalColorInfo) throws FrameProcessingException { - this( - createGlProgram( - context, - inputElectricalColorsFromExternalTexture, - ColorInfo.isTransferHdr(electricalColorInfo), - outputElectricalColors), - matrixTransformations, - ColorInfo.isTransferHdr(electricalColorInfo)); - if (!ColorInfo.isTransferHdr(electricalColorInfo)) { - return; - } + boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); + String vertexShaderFilePath = + useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; + String fragmentShaderFilePath = + useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); - @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; - checkArgument( - colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); - if (inputElectricalColorsFromExternalTexture) { + // TODO(b/241902517): Implement gamma transfer functions. + if (useHdr) { // In HDR editing mode the decoder output is sampled in YUV. if (!GlUtil.isYuvTargetExtensionSupported()) { throw new FrameProcessingException( @@ -214,14 +178,102 @@ import java.util.Arrays; ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); - // TODO(b/241902517): Implement gamma transfer functions. + @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; + checkArgument( + colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); + glProgram.setIntUniform("uEotfColorTransfer", colorTransfer); + } - // If electrical colors are both input and output, no EOTF is needed. - glProgram.setIntUniform( - "uEotfColorTransfer", outputElectricalColors ? Format.NO_VALUE : colorTransfer); - } else if (outputElectricalColors) { + return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr); + } + + /** + * Creates a new instance. + * + *

Applies the {@code electricalColorInfo} OETF to convert from intermediate optical {@link + * GlTextureProcessor} color input, to electrical color output, after {@code + * matrixTransformations} are applied. + * + *

Intermediate optical colors are linear RGB BT.2020 if {@code electricalColorInfo} is + * {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 if not. + * + * @param context The {@link Context}. + * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to + * apply to each frame in order. + * @param electricalColorInfo The electrical {@link ColorInfo} describing output colors. + * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL + * operation fails or is unsupported. + */ + public static MatrixTransformationProcessor createApplyingOetf( + Context context, + ImmutableList matrixTransformations, + ColorInfo electricalColorInfo) + throws FrameProcessingException { + boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); + String vertexShaderFilePath = + useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; + String fragmentShaderFilePath = + useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH; + GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); + + // TODO(b/241902517): Implement gamma transfer functions. + if (useHdr) { + @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; + checkArgument( + colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); glProgram.setIntUniform("uOetfColorTransfer", colorTransfer); } + + return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr); + } + + /** + * Creates a new instance. + * + *

Input will be sampled from an external texture. The caller should use {@link + * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the + * external texture. + * + *

Applies the OETF, {@code matrixTransformations}, then the EOTF, to convert from and to input + * and output electrical colors. + * + * @param context The {@link Context}. + * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to + * apply to each frame in order. + * @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors. + * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL + * operation fails or is unsupported. + */ + public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotfThenOetf( + Context context, + ImmutableList matrixTransformations, + ColorInfo electricalColorInfo) + throws FrameProcessingException { + boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); + String vertexShaderFilePath = + useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; + String fragmentShaderFilePath = + useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); + + // TODO(b/241902517): Implement gamma transfer functions. + if (useHdr) { + // In HDR editing mode the decoder output is sampled in YUV. + if (!GlUtil.isYuvTargetExtensionSupported()) { + throw new FrameProcessingException( + "The EXT_YUV_target extension is required for HDR editing input."); + } + glProgram.setFloatsUniform( + "uYuvToRgbColorTransform", + electricalColorInfo.colorRange == C.COLOR_RANGE_FULL + ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX + : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); + + // No transfer functions needed, because the EOTF and OETF cancel out. + glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE); + } + + return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr); } /** @@ -243,36 +295,15 @@ import java.util.Arrays; transformationMatrixCache = new float[matrixTransformations.size()][16]; compositeTransformationMatrix = new float[16]; - tempResultMatrix = new float[16]; Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); + tempResultMatrix = new float[16]; visiblePolygon = NDC_SQUARE; } private static GlProgram createGlProgram( - Context context, - boolean inputElectricalColorsFromExternalTexture, - boolean useHdr, - boolean outputElectricalColors) + Context context, String vertexShaderFilePath, String fragmentShaderFilePath) throws FrameProcessingException { - String vertexShaderFilePath; - String fragmentShaderFilePath; - if (inputElectricalColorsFromExternalTexture) { - if (useHdr) { - vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH; - fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH; - } else { - vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; - fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_PATH; - } - } else if (outputElectricalColors && useHdr) { - vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH; - fragmentShaderFilePath = FRAGMENT_SHADER_OETF_ES3_PATH; - } else { - vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; - fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH; - } - GlProgram glProgram; try { glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);