From 5fe10d76520137d860fe9f847f467cb4926427e0 Mon Sep 17 00:00:00 2001 From: tofunmi Date: Wed, 17 May 2023 18:11:48 +0000 Subject: [PATCH] HDR texture asset loading PiperOrigin-RevId: 532846248 --- ...hader_transformation_external_yuv_es3.glsl | 2 +- ...hader_transformation_hdr_internal_es3.glsl | 232 +++++++++++++++ .../media3/effect/DefaultShaderProgram.java | 56 ++-- .../effect/DefaultVideoFrameProcessor.java | 10 +- ...oFrameProcessorTextureOutputPixelTest.java | 275 ++++++++++++++---- 5 files changed, 485 insertions(+), 90 deletions(-) create mode 100644 libraries/effect/src/main/assets/shaders/fragment_shader_transformation_hdr_internal_es3.glsl diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl index a195f5de20..84d05c0f97 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl @@ -29,7 +29,7 @@ // for outputting to intermediate shaders, or COLOR_TRANSFER_ST2084 / // COLOR_TRANSFER_HLG to output electrical colors via an OETF (e.g. to an // encoder). -// The output will be red if an error has occurred. +// The output will be red or blue if an error has occurred. #extension GL_OES_EGL_image_external : require #extension GL_EXT_YUV_target : require diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_hdr_internal_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_hdr_internal_es3.glsl new file mode 100644 index 0000000000..6b0e377a29 --- /dev/null +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_hdr_internal_es3.glsl @@ -0,0 +1,232 @@ +#version 300 es +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 3 fragment shader that: +// 1. Samples electrical (HLG or PQ) BT.2020 RGB from an internal texture. +// 2. Applies an EOTF based on uInputColorTransfer, yielding optical linear +// BT.2020 RGB. +// 3. Optionally applies a BT2020 to BT709 OOTF, if OpenGL tone-mapping is +// requested via uApplyHdrToSdrToneMapping. +// 4. Applies a 4x4 RGB color matrix to change the pixel colors. +// 5. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR +// for outputting to intermediate shaders, or COLOR_TRANSFER_ST2084 / +// COLOR_TRANSFER_HLG to output electrical colors via an OETF (e.g. to an +// encoder). +// The output will be red or blue if an error has occurred. + +precision mediump float; +uniform sampler2D uTexSampler; +uniform mat4 uRgbMatrix; +// C.java#ColorTransfer value. +// Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed. +uniform int uInputColorTransfer; +uniform int uApplyHdrToSdrToneMapping; +// C.java#ColorTransfer value. +// Only COLOR_TRANSFER_LINEAR, COLOR_TRANSFER_GAMMA_2_2, COLOR_TRANSFER_ST2084, +// and COLOR_TRANSFER_HLG are allowed. +uniform int uOutputColorTransfer; +in vec2 vTexSamplingCoord; +out vec4 outColor; + +// LINT.IfChange(color_transfer) +const int COLOR_TRANSFER_LINEAR = 1; +const int COLOR_TRANSFER_GAMMA_2_2 = 10; +const int COLOR_TRANSFER_ST2084 = 6; +const int COLOR_TRANSFER_HLG = 7; + +// TODO(b/227624622): Consider using mediump to save precision, if it won't lead +// to noticeable quantization errors. + +// BT.2100 / BT.2020 HLG EOTF for one channel. +highp float hlgEotfSingleChannel(highp float hlgChannel) { + // Specification: + // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float a = 0.17883277; + const highp float b = 0.28466892; + const highp float c = 0.55991073; + return hlgChannel <= 0.5 ? hlgChannel * hlgChannel / 3.0 : + (b + exp((hlgChannel - c) / a)) / 12.0; +} + +// BT.2100 / BT.2020 HLG EOTF. +highp vec3 hlgEotf(highp vec3 hlgColor) { + return vec3( + hlgEotfSingleChannel(hlgColor.r), + hlgEotfSingleChannel(hlgColor.g), + hlgEotfSingleChannel(hlgColor.b) + ); +} + +// BT.2100 / BT.2020 PQ EOTF. +highp vec3 pqEotf(highp vec3 pqColor) { + // Specification: + // https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=250-263;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float m1 = (2610.0 / 16384.0); + const highp float m2 = (2523.0 / 4096.0) * 128.0; + const highp float c1 = (3424.0 / 4096.0); + const highp float c2 = (2413.0 / 4096.0) * 32.0; + const highp float c3 = (2392.0 / 4096.0) * 32.0; + + highp vec3 temp = pow(clamp(pqColor, 0.0, 1.0), 1.0 / vec3(m2)); + temp = max(temp - c1, 0.0) / (c2 - c3 * temp); + return pow(temp, 1.0 / vec3(m1)); +} + +// Applies the appropriate EOTF to convert nonlinear electrical values to linear +// optical values. Input and output are both normalized to [0, 1]. +highp vec3 applyEotf(highp vec3 electricalColor) { + if (uInputColorTransfer == COLOR_TRANSFER_ST2084) { + return pqEotf(electricalColor); + } else if (uInputColorTransfer == COLOR_TRANSFER_HLG) { + return hlgEotf(electricalColor); + } else { + // Output red as an obviously visible error. + return vec3(1.0, 0.0, 0.0); + } +} + +// Apply the HLG BT2020 to BT709 OOTF. +highp vec3 applyHlgBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) { + // Reference ("HLG Reference OOTF" section): + // https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.2100-2-201807-I!!PDF-E.pdf + // Matrix values based on computeXYZMatrix(BT2020Primaries, BT2020WhitePoint) + // https://cs.android.com/android/platform/superproject/+/master:frameworks/base/libs/hwui/utils/HostColorSpace.cpp;l=200-232;drc=86bd214059cd6150304888a285941bf74af5b687 + const mat3 RGB_TO_XYZ_BT2020 = mat3( + 0.63695805f, 0.26270021f, 0.00000000f, + 0.14461690f, 0.67799807f, 0.02807269f, + 0.16888098f, 0.05930172f, 1.06098506f); + // Matrix values based on computeXYZMatrix(BT709Primaries, BT709WhitePoint) + const mat3 XYZ_TO_RGB_BT709 = mat3( + 3.24096994f, -0.96924364f, 0.05563008f, + -1.53738318f, 1.87596750f, -0.20397696f, + -0.49861076f, 0.04155506f, 1.05697151f); + // hlgGamma is 1.2 + 0.42 * log10(nominalPeakLuminance/1000); + // nominalPeakLuminance was selected to use a 500 as a typical value, used + // in https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/tonemap/tonemap.cpp;drc=7a577450e536aa1e99f229a0cb3d3531c82e8a8d;l=62, + // b/199162498#comment35, and + // https://www.microsoft.com/applied-sciences/uploads/projects/investigation-of-hdr-vs-tone-mapped-sdr/investigation-of-hdr-vs-tone-mapped-sdr.pdf. + const float hlgGamma = 1.0735674018211279; + + vec3 linearXyzBt2020 = RGB_TO_XYZ_BT2020 * linearRgbBt2020; + vec3 linearXyzBt709 = + linearXyzBt2020 * pow(linearXyzBt2020[1], hlgGamma - 1.0); + vec3 linearRgbBt709 = clamp((XYZ_TO_RGB_BT709 * linearXyzBt709), 0.0, 1.0); + return linearRgbBt709; +} + +// Apply the PQ BT2020 to BT709 OOTF. +highp vec3 applyPqBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) { + float pqPeakLuminance = 10000.0; + float sdrPeakLuminance = 500.0; + + return linearRgbBt2020 * pqPeakLuminance / sdrPeakLuminance; +} + +highp vec3 applyBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) { + if (uInputColorTransfer == COLOR_TRANSFER_ST2084) { + return applyPqBt2020ToBt709Ootf(linearRgbBt2020); + } else if (uInputColorTransfer == COLOR_TRANSFER_HLG) { + return applyHlgBt2020ToBt709Ootf(linearRgbBt2020); + } else { + // Output green as an obviously visible error. + return vec3(0.0, 1.0, 0.0); + } +} + +// BT.2100 / BT.2020 HLG OETF for one channel. +highp float hlgOetfSingleChannel(highp float linearChannel) { + // Specification: + // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float a = 0.17883277; + const highp float b = 0.28466892; + const highp float c = 0.55991073; + + return linearChannel <= 1.0 / 12.0 ? sqrt(3.0 * linearChannel) : + a * log(12.0 * linearChannel - b) + c; +} + +// BT.2100 / BT.2020 HLG OETF. +highp vec3 hlgOetf(highp vec3 linearColor) { + return vec3( + hlgOetfSingleChannel(linearColor.r), + hlgOetfSingleChannel(linearColor.g), + hlgOetfSingleChannel(linearColor.b) + ); +} + +// BT.2100 / BT.2020, PQ / ST2084 OETF. +highp vec3 pqOetf(highp vec3 linearColor) { + // Specification: + // https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=514-527;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float m1 = (2610.0 / 16384.0); + const highp float m2 = (2523.0 / 4096.0) * 128.0; + const highp float c1 = (3424.0 / 4096.0); + const highp float c2 = (2413.0 / 4096.0) * 32.0; + const highp float c3 = (2392.0 / 4096.0) * 32.0; + + highp vec3 temp = pow(linearColor, vec3(m1)); + temp = (c1 + c2 * temp) / (1.0 + c3 * temp); + return pow(temp, vec3(m2)); +} + +// BT.709 gamma 2.2 OETF for one channel. +float gamma22OetfSingleChannel(highp float linearChannel) { + // Reference: + // https://developer.android.com/reference/android/hardware/DataSpace#TRANSFER_GAMMA2_2 + return pow(linearChannel, (1.0 / 2.2)); +} + +// BT.709 gamma 2.2 OETF. +vec3 gamma22Oetf(highp vec3 linearColor) { + return vec3( + gamma22OetfSingleChannel(linearColor.r), + gamma22OetfSingleChannel(linearColor.g), + gamma22OetfSingleChannel(linearColor.b)); +} + +// Applies the appropriate OETF to convert linear optical signals to nonlinear +// electrical signals. Input and output are both normalized to [0, 1]. +highp vec3 applyOetf(highp vec3 linearColor) { + if (uOutputColorTransfer == COLOR_TRANSFER_ST2084) { + return pqOetf(linearColor); + } else if (uOutputColorTransfer == COLOR_TRANSFER_HLG) { + return hlgOetf(linearColor); + } else if (uOutputColorTransfer == COLOR_TRANSFER_GAMMA_2_2) { + return gamma22Oetf(linearColor); + } else if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) { + return linearColor; + } else { + // Output blue as an obviously visible error. + return vec3(0.0, 0.0, 1.0); + } +} + +void main() { + vec3 opticalColorBt2020 = applyEotf( + texture(uTexSampler, vTexSamplingCoord).xyz); + vec4 opticalColor = (uApplyHdrToSdrToneMapping == 1) + ? vec4(applyBt2020ToBt709Ootf(opticalColorBt2020), 1.0) + : vec4(opticalColorBt2020, 1.0); + vec4 transformedColors = uRgbMatrix * opticalColor; + outColor = vec4(applyOetf(transformedColors.rgb), 1.0); +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java index 0e1a94deea..61a4b8faa8 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java @@ -71,6 +71,8 @@ import java.util.List; "shaders/fragment_shader_transformation_external_yuv_es3.glsl"; private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH = "shaders/fragment_shader_transformation_sdr_external_es2.glsl"; + private static final String FRAGMENT_SHADER_TRANSFORMATION_HDR_INTERNAL_ES3_PATH = + "shaders/fragment_shader_transformation_hdr_internal_es3.glsl"; private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH = "shaders/fragment_shader_transformation_sdr_internal_es2.glsl"; private static final ImmutableList NDC_SQUARE = @@ -204,16 +206,18 @@ import java.util.List; boolean enableColorTransfers, @InputType int inputType) throws VideoFrameProcessingException { - checkState( - !ColorInfo.isTransferHdr(inputColorInfo), - "DefaultShaderProgram doesn't support HDR internal sampler input yet."); checkState( inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB || inputType == INPUT_TYPE_BITMAP); - GlProgram glProgram = - createGlProgram( - context, - VERTEX_SHADER_TRANSFORMATION_PATH, - FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH); + boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo); + String vertexShaderFilePath = + isInputTransferHdr + ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH + : VERTEX_SHADER_TRANSFORMATION_PATH; + String fragmentShaderFilePath = + isInputTransferHdr + ? FRAGMENT_SHADER_TRANSFORMATION_HDR_INTERNAL_ES3_PATH + : FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH; + GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer); return createWithSampler( glProgram, @@ -268,8 +272,23 @@ import java.util.List; isInputTransferHdr ? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH; + GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); + if (isInputTransferHdr) { + // In HDR editing mode the decoder output is sampled in YUV. + if (!GlUtil.isYuvTargetExtensionSupported()) { + throw new VideoFrameProcessingException( + "The EXT_YUV_target extension is required for HDR editing input."); + } + glProgram.setFloatsUniform( + "uYuvToRgbColorTransform", + inputColorInfo.colorRange == C.COLOR_RANGE_FULL + ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX + : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); + glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer); + } + return createWithSampler( - createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath), + glProgram, matrixTransformations, rgbMatrices, inputColorInfo, @@ -343,31 +362,16 @@ import java.util.List; List rgbMatrices, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - boolean enableColorTransfers) - throws VideoFrameProcessingException { + boolean enableColorTransfers) { boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo); @C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer; if (isInputTransferHdr) { checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020); checkArgument(enableColorTransfers); - - // In HDR editing mode the decoder output is sampled in YUV. - if (!GlUtil.isYuvTargetExtensionSupported()) { - throw new VideoFrameProcessingException( - "The EXT_YUV_target extension is required for HDR editing input."); - } - glProgram.setFloatsUniform( - "uYuvToRgbColorTransform", - inputColorInfo.colorRange == C.COLOR_RANGE_FULL - ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX - : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); - - checkArgument(ColorInfo.isTransferHdr(inputColorInfo)); - glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer); // TODO(b/239735341): Add a setBooleanUniform method to GlProgram. glProgram.setIntUniform( "uApplyHdrToSdrToneMapping", - /* value= */ (outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020) ? 1 : 0); + /* value= */ (outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020) ? GL_TRUE : GL_FALSE); checkArgument( outputColorTransfer != Format.NO_VALUE && outputColorTransfer != C.COLOR_TRANSFER_SDR); glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 0e811e1c58..f29eda1147 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -620,12 +620,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { inputSwitcher.registerInput(INPUT_TYPE_SURFACE); if (!ColorInfo.isTransferHdr(inputColorInfo)) { - // HDR bitmap or texture input is not supported. + // HDR bitmap input is not supported. inputSwitcher.registerInput(INPUT_TYPE_BITMAP); - if (inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB) { - // Image and textureId concatenation not supported. - inputSwitcher.registerInput(INPUT_TYPE_TEXTURE_ID); - } + } + if (inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB) { + // Image and textureId concatenation not supported. + inputSwitcher.registerInput(INPUT_TYPE_TEXTURE_ID); } inputSwitcher.setDownstreamShaderProgram(effectsShaderPrograms.get(0)); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java index eec751a942..63064b4746 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java @@ -15,6 +15,7 @@ */ package androidx.media3.transformer.mh; +import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; @@ -34,6 +35,7 @@ import android.content.Context; import android.graphics.Bitmap; import android.view.Surface; import androidx.media3.common.ColorInfo; +import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; @@ -55,6 +57,7 @@ import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.EncoderUtil; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; +import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; @@ -89,6 +92,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { /** Input HLG video of which we only use the first frame. */ private static final String INPUT_HLG10_MP4_ASSET_STRING = "media/mp4/hlg-1080p.mp4"; + // A passthrough effect allows for testing having an intermediate effect injected, which uses + // different OpenGL shaders from having no effects. private static final GlEffect NO_OP_EFFECT = new GlEffectWrapper(new ScaleAndRotateTransformation.Builder().build()); @@ -111,7 +116,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { /* outputFormat= */ null)) { return; } - videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build(); + videoFrameProcessorTestRunner = getSurfaceInputFrameProcessorTestRunnerBuilder(testId).build(); Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); videoFrameProcessorTestRunner.processFirstFrameAndEnd(); @@ -134,23 +139,14 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { /* outputFormat= */ null)) { return; } - TextureBitmapReader producersBitmapReader = new TextureBitmapReader(); TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); - DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = - new DefaultVideoFrameProcessor.Factory.Builder() - .setTextureOutput( - (outputTexture, presentationTimeUs) -> - inputTextureIntoVideoFrameProcessor( - testId, consumersBitmapReader, outputTexture, presentationTimeUs), - /* textureOutputCapacity= */ 1) - .build(); VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = - new VideoFrameProcessorTestRunner.Builder() - .setTestId(testId) - .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) - .setVideoAssetPath(INPUT_SDR_MP4_ASSET_STRING) - .setBitmapReader(producersBitmapReader) - .build(); + getTexIdProducingFrameProcessorTestRunner( + testId, + consumersBitmapReader, + INPUT_SDR_MP4_ASSET_STRING, + SDR_BT709_LIMITED, + ImmutableList.of()); Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd(); @@ -177,7 +173,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap); videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder(testId) + getSurfaceInputFrameProcessorTestRunnerBuilder(testId) .setEffects(new OverlayEffect(ImmutableList.of(bitmapOverlay))) .build(); Bitmap expectedBitmap = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH); @@ -203,28 +199,16 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { } Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap); - TextureBitmapReader producersBitmapReader = new TextureBitmapReader(); + ImmutableList effects = + ImmutableList.of(new OverlayEffect(ImmutableList.of(bitmapOverlay))); TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); - DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = - new DefaultVideoFrameProcessor.Factory.Builder() - .setTextureOutput( - (outputTexture, presentationTimeUs) -> - inputTextureIntoVideoFrameProcessor( - testId, consumersBitmapReader, outputTexture, presentationTimeUs), - /* textureOutputCapacity= */ 1) - .build(); VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = - new VideoFrameProcessorTestRunner.Builder() - .setTestId(testId) - .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) - .setVideoAssetPath(INPUT_SDR_MP4_ASSET_STRING) - .setBitmapReader(producersBitmapReader) - .setEffects(new OverlayEffect(ImmutableList.of(bitmapOverlay))) - .build(); - texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd(); - texIdProducingVideoFrameProcessorTestRunner.release(); + getTexIdProducingFrameProcessorTestRunner( + testId, consumersBitmapReader, INPUT_SDR_MP4_ASSET_STRING, SDR_BT709_LIMITED, effects); Bitmap expectedBitmap = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH); + texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd(); + texIdProducingVideoFrameProcessorTestRunner.release(); Bitmap actualBitmap = consumersBitmapReader.getBitmap(); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. @@ -249,7 +233,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { } ColorInfo colorInfo = checkNotNull(format.colorInfo); videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder(testId) + getSurfaceInputFrameProcessorTestRunnerBuilder(testId) .setInputColorInfo(colorInfo) .setOutputColorInfo(colorInfo) .setVideoAssetPath(INPUT_HLG10_MP4_ASSET_STRING) @@ -267,11 +251,9 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); } - // A passthrough effect allows for testing having an intermediate effect injected, which uses - // different OpenGL shaders from having no effects. @Test - public void noOpEffect_hlg10Input_matchesGoldenFile() throws Exception { - String testId = "noOpEffect_hlg10Input_matchesGoldenFile"; + public void noEffects_hlg10TextureInput_matchesGoldenFile() throws Exception { + String testId = "noEffects_hlg10TextureInput_matchesGoldenFile"; Context context = getApplicationContext(); Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT; if (!deviceSupportsHdrEditing(format)) { @@ -283,17 +265,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { return; } ColorInfo colorInfo = checkNotNull(format.colorInfo); - videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder(testId) - .setInputColorInfo(colorInfo) - .setOutputColorInfo(colorInfo) - .setVideoAssetPath(INPUT_HLG10_MP4_ASSET_STRING) - .setEffects(NO_OP_EFFECT) - .build(); + TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); + VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = + getTexIdProducingFrameProcessorTestRunner( + testId, + consumersBitmapReader, + INPUT_HLG10_MP4_ASSET_STRING, + colorInfo, + ImmutableList.of()); Bitmap expectedBitmap = readBitmap(ORIGINAL_HLG10_PNG_ASSET_PATH); - videoFrameProcessorTestRunner.processFirstFrameAndEnd(); - Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); + texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd(); + texIdProducingVideoFrameProcessorTestRunner.release(); + Bitmap actualBitmap = consumersBitmapReader.getBitmap(); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. float averagePixelAbsoluteDifference = @@ -309,7 +293,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { Context context = getApplicationContext(); Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; if (!deviceSupportsHdrEditing(format)) { - recordTestSkipped(context, testId, "No HLG editing support"); + recordTestSkipped(context, testId, "No PQ editing support"); return; } if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( @@ -318,7 +302,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { } ColorInfo colorInfo = checkNotNull(format.colorInfo); videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder(testId) + getSurfaceInputFrameProcessorTestRunnerBuilder(testId) .setInputColorInfo(colorInfo) .setOutputColorInfo(colorInfo) .setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING) @@ -336,13 +320,47 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); } - // A passthrough effect allows for testing having an intermediate effect injected, which uses - // different OpenGL shaders from having no effects. @Test - public void noOpEffect_hdr10Input_matchesGoldenFile() throws Exception { - String testId = "noOpEffect_hdr10Input_matchesGoldenFile"; + public void noEffects_hdr10TextureInput_matchesGoldenFile() throws Exception { + String testId = "noEffects_hdr10TextureInput_matchesGoldenFile"; Context context = getApplicationContext(); Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; + if (!deviceSupportsHdrEditing(format)) { + recordTestSkipped(context, testId, "No PQ editing support"); + return; + } + if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( + context, testId, /* inputFormat= */ format, /* outputFormat= */ null)) { + return; + } + ColorInfo colorInfo = checkNotNull(format.colorInfo); + TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); + VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = + getTexIdProducingFrameProcessorTestRunner( + testId, + consumersBitmapReader, + INPUT_PQ_MP4_ASSET_STRING, + colorInfo, + ImmutableList.of()); + Bitmap expectedBitmap = readBitmap(ORIGINAL_HDR10_PNG_ASSET_PATH); + + texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd(); + texIdProducingVideoFrameProcessorTestRunner.release(); + Bitmap actualBitmap = consumersBitmapReader.getBitmap(); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16( + expectedBitmap, actualBitmap); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); + } + + @Test + public void noOpEffect_hlg10Input_matchesGoldenFile() throws Exception { + String testId = "noOpEffect_hlg10Input_matchesGoldenFile"; + Context context = getApplicationContext(); + Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT; if (!deviceSupportsHdrEditing(format)) { recordTestSkipped(context, testId, "No HLG editing support"); return; @@ -353,7 +371,77 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { } ColorInfo colorInfo = checkNotNull(format.colorInfo); videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder(testId) + getSurfaceInputFrameProcessorTestRunnerBuilder(testId) + .setInputColorInfo(colorInfo) + .setOutputColorInfo(colorInfo) + .setVideoAssetPath(INPUT_HLG10_MP4_ASSET_STRING) + .setEffects(NO_OP_EFFECT) + .build(); + Bitmap expectedBitmap = readBitmap(ORIGINAL_HLG10_PNG_ASSET_PATH); + + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16( + expectedBitmap, actualBitmap); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); + } + + @Test + public void noOpEffect_hlg10TextureInput_matchesGoldenFile() throws Exception { + String testId = "noOpEffect_hlg10TextureInput_matchesGoldenFile"; + Context context = getApplicationContext(); + Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT; + if (!deviceSupportsHdrEditing(format)) { + recordTestSkipped(context, testId, "No HLG editing support"); + return; + } + if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( + context, testId, /* inputFormat= */ format, /* outputFormat= */ null)) { + return; + } + ColorInfo colorInfo = checkNotNull(format.colorInfo); + TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); + VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = + getTexIdProducingFrameProcessorTestRunner( + testId, + consumersBitmapReader, + INPUT_HLG10_MP4_ASSET_STRING, + colorInfo, + ImmutableList.of(NO_OP_EFFECT)); + Bitmap expectedBitmap = readBitmap(ORIGINAL_HLG10_PNG_ASSET_PATH); + + texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd(); + texIdProducingVideoFrameProcessorTestRunner.release(); + Bitmap actualBitmap = consumersBitmapReader.getBitmap(); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16( + expectedBitmap, actualBitmap); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); + } + + @Test + public void noOpEffect_hdr10Input_matchesGoldenFile() throws Exception { + String testId = "noOpEffect_hdr10Input_matchesGoldenFile"; + Context context = getApplicationContext(); + Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; + if (!deviceSupportsHdrEditing(format)) { + recordTestSkipped(context, testId, "No PQ editing support"); + return; + } + if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( + context, testId, /* inputFormat= */ format, /* outputFormat= */ null)) { + return; + } + ColorInfo colorInfo = checkNotNull(format.colorInfo); + videoFrameProcessorTestRunner = + getSurfaceInputFrameProcessorTestRunnerBuilder(testId) .setInputColorInfo(colorInfo) .setOutputColorInfo(colorInfo) .setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING) @@ -372,9 +460,78 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); } + @Test + public void noOpEffect_hdr10TextureInput_matchesGoldenFile() throws Exception { + String testId = "noOpEffect_hdr10TextureInput_matchesGoldenFile"; + Context context = getApplicationContext(); + Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; + if (!deviceSupportsHdrEditing(format)) { + recordTestSkipped(context, testId, "No PQ editing support"); + return; + } + if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( + context, testId, /* inputFormat= */ format, /* outputFormat= */ null)) { + return; + } + ColorInfo colorInfo = checkNotNull(format.colorInfo); + TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); + VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = + getTexIdProducingFrameProcessorTestRunner( + testId, + consumersBitmapReader, + INPUT_PQ_MP4_ASSET_STRING, + colorInfo, + ImmutableList.of(NO_OP_EFFECT)); + Bitmap expectedBitmap = readBitmap(ORIGINAL_HDR10_PNG_ASSET_PATH); + + texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd(); + texIdProducingVideoFrameProcessorTestRunner.release(); + Bitmap actualBitmap = consumersBitmapReader.getBitmap(); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16( + expectedBitmap, actualBitmap); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); + } + + private VideoFrameProcessorTestRunner getTexIdProducingFrameProcessorTestRunner( + String testId, + TextureBitmapReader consumersBitmapReader, + String videoAssetPath, + ColorInfo colorInfo, + List effects) + throws VideoFrameProcessingException { + TextureBitmapReader producersBitmapReader = new TextureBitmapReader(); + DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = + new DefaultVideoFrameProcessor.Factory.Builder() + .setTextureOutput( + (outputTexture, presentationTimeUs) -> + inputTextureIntoVideoFrameProcessor( + testId, + consumersBitmapReader, + colorInfo, + effects, + outputTexture, + presentationTimeUs), + /* textureOutputCapacity= */ 1) + .build(); + return new VideoFrameProcessorTestRunner.Builder() + .setTestId(testId) + .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) + .setVideoAssetPath(videoAssetPath) + .setInputColorInfo(colorInfo) + .setOutputColorInfo(colorInfo) + .setBitmapReader(producersBitmapReader) + .build(); + } + private void inputTextureIntoVideoFrameProcessor( String testId, TextureBitmapReader bitmapReader, + ColorInfo colorInfo, + List effects, GlTextureInfo texture, long presentationTimeUs) throws VideoFrameProcessingException { @@ -389,9 +546,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { new VideoFrameProcessorTestRunner.Builder() .setTestId(testId) .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) - .setVideoAssetPath(INPUT_SDR_MP4_ASSET_STRING) + .setInputColorInfo(colorInfo) + .setOutputColorInfo(colorInfo) .setBitmapReader(bitmapReader) .setInputType(VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID) + .setEffects(effects) .build(); videoFrameProcessorTestRunner.queueInputTexture(texture, presentationTimeUs); @@ -402,7 +561,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { } } - private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder( + private VideoFrameProcessorTestRunner.Builder getSurfaceInputFrameProcessorTestRunnerBuilder( String testId) { TextureBitmapReader textureBitmapReader = new TextureBitmapReader(); DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =