diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java index 0cfed24aec..3bad2334eb 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java @@ -67,7 +67,7 @@ import java.util.Locale; * * @param context The {@link Context}. * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. * @throws FrameProcessingException If a problem occurs while reading shader files. */ public BitmapOverlayProcessor(Context context, boolean useHdr) throws FrameProcessingException { diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java index 941feb3d8f..53fa4d716d 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java @@ -53,7 +53,7 @@ import java.io.IOException; * * @param context The {@link Context}. * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. * @param centerX The x-coordinate of the center of the effect. * @param centerY The y-coordinate of the center of the effect. * @param minInnerRadius The lower bound of the radius that is unaffected by the effect. @@ -71,7 +71,6 @@ import java.io.IOException; float outerRadius) throws FrameProcessingException { super(useHdr); - checkArgument(!useHdr, "PeriodicVignetteProcessor does not support HDR color spaces."); checkArgument(minInnerRadius <= maxInnerRadius); checkArgument(maxInnerRadius <= outerRadius); this.minInnerRadius = minInnerRadius; diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java index 4a0390b1d2..538e377c56 100644 --- a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java @@ -72,7 +72,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * * @param context The {@link Context}. * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. * @param graphName Name of a MediaPipe graph asset to load. * @param inputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph. diff --git a/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl b/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl index 12f59e3c05..9460a4db17 100644 --- a/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl +++ b/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl @@ -13,24 +13,62 @@ // See the License for the specific language governing permissions and // limitations under the License. -// ES 3 fragment shader that samples from an external texture with uTexSampler, -// copying from this texture to the current output while applying the specified -// color transform uColorTransform, which should be a YUV to RGB conversion -// matrix. The sampler uses the using the EXT_YUV_target extension: -// https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt. +// ES 3 fragment shader that: +// 1. samples HLG BT.2020 YUV from an external texture with uTexSampler, where +// the sampler uses the EXT_YUV_target extension specified at +// https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt, +// 2. Applies a YUV to RGB conversion using the specified color transform +// uColorTransform, yielding HLG BT.2020 RGB, +// 3. If uApplyHlgOetf is 1, outputs HLG BT.2020 RGB. If 0, outputs +// linear BT.2020 RGB for intermediate shaders by applying the HLG OETF. +// 4. Copies this converted texture color to the current output. #extension GL_OES_EGL_image_external : require #extension GL_EXT_YUV_target : require precision mediump float; uniform __samplerExternal2DY2YEXT uTexSampler; +// YUV to RGB conversion matrix. uniform mat3 uColorTransform; +uniform float uApplyHlgOetf; in vec2 vTexSamplingCoord; out vec4 outColor; + +// TODO(b/227624622): Consider using mediump to save precision, if it won't lead +// to noticeable quantization errors. + +// HLG OETF for one channel. +highp float hlgOetfSingleChannel(highp float hlgChannel) { + // Specification: + // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float a = 0.17883277; + const highp float b = 0.28466892; + const highp float c = 0.55991073; + + return hlgChannel <= 1.0 / 12.0 ? sqrt(3.0 * hlgChannel) : + a * log(12.0 * hlgChannel - b) + c; +} + + // BT.2100-0 HLG OETF. Converts nonlinear relative display light to linear + // signal values, both normalized to [0, 1]. +highp vec4 hlgOetf(highp vec4 hlgColor) { + return vec4( + hlgOetfSingleChannel(hlgColor.r), + hlgOetfSingleChannel(hlgColor.g), + hlgOetfSingleChannel(hlgColor.b), + hlgColor.a + ); +} + +/** Convert YUV to RGBA. */ +vec4 yuvToRgba(vec3 yuv) { + vec3 yuvOffset = vec3(yuv.x - 0.0625, yuv.y - 0.5, yuv.z - 0.5); + return vec4(uColorTransform * yuvOffset, 1.0); +} + void main() { vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz; - vec3 yuvOffset; - yuvOffset.x = srcYuv.r - 0.0625; - yuvOffset.y = srcYuv.g - 0.5; - yuvOffset.z = srcYuv.b - 0.5; - outColor = vec4(uColorTransform * yuvOffset, 1.0); + outColor = yuvToRgba(srcYuv); + outColor = (uApplyHlgOetf == 1.0) ? hlgOetf(outColor) : outColor; } diff --git a/libraries/transformer/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl b/libraries/transformer/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl new file mode 100644 index 0000000000..247d9e948e --- /dev/null +++ b/libraries/transformer/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl @@ -0,0 +1,55 @@ +#version 300 es +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 3 fragment shader that: +// 1. samples linear BT.2020 RGB from a (non-external) texture with uTexSampler, +// 2. applies the HLG OETF to yield HLG BT.2020 RGB, and +// 3. copies this converted texture color to the current output. + +precision mediump float; +uniform sampler2D uTexSampler; +in vec2 vTexSamplingCoord; +out vec4 outColor; +uniform mat3 uColorTransform; + +// TODO(b/227624622): Consider using mediump to save precision, if it won't lead +// to noticeable quantization. +// HLG OETF for one channel. +highp float hlgEotfSingleChannel(highp float linearChannel) { + // Specification: + // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float a = 0.17883277; + const highp float b = 0.28466892; + const highp float c = 0.55991073; + return linearChannel <= 0.5 ? linearChannel * linearChannel / 3.0 : + (b + exp((linearChannel - c) / a)) / 12.0; +} + +// BT.2100-0 HLG EOTF. Converts nonlinear signal values to linear relative +// display light, both normalized to [0,1]. +highp vec4 hlgEotf(highp vec4 linearColor) { + return vec4( + hlgEotfSingleChannel(linearColor.r), + hlgEotfSingleChannel(linearColor.g), + hlgEotfSingleChannel(linearColor.b), + linearColor.a + ); +} + +void main() { + outColor = hlgEotf(texture(uTexSampler, vTexSamplingCoord)); +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java index 9aeca9d85e..3a844df7a2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java @@ -251,7 +251,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; MatrixTransformationProcessor matrixTransformationProcessor = new MatrixTransformationProcessor( - context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr); + context, + matrixTransformationListBuilder.build(), + sampleFromExternalTexture, + useHdr, + /* outputOpticalColors= */ true); matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight); checkState(outputSize.getWidth() == outputSurfaceInfo.width); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java index 8c4463551c..dde2b52012 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java @@ -33,8 +33,12 @@ public interface GlEffect { * * @param context A {@link Context}. * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. */ + // TODO(b/227624622): PQ input files will actually have the incorrect HLG OETF applied, so that + // the intermediate color space will be PQ with the HLG OETF applied. This means intermediate + // GlEffects affecting color will look incorrect on PQ input. Fix this by implementing proper PQ + // OETF / EOTF support. GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index 3450de3ef2..eed56123ab 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -185,7 +185,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) { textureProcessorListBuilder.add( new MatrixTransformationProcessor( - context, matrixTransformations, sampleFromExternalTexture, useHdr)); + context, + matrixTransformations, + sampleFromExternalTexture, + useHdr, + /* outputOpticalColors= */ false)); matrixTransformationListBuilder = new ImmutableList.Builder<>(); sampleFromExternalTexture = false; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java index cd4cd25b62..197f091d2c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java @@ -50,6 +50,8 @@ import java.util.Arrays; private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH = "shaders/vertex_shader_transformation_es3.glsl"; private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl"; + private static final String FRAGMENT_SHADER_HLG_EOTF_ES3_PATH = + "shaders/fragment_shader_hlg_eotf_es3.glsl"; private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = "shaders/fragment_shader_copy_external_es2.glsl"; private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = @@ -100,7 +102,7 @@ import java.util.Arrays; * * @param context The {@link Context}. * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. * @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation * matrix to use for each frame. * @throws FrameProcessingException If a problem occurs while reading shader files. @@ -112,7 +114,8 @@ import java.util.Arrays; context, ImmutableList.of(matrixTransformation), /* sampleFromExternalTexture= */ false, - useHdr); + useHdr, + /* outputOpticalColors= */ false); } /** @@ -120,7 +123,7 @@ import java.util.Arrays; * * @param context The {@link Context}. * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. * @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation * matrix to use for each frame. * @throws FrameProcessingException If a problem occurs while reading shader files. @@ -132,7 +135,8 @@ import java.util.Arrays; context, ImmutableList.of(matrixTransformation), /* sampleFromExternalTexture= */ false, - useHdr); + useHdr, + /* outputOpticalColors= */ false); } /** @@ -146,6 +150,9 @@ import java.util.Arrays; * provide the transformation matrix associated with the external texture. * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code * EXT_YUV_target} OpenGL extension. + * @param outputOpticalColors If {@code true} and {@code useHdr} is also {@code true}, outputs a + * non-linear optical, or display light colors, possibly by applying the EOTF (Electro-optical + * transfer function). Otherwise, outputs linear electrical colors. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. */ @@ -153,12 +160,13 @@ import java.util.Arrays; Context context, ImmutableList matrixTransformations, boolean sampleFromExternalTexture, - boolean useHdr) + boolean useHdr, + boolean outputOpticalColors) throws FrameProcessingException { super(useHdr); if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) { throw new FrameProcessingException( - "The EXT_YUV_target extension is required for HDR editing."); + "The EXT_YUV_target extension is required for HDR editing input."); } this.matrixTransformations = matrixTransformations; @@ -176,6 +184,11 @@ import java.util.Arrays; useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; fragmentShaderFilePath = useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + } else if (outputOpticalColors) { + vertexShaderFilePath = + useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; + fragmentShaderFilePath = + useHdr ? FRAGMENT_SHADER_HLG_EOTF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH; } else { vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH; @@ -190,6 +203,11 @@ import java.util.Arrays; if (useHdr && sampleFromExternalTexture) { // In HDR editing mode the decoder output is sampled in YUV. glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); + // TODO(b/227624622): Implement PQ, and use an @IntDef to select between HLG, PQ, and no + // transfer function. + // Applying the OETF will output a linear signal. Not applying the OETF will output an optical + // signal. + glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f); } float[] identityMatrix = new float[16]; Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java index c5c3be3bb7..4b2624c8f8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java @@ -46,7 +46,7 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso * Creates a {@code SingleFrameGlTextureProcessor} instance. * * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be - * in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. */ public SingleFrameGlTextureProcessor(boolean useHdr) { this.useHdr = useHdr;