diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl index 31dd05b80a..fe56dc8ee5 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl @@ -14,12 +14,13 @@ // limitations under the License. // ES 3 fragment shader that: -// 1. samples optical linear BT.2020 RGB from a (non-external) texture with -// uTexSampler, and applies a 4x4 RGB color matrix to change the pixel -// colors, -// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB, -// and -// 3. copies this converted texture color to the current output. +// 1. Samples optical linear BT.2020 RGB from a (non-external) texture with +// uTexSampler. +// 2. Applies a 4x4 RGB color matrix to change the pixel colors. +// 3. Applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB, +// based on uOetfColorTransfer. +// 4. Copies this converted texture color to the current output. +// The output will be red if an error has occurred. precision mediump float; uniform sampler2D uTexSampler; @@ -75,8 +76,8 @@ highp vec3 pqOetf(highp vec3 linearColor) { } // Applies the appropriate OETF to convert linear optical signals to nonlinear -// electrical signals. Input and output are both normalzied to [0, 1]. -highp vec3 getElectricalColor(highp vec3 linearColor) { +// electrical signals. Input and output are both normalized to [0, 1]. +highp vec3 applyOetf(highp vec3 linearColor) { // LINT.IfChange(color_transfer) const int COLOR_TRANSFER_ST2084 = 6; const int COLOR_TRANSFER_HLG = 7; @@ -94,5 +95,5 @@ void main() { vec4 inputColor = texture(uTexSampler, vTexSamplingCoord); // transformedColors is an optical color. vec4 transformedColors = uRgbMatrix * vec4(inputColor.rgb, 1); - outColor = vec4(getElectricalColor(transformedColors.rgb), inputColor.a); + outColor = vec4(applyOetf(transformedColors.rgb), inputColor.a); } diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl index f06e6967e1..aaabf57ac4 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl @@ -20,11 +20,13 @@ // https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt, // 2. Applies a YUV to RGB conversion using the specified color transform // uYuvToRgbColorTransform, yielding electrical (HLG or PQ) BT.2020 RGB, -// 3. If uEotfColorTransfer is COLOR_TRANSFER_NO_VALUE, outputs electrical -// (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for -// intermediate shaders by applying the HLG or PQ EOTF. -// 4. Copies this converted texture color to the current output, with alpha = 1, -// while applying a 4x4 RGB color matrix to change the pixel colors. +// 3. Applies an EOTF based on uEotfColorTransfer, yielding optical linear +// BT.2020 RGB. +// 4. Applies a 4x4 RGB color matrix to change the pixel colors. +// 5. If uOetfColorTransfer is COLOR_TRANSFER_LINEAR, outputs linear colors as +// is to intermediate shaders. Otherwise, applies the HLG or PQ OETF, based +// on uOetfColorTransfer, to provide the corresponding output electrical color. +// The output will be red if an error has occurred. #extension GL_OES_EGL_image_external : require #extension GL_EXT_YUV_target : require @@ -33,7 +35,12 @@ uniform __samplerExternal2DY2YEXT uTexSampler; uniform mat3 uYuvToRgbColorTransform; uniform mat4 uRgbMatrix; // C.java#ColorTransfer value. +// Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed. uniform int uEotfColorTransfer; +// C.java#ColorTransfer value. +// Only COLOR_TRANSFER_LINEAR, COLOR_TRANSFER_ST2084, and COLOR_TRANSFER_HLG are +// allowed. +uniform int uOetfColorTransfer; in vec2 vTexSamplingCoord; out vec4 outColor; @@ -81,19 +88,74 @@ highp vec3 pqEotf(highp vec3 pqColor) { // Applies the appropriate EOTF to convert nonlinear electrical values to linear // optical values. Input and output are both normalized to [0, 1]. -highp vec3 getOpticalColor(highp vec3 electricalColor) { +highp vec3 applyEotf(highp vec3 electricalColor) { // LINT.IfChange(color_transfer) const int COLOR_TRANSFER_ST2084 = 6; const int COLOR_TRANSFER_HLG = 7; - const int FORMAT_NO_VALUE = -1; - if (uEotfColorTransfer == COLOR_TRANSFER_ST2084) { return pqEotf(electricalColor); } else if (uEotfColorTransfer == COLOR_TRANSFER_HLG) { return hlgEotf(electricalColor); - } else if (uEotfColorTransfer == FORMAT_NO_VALUE) { - return electricalColor; + } else { + // Output red as an obviously visible error. + return vec3(1.0, 0.0, 0.0); + } +} + +// HLG OETF for one channel. +highp float hlgOetfSingleChannel(highp float linearChannel) { + // Specification: + // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float a = 0.17883277; + const highp float b = 0.28466892; + const highp float c = 0.55991073; + + return linearChannel <= 1.0 / 12.0 ? sqrt(3.0 * linearChannel) : + a * log(12.0 * linearChannel - b) + c; +} + +// BT.2100 / BT.2020 HLG OETF. +highp vec3 hlgOetf(highp vec3 linearColor) { + return vec3( + hlgOetfSingleChannel(linearColor.r), + hlgOetfSingleChannel(linearColor.g), + hlgOetfSingleChannel(linearColor.b) + ); +} + +// BT.2100 / BT.2020, PQ / ST2084 OETF. +highp vec3 pqOetf(highp vec3 linearColor) { + // Specification: + // https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=514-527;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float m1 = (2610.0 / 16384.0); + const highp float m2 = (2523.0 / 4096.0) * 128.0; + const highp float c1 = (3424.0 / 4096.0); + const highp float c2 = (2413.0 / 4096.0) * 32.0; + const highp float c3 = (2392.0 / 4096.0) * 32.0; + + highp vec3 temp = pow(linearColor, vec3(m1)); + temp = (c1 + c2 * temp) / (1.0 + c3 * temp); + return pow(temp, vec3(m2)); +} + +// Applies the appropriate OETF to convert linear optical signals to nonlinear +// electrical signals. Input and output are both normalized to [0, 1]. +highp vec3 applyOetf(highp vec3 linearColor) { + // LINT.IfChange(color_transfer_oetf) + const int COLOR_TRANSFER_LINEAR = 1; + const int COLOR_TRANSFER_ST2084 = 6; + const int COLOR_TRANSFER_HLG = 7; + if(uOetfColorTransfer == COLOR_TRANSFER_ST2084) { + return pqOetf(linearColor); + } else if(uOetfColorTransfer == COLOR_TRANSFER_HLG) { + return hlgOetf(linearColor); + } else if (uOetfColorTransfer == COLOR_TRANSFER_LINEAR) { + return linearColor; } else { // Output red as an obviously visible error. return vec3(1.0, 0.0, 0.0); @@ -107,7 +169,7 @@ vec3 yuvToRgb(vec3 yuv) { void main() { vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz; - vec3 rgb = yuvToRgb(srcYuv); - outColor = uRgbMatrix * vec4(getOpticalColor(rgb), 1.0); - // TODO(b/241902517): Transform optical to electrical colors. + vec4 opticalColor = vec4(applyEotf(yuvToRgb(srcYuv)), 1.0); + vec4 transformedColors = uRgbMatrix * opticalColor; + outColor = vec4(applyOetf(transformedColors.rgb), 1.0); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java index 6f4f7bee74..31293e0182 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java @@ -24,7 +24,6 @@ import android.opengl.Matrix; import android.util.Pair; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; -import androidx.media3.common.Format; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; @@ -154,7 +153,6 @@ import java.util.List; context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH); // No transfer functions needed, because input and output are both optical colors. - // TODO(b/241902517): Add transfer functions since existing color filters may change the colors. return new MatrixTextureProcessor( glProgram, ImmutableList.copyOf(matrixTransformations), @@ -216,6 +214,8 @@ import java.util.List; checkArgument( colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); glProgram.setIntUniform("uEotfColorTransfer", colorTransfer); + // No OETF needed, because the intended output here is optical colors. + glProgram.setIntUniform("uOetfColorTransfer", C.COLOR_TRANSFER_LINEAR); } else { glProgram.setIntUniform("uApplyOetf", 0); } @@ -319,8 +319,11 @@ import java.util.List; ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); - // No transfer functions needed, because the EOTF and OETF cancel out. - glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE); + @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; + checkArgument( + colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); + glProgram.setIntUniform("uEotfColorTransfer", colorTransfer); + glProgram.setIntUniform("uOetfColorTransfer", colorTransfer); } else { glProgram.setIntUniform("uApplyOetf", 1); }