diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java b/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java index 63976b776d..a167c18541 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java @@ -183,12 +183,17 @@ public final class GlProgram { checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, texUnitIndex); } - /** Sets a float type uniform. */ + /** Sets an {@code int} type uniform. */ + public void setIntUniform(String name, int value) { + checkNotNull(uniformByName.get(name)).setInt(value); + } + + /** Sets a {@code float} type uniform. */ public void setFloatUniform(String name, float value) { checkNotNull(uniformByName.get(name)).setFloat(value); } - /** Sets a float array type uniform. */ + /** Sets a {@code float[]} type uniform. */ public void setFloatsUniform(String name, float[] value) { checkNotNull(uniformByName.get(name)).setFloats(value); } @@ -322,16 +327,17 @@ public final class GlProgram { private final int location; private final int type; - private final float[] value; + private final float[] floatValue; - private int texId; + private int intValue; + private int texIdValue; private int texUnitIndex; private Uniform(String name, int location, int type) { this.name = name; this.location = location; this.type = type; - this.value = new float[16]; + this.floatValue = new float[16]; } /** @@ -341,18 +347,22 @@ public final class GlProgram { * @param texUnitIndex The GL texture unit index. */ public void setSamplerTexId(int texId, int texUnitIndex) { - this.texId = texId; + this.texIdValue = texId; this.texUnitIndex = texUnitIndex; } - - /** Configures {@link #bind()} to use the specified float {@code value} for this uniform. */ - public void setFloat(float value) { - this.value[0] = value; + /** Configures {@link #bind()} to use the specified {@code int} {@code value}. */ + public void setInt(int value) { + this.intValue = value; } - /** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */ + /** Configures {@link #bind()} to use the specified {@code float} {@code value}. */ + public void setFloat(float value) { + this.floatValue[0] = value; + } + + /** Configures {@link #bind()} to use the specified {@code float[]} {@code value}. */ public void setFloats(float[] value) { - System.arraycopy(value, /* srcPos= */ 0, this.value, /* destPos= */ 0, value.length); + System.arraycopy(value, /* srcPos= */ 0, this.floatValue, /* destPos= */ 0, value.length); } /** @@ -363,32 +373,35 @@ public final class GlProgram { */ public void bind() throws GlUtil.GlException { switch (type) { + case GLES20.GL_INT: + GLES20.glUniform1i(location, intValue); + break; case GLES20.GL_FLOAT: - GLES20.glUniform1fv(location, /* count= */ 1, value, /* offset= */ 0); + GLES20.glUniform1fv(location, /* count= */ 1, floatValue, /* offset= */ 0); GlUtil.checkGlError(); break; case GLES20.GL_FLOAT_VEC2: - GLES20.glUniform2fv(location, /* count= */ 1, value, /* offset= */ 0); + GLES20.glUniform2fv(location, /* count= */ 1, floatValue, /* offset= */ 0); GlUtil.checkGlError(); break; case GLES20.GL_FLOAT_VEC3: - GLES20.glUniform3fv(location, /* count= */ 1, value, /* offset= */ 0); + GLES20.glUniform3fv(location, /* count= */ 1, floatValue, /* offset= */ 0); GlUtil.checkGlError(); break; case GLES20.GL_FLOAT_MAT3: GLES20.glUniformMatrix3fv( - location, /* count= */ 1, /* transpose= */ false, value, /* offset= */ 0); + location, /* count= */ 1, /* transpose= */ false, floatValue, /* offset= */ 0); GlUtil.checkGlError(); break; case GLES20.GL_FLOAT_MAT4: GLES20.glUniformMatrix4fv( - location, /* count= */ 1, /* transpose= */ false, value, /* offset= */ 0); + location, /* count= */ 1, /* transpose= */ false, floatValue, /* offset= */ 0); GlUtil.checkGlError(); break; case GLES20.GL_SAMPLER_2D: case GLES11Ext.GL_SAMPLER_EXTERNAL_OES: case GL_SAMPLER_EXTERNAL_2D_Y2Y_EXT: - if (texId == 0) { + if (texIdValue == 0) { throw new IllegalStateException("No call to setSamplerTexId() before bind."); } GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + texUnitIndex); @@ -397,7 +410,7 @@ public final class GlProgram { type == GLES20.GL_SAMPLER_2D ? GLES20.GL_TEXTURE_2D : GLES11Ext.GL_TEXTURE_EXTERNAL_OES, - texId); + texIdValue); GLES20.glUniform1i(location, texUnitIndex); GlUtil.checkGlError(); break; diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl index 4e6bfda4fb..89b796a3fb 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl @@ -14,61 +14,92 @@ // limitations under the License. // ES 3 fragment shader that: -// 1. samples HLG BT.2020 YUV from an external texture with uTexSampler, where -// the sampler uses the EXT_YUV_target extension specified at +// 1. Samples electrical (HLG or PQ) BT.2020 YUV from an external texture with +// uTexSampler, where the sampler uses the EXT_YUV_target extension specified +// at // https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt, // 2. Applies a YUV to RGB conversion using the specified color transform -// uYuvToRgbColorTransform, yielding HLG BT.2020 RGB, -// 3. If uApplyHlgOetf is 1, outputs HLG BT.2020 RGB. If 0, outputs -// linear BT.2020 RGB for intermediate shaders by applying the HLG OETF. -// 4. Copies this converted texture color to the current output. +// uYuvToRgbColorTransform, yielding electrical (HLG or PQ) BT.2020 RGB, +// 3. If uEotfGlColorTransfer is GL_COLOR_TRANSFER_NO_VALUE, outputs electrical +// (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for +// intermediate shaders by applying the HLG or PQ EOTF. +// 4. Copies this converted texture color to the current output, with alpha = 1. #extension GL_OES_EGL_image_external : require #extension GL_EXT_YUV_target : require precision mediump float; uniform __samplerExternal2DY2YEXT uTexSampler; uniform mat3 uYuvToRgbColorTransform; -uniform float uApplyHlgOetf; +// C.java#GlColorTransfer value. +uniform int uEotfGlColorTransfer; in vec2 vTexSamplingCoord; out vec4 outColor; // TODO(b/227624622): Consider using mediump to save precision, if it won't lead // to noticeable quantization errors. -// HLG OETF for one channel. -highp float hlgOetfSingleChannel(highp float hlgChannel) { - // Specification: - // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG - // Reference implementation: - // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa +// HLG EOTF for one channel. +highp float hlgEotfSingleChannel(highp float hlgChannel) { + // Specification: + // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa const highp float a = 0.17883277; const highp float b = 0.28466892; const highp float c = 0.55991073; - - return hlgChannel <= 1.0 / 12.0 ? sqrt(3.0 * hlgChannel) : - a * log(12.0 * hlgChannel - b) + c; + return hlgChannel <= 0.5 ? hlgChannel * hlgChannel / 3.0 : + (b + exp((hlgChannel - c) / a)) / 12.0; } - // BT.2100-0 HLG OETF. Converts nonlinear relative display light to linear - // signal values, both normalized to [0, 1]. -highp vec4 hlgOetf(highp vec4 hlgColor) { - return vec4( - hlgOetfSingleChannel(hlgColor.r), - hlgOetfSingleChannel(hlgColor.g), - hlgOetfSingleChannel(hlgColor.b), - hlgColor.a +// BT.2100 / BT.2020 HLG EOTF. +highp vec3 hlgEotf(highp vec3 hlgColor) { + return vec3( + hlgEotfSingleChannel(hlgColor.r), + hlgEotfSingleChannel(hlgColor.g), + hlgEotfSingleChannel(hlgColor.b) ); } -/** Convert YUV to RGBA. */ -vec4 yuvToRgba(vec3 yuv) { +// BT.2100 / BT.2020 PQ EOTF. +highp vec3 pqEotf(highp vec3 pqColor) { + // Specification: + // https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=250-263;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float m1 = (2610.0 / 16384.0); + const highp float m2 = (2523.0 / 4096.0) * 128.0; + const highp float c1 = (3424.0 / 4096.0); + const highp float c2 = (2413.0 / 4096.0) * 32.0; + const highp float c3 = (2392.0 / 4096.0) * 32.0; + + highp vec3 temp = pow(clamp(pqColor, 0.0, 1.0), 1.0 / vec3(m2)); + temp = max(temp - c1, 0.0) / (c2 - c3 * temp); + return pow(temp, 1.0 / vec3(m1)); +} + +// Applies the appropriate EOTF to convert nonlinear electrical values to linear +// optical values. Input and output are both normalized to [0, 1]. +highp vec3 getOpticalColor(highp vec3 electricalColor) { + // LINT.IfChange(color_transfer) + const int COLOR_TRANSFER_ST2084 = 6; + const int COLOR_TRANSFER_HLG = 7; + + if (uEotfGlColorTransfer == COLOR_TRANSFER_ST2084) { + return pqEotf(electricalColor); + } else if (uEotfGlColorTransfer == COLOR_TRANSFER_HLG) { + return hlgEotf(electricalColor); + } else { + return electricalColor; + } +} + +vec3 yuvToRgb(vec3 yuv) { const vec3 yuvOffset = vec3(0.0625, 0.5, 0.5); - vec3 rgb = clamp(uYuvToRgbColorTransform * (yuv - yuvOffset), 0.0, 1.0); - return vec4(rgb, 1.0); + return clamp(uYuvToRgbColorTransform * (yuv - yuvOffset), 0.0, 1.0); } void main() { vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz; - outColor = yuvToRgba(srcYuv); - outColor = (uApplyHlgOetf == 1.0) ? hlgOetf(outColor) : outColor; + vec3 rgb = yuvToRgb(srcYuv); + outColor = vec4(getOpticalColor(rgb), 1.0); } diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl deleted file mode 100644 index 247d9e948e..0000000000 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl +++ /dev/null @@ -1,55 +0,0 @@ -#version 300 es -// Copyright 2022 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// ES 3 fragment shader that: -// 1. samples linear BT.2020 RGB from a (non-external) texture with uTexSampler, -// 2. applies the HLG OETF to yield HLG BT.2020 RGB, and -// 3. copies this converted texture color to the current output. - -precision mediump float; -uniform sampler2D uTexSampler; -in vec2 vTexSamplingCoord; -out vec4 outColor; -uniform mat3 uColorTransform; - -// TODO(b/227624622): Consider using mediump to save precision, if it won't lead -// to noticeable quantization. -// HLG OETF for one channel. -highp float hlgEotfSingleChannel(highp float linearChannel) { - // Specification: - // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG - // Reference implementation: - // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa - const highp float a = 0.17883277; - const highp float b = 0.28466892; - const highp float c = 0.55991073; - return linearChannel <= 0.5 ? linearChannel * linearChannel / 3.0 : - (b + exp((linearChannel - c) / a)) / 12.0; -} - -// BT.2100-0 HLG EOTF. Converts nonlinear signal values to linear relative -// display light, both normalized to [0,1]. -highp vec4 hlgEotf(highp vec4 linearColor) { - return vec4( - hlgEotfSingleChannel(linearColor.r), - hlgEotfSingleChannel(linearColor.g), - hlgEotfSingleChannel(linearColor.b), - linearColor.a - ); -} - -void main() { - outColor = hlgEotf(texture(uTexSampler, vTexSamplingCoord)); -} diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl new file mode 100644 index 0000000000..1c35131ed3 --- /dev/null +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl @@ -0,0 +1,87 @@ +#version 300 es +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 3 fragment shader that: +// 1. samples optical linear BT.2020 RGB from a (non-external) texture with +// uTexSampler, +// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB, +// and +// 3. copies this converted texture color to the current output. + +precision mediump float; +uniform sampler2D uTexSampler; +in vec2 vTexSamplingCoord; +out vec4 outColor; +// C.java#GlColorTransfer value. +// Only GL_COLOR_TRANSFER_ST2084 and GL_COLOR_TRANSFER_HLG are allowed. +uniform int uOetfGlColorTransfer; +uniform mat3 uColorTransform; + +// TODO(b/227624622): Consider using mediump to save precision, if it won't lead +// to noticeable quantization. + +// HLG OETF for one channel. +highp float hlgOetfSingleChannel(highp float linearChannel) { + // Specification: + // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float a = 0.17883277; + const highp float b = 0.28466892; + const highp float c = 0.55991073; + + return linearChannel <= 1.0 / 12.0 ? sqrt(3.0 * linearChannel) : + a * log(12.0 * linearChannel - b) + c; +} + +// BT.2100 / BT.2020 HLG OETF. +highp vec3 hlgOetf(highp vec3 linearColor) { + return vec3( + hlgOetfSingleChannel(linearColor.r), + hlgOetfSingleChannel(linearColor.g), + hlgOetfSingleChannel(linearColor.b) + ); +} + +// BT.2100 / BT.2020, PQ / ST2084 OETF. +highp vec3 pqOetf(highp vec3 linearColor) { + // Specification: + // https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ + // Reference implementation: + // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=514-527;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa + const highp float m1 = (2610.0 / 16384.0); + const highp float m2 = (2523.0 / 4096.0) * 128.0; + const highp float c1 = (3424.0 / 4096.0); + const highp float c2 = (2413.0 / 4096.0) * 32.0; + const highp float c3 = (2392.0 / 4096.0) * 32.0; + + highp vec3 temp = pow(linearColor, vec3(m1)); + temp = (c1 + c2 * temp) / (1.0 + c3 * temp); + return pow(temp, vec3(m2)); +} + +// Applies the appropriate OETF to convert linear optical signals to nonlinear +// electrical signals. Input and output are both normalzied to [0, 1]. +highp vec3 getElectricalColor(highp vec3 linearColor) { + // LINT.IfChange(color_transfer) + const int GL_COLOR_TRANSFER_ST2084 = 6; + return (uOetfGlColorTransfer == GL_COLOR_TRANSFER_ST2084) ? + pqOetf(linearColor) : hlgOetf(linearColor); +} + +void main() { + vec4 inputColor = texture(uTexSampler, vTexSamplingCoord); + outColor = vec4(getElectricalColor(inputColor.rgb), inputColor.a); +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java index eff8b8d18b..24e2a1cf37 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java @@ -374,7 +374,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; matrixTransformationListBuilder.build(), sampleFromExternalTexture, colorInfo, - /* outputOpticalColors= */ true); + /* outputElectricalColors= */ true); matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); Pair outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java index b93da70075..d19956e994 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java @@ -37,10 +37,6 @@ public interface GlEffect extends Effect { * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. */ - // TODO(b/227624622): PQ input files will actually have the incorrect HLG OETF applied, so that - // the intermediate color space will be PQ with the HLG OETF applied. This means intermediate - // GlEffects affecting color will look incorrect on PQ input. Fix this by implementing proper PQ - // OETF / EOTF support. GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java index 324ff05c4e..11bddb4e13 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java @@ -212,7 +212,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { matrixTransformations, sampleFromExternalTexture, colorInfo, - /* outputOpticalColors= */ false)); + /* outputElectricalColors= */ false)); matrixTransformationListBuilder = new ImmutableList.Builder<>(); sampleFromExternalTexture = false; } @@ -242,7 +242,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ImmutableList.of(), sampleFromExternalTexture, colorInfo, - /* outputOpticalColors= */ false)); + /* outputElectricalColors= */ false)); sampleFromExternalTexture = false; } textureProcessorListBuilder.add( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java index 40c98ee2dd..fd458bfdc0 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java @@ -15,6 +15,7 @@ */ package androidx.media3.effect; +import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; @@ -23,6 +24,7 @@ import android.opengl.Matrix; import android.util.Pair; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; +import androidx.media3.common.Format; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; @@ -53,8 +55,8 @@ import java.util.Arrays; private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH = "shaders/vertex_shader_transformation_es3.glsl"; private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl"; - private static final String FRAGMENT_SHADER_HLG_EOTF_ES3_PATH = - "shaders/fragment_shader_hlg_eotf_es3.glsl"; + private static final String FRAGMENT_SHADER_OETF_ES3_PATH = + "shaders/fragment_shader_oetf_es3.glsl"; private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = "shaders/fragment_shader_copy_external_es2.glsl"; private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = @@ -123,9 +125,9 @@ import java.util.Arrays; this( createGlProgram( context, - /* inputOpticalColorsFromExternalTexture= */ false, + /* inputElectricalColorsFromExternalTexture= */ false, useHdr, - /* outputOpticalColors= */ false), + /* outputElectricalColors= */ false), ImmutableList.of(matrixTransformation), useHdr); } @@ -146,9 +148,9 @@ import java.util.Arrays; this( createGlProgram( context, - /* inputOpticalColorsFromExternalTexture= */ false, + /* inputElectricalColorsFromExternalTexture= */ false, useHdr, - /* outputOpticalColors= */ false), + /* outputElectricalColors= */ false), ImmutableList.of(matrixTransformation), useHdr); } @@ -156,65 +158,70 @@ import java.util.Arrays; /** * Creates a new instance. * - *

Able to convert optical {@link ColorInfo} inputs and outputs to and from the intermediate - * {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma RGB BT.709 for SDR. + *

Able to convert nonlinear electrical {@link ColorInfo} inputs and outputs to and from the + * intermediate optical {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma + * RGB BT.709 for SDR. * * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * apply to each frame in order. - * @param inputOpticalColorsFromExternalTexture Whether optical color input will be provided using - * an external texture. If {@code true}, the caller should use {@link + * @param inputElectricalColorsFromExternalTexture Whether electrical color input will be provided + * using an external texture. If {@code true}, the caller should use {@link * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with * the external texture. - * @param opticalColorInfo The optical {@link ColorInfo}, only used to transform between color - * spaces and transfers, when {@code inputOpticalColorsFromExternalTexture} or {@code - * outputOpticalColors} are {@code true}. If it {@link ColorInfo#isTransferHdr(ColorInfo)}, - * intermediate {@link GlTextureProcessor} colors will be in linear RGB BT.2020. Otherwise, - * these colors will be in gamma RGB BT.709. - * @param outputOpticalColors If {@code true}, outputs {@code opticalColorInfo}. If {@code false}, - * outputs intermediate colors of linear RGB BT.2020 if {@code opticalColorInfo} {@link - * ColorInfo#isTransferHdr(ColorInfo)}, and gamma RGB BT.709 otherwise. + * @param electricalColorInfo The electrical {@link ColorInfo}, only used to transform between + * color spaces and transfers, when {@code inputElectricalColorsFromExternalTexture} or {@code + * outputElectricalColors} are {@code true}. If it is {@linkplain + * ColorInfo#isTransferHdr(ColorInfo) HDR}, intermediate {@link GlTextureProcessor} colors + * will be in linear RGB BT.2020. Otherwise, these colors will be in gamma RGB BT.709. + * @param outputElectricalColors If {@code true}, outputs {@code electricalColorInfo}. If {@code + * false}, outputs intermediate colors of linear RGB BT.2020 if {@code electricalColorInfo} is + * {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 otherwise. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. */ public MatrixTransformationProcessor( Context context, ImmutableList matrixTransformations, - boolean inputOpticalColorsFromExternalTexture, - ColorInfo opticalColorInfo, - boolean outputOpticalColors) + boolean inputElectricalColorsFromExternalTexture, + ColorInfo electricalColorInfo, + boolean outputElectricalColors) throws FrameProcessingException { this( createGlProgram( context, - inputOpticalColorsFromExternalTexture, - ColorInfo.isTransferHdr(opticalColorInfo), - outputOpticalColors), + inputElectricalColorsFromExternalTexture, + ColorInfo.isTransferHdr(electricalColorInfo), + outputElectricalColors), matrixTransformations, - ColorInfo.isTransferHdr(opticalColorInfo)); - if (!ColorInfo.isTransferHdr(opticalColorInfo) || !inputOpticalColorsFromExternalTexture) { + ColorInfo.isTransferHdr(electricalColorInfo)); + if (!ColorInfo.isTransferHdr(electricalColorInfo)) { return; } - // TODO(b/227624622): Implement YUV to RGB conversions in COLOR_RANGE_LIMITED as well, using - // opticalColorInfo.colorRange to select between them. - // In HDR editing mode the decoder output is sampled in YUV. - if (!GlUtil.isYuvTargetExtensionSupported()) { - throw new FrameProcessingException( - "The EXT_YUV_target extension is required for HDR editing input."); + @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; + checkArgument( + colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); + if (inputElectricalColorsFromExternalTexture) { + // In HDR editing mode the decoder output is sampled in YUV. + if (!GlUtil.isYuvTargetExtensionSupported()) { + throw new FrameProcessingException( + "The EXT_YUV_target extension is required for HDR editing input."); + } + glProgram.setFloatsUniform( + "uYuvToRgbColorTransform", + electricalColorInfo.colorRange == C.COLOR_RANGE_FULL + ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX + : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); + + // TODO(b/241902517): Implement gamma transfer functions. + + // If electrical colors are both input and output, no EOTF is needed. + glProgram.setIntUniform( + "uEotfGlColorTransfer", outputElectricalColors ? Format.NO_VALUE : colorTransfer); + } else if (outputElectricalColors) { + glProgram.setIntUniform("uOetfGlColorTransfer", colorTransfer); } - glProgram.setFloatsUniform( - "uYuvToRgbColorTransform", - opticalColorInfo.colorRange == C.COLOR_RANGE_FULL - ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX - : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); - - // TODO(b/227624622): Implement PQ and gamma TFs, and use an @IntDef to select between HLG, - // PQ, and gamma, coming from opticalColorInfo.colorTransfer. - - // Applying the OETF will output a linear signal. Not applying the OETF will output an optical - // signal. - glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f); } /** @@ -243,14 +250,14 @@ import java.util.Arrays; private static GlProgram createGlProgram( Context context, - boolean inputOpticalColorsFromExternalTexture, + boolean inputElectricalColorsFromExternalTexture, boolean useHdr, - boolean outputOpticalColors) + boolean outputElectricalColors) throws FrameProcessingException { String vertexShaderFilePath; String fragmentShaderFilePath; - if (inputOpticalColorsFromExternalTexture) { + if (inputElectricalColorsFromExternalTexture) { if (useHdr) { vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH; @@ -258,9 +265,9 @@ import java.util.Arrays; vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_PATH; } - } else if (outputOpticalColors && useHdr) { + } else if (outputElectricalColors && useHdr) { vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH; - fragmentShaderFilePath = FRAGMENT_SHADER_HLG_EOTF_ES3_PATH; + fragmentShaderFilePath = FRAGMENT_SHADER_OETF_ES3_PATH; } else { vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;