loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDR: Add PQ support.

Browse Source 
Use the PQ OETF and EOTF to ensure that intermediate fragment shader operations
using PQ are in linear BT.2020 rather than PQ and HLG-1 BT.2020.

Also, swap the OETF and EOTF in shaders, as they were used incorrectly before

Manually tested by verifying transformer demo HLG and PQ videos look the same with and without this CL, including with a BitmapOverlayProcessor enabled to test flows both with one MatrixTransformationProcessor that skips HDR TFs, and with one that doesn't.

PiperOrigin-RevId: 469736067
This commit is contained in:
huangdarwin 2022-08-24 15:44:22 +00:00 committed by Marc Baechinger
parent b9bcf5224f
commit 2ad07e88ee
8 changed files with 239 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java
View File

@ -183,12 +183,17 @@ public final class GlProgram {
checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, texUnitIndex);
}
/** Sets a float type uniform. */
/** Sets an {@code int} type uniform. */
public void setIntUniform(String name, int value) {
checkNotNull(uniformByName.get(name)).setInt(value);
}
/** Sets a {@code float} type uniform. */
public void setFloatUniform(String name, float value) {
checkNotNull(uniformByName.get(name)).setFloat(value);
}
/** Sets a float array type uniform. */
/** Sets a {@code float[]} type uniform. */
public void setFloatsUniform(String name, float[] value) {
checkNotNull(uniformByName.get(name)).setFloats(value);
}
@ -322,16 +327,17 @@ public final class GlProgram {
private final int location;
private final int type;
private final float[] value;
private final float[] floatValue;
private int texId;
private int intValue;
private int texIdValue;
private int texUnitIndex;
private Uniform(String name, int location, int type) {
this.name = name;
this.location = location;
this.type = type;
this.value = new float[16];
this.floatValue = new float[16];
}
/**
@ -341,18 +347,22 @@ public final class GlProgram {
* @param texUnitIndex The GL texture unit index.
*/
public void setSamplerTexId(int texId, int texUnitIndex) {
this.texId = texId;
this.texIdValue = texId;
this.texUnitIndex = texUnitIndex;
}
/** Configures {@link #bind()} to use the specified float {@code value} for this uniform. */
public void setFloat(float value) {
this.value[0] = value;
/** Configures {@link #bind()} to use the specified {@code int} {@code value}. */
public void setInt(int value) {
this.intValue = value;
}
/** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */
/** Configures {@link #bind()} to use the specified {@code float} {@code value}. */
public void setFloat(float value) {
this.floatValue[0] = value;
}
/** Configures {@link #bind()} to use the specified {@code float[]} {@code value}. */
public void setFloats(float[] value) {
System.arraycopy(value, /* srcPos= */ 0, this.value, /* destPos= */ 0, value.length);
System.arraycopy(value, /* srcPos= */ 0, this.floatValue, /* destPos= */ 0, value.length);
}
/**
@ -363,32 +373,35 @@ public final class GlProgram {
*/
public void bind() throws GlUtil.GlException {
switch (type) {
case GLES20.GL_INT:
GLES20.glUniform1i(location, intValue);
break;
case GLES20.GL_FLOAT:
GLES20.glUniform1fv(location, /* count= */ 1, value, /* offset= */ 0);
GLES20.glUniform1fv(location, /* count= */ 1, floatValue, /* offset= */ 0);
GlUtil.checkGlError();
break;
case GLES20.GL_FLOAT_VEC2:
GLES20.glUniform2fv(location, /* count= */ 1, value, /* offset= */ 0);
GLES20.glUniform2fv(location, /* count= */ 1, floatValue, /* offset= */ 0);
GlUtil.checkGlError();
break;
case GLES20.GL_FLOAT_VEC3:
GLES20.glUniform3fv(location, /* count= */ 1, value, /* offset= */ 0);
GLES20.glUniform3fv(location, /* count= */ 1, floatValue, /* offset= */ 0);
GlUtil.checkGlError();
break;
case GLES20.GL_FLOAT_MAT3:
GLES20.glUniformMatrix3fv(
location, /* count= */ 1, /* transpose= */ false, value, /* offset= */ 0);
location, /* count= */ 1, /* transpose= */ false, floatValue, /* offset= */ 0);
GlUtil.checkGlError();
break;
case GLES20.GL_FLOAT_MAT4:
GLES20.glUniformMatrix4fv(
location, /* count= */ 1, /* transpose= */ false, value, /* offset= */ 0);
location, /* count= */ 1, /* transpose= */ false, floatValue, /* offset= */ 0);
GlUtil.checkGlError();
break;
case GLES20.GL_SAMPLER_2D:
case GLES11Ext.GL_SAMPLER_EXTERNAL_OES:
case GL_SAMPLER_EXTERNAL_2D_Y2Y_EXT:
if (texId == 0) {
if (texIdValue == 0) {
throw new IllegalStateException("No call to setSamplerTexId() before bind.");
}
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + texUnitIndex);
@ -397,7 +410,7 @@ public final class GlProgram {
type == GLES20.GL_SAMPLER_2D
? GLES20.GL_TEXTURE_2D
: GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
texId);
texIdValue);
GLES20.glUniform1i(location, texUnitIndex);
GlUtil.checkGlError();
break;

85
libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl
View File

@ -14,61 +14,92 @@
// limitations under the License.
// ES 3 fragment shader that:
// 1. samples HLG BT.2020 YUV from an external texture with uTexSampler, where
// the sampler uses the EXT_YUV_target extension specified at
// 1. Samples electrical (HLG or PQ) BT.2020 YUV from an external texture with
// uTexSampler, where the sampler uses the EXT_YUV_target extension specified
// at
// https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt,
// 2. Applies a YUV to RGB conversion using the specified color transform
// uYuvToRgbColorTransform, yielding HLG BT.2020 RGB,
// 3. If uApplyHlgOetf is 1, outputs HLG BT.2020 RGB. If 0, outputs
// linear BT.2020 RGB for intermediate shaders by applying the HLG OETF.
// 4. Copies this converted texture color to the current output.
// uYuvToRgbColorTransform, yielding electrical (HLG or PQ) BT.2020 RGB,
// 3. If uEotfGlColorTransfer is GL_COLOR_TRANSFER_NO_VALUE, outputs electrical
// (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for
// intermediate shaders by applying the HLG or PQ EOTF.
// 4. Copies this converted texture color to the current output, with alpha = 1.
#extension GL_OES_EGL_image_external : require
#extension GL_EXT_YUV_target : require
precision mediump float;
uniform __samplerExternal2DY2YEXT uTexSampler;
uniform mat3 uYuvToRgbColorTransform;
uniform float uApplyHlgOetf;
// C.java#GlColorTransfer value.
uniform int uEotfGlColorTransfer;
in vec2 vTexSamplingCoord;
out vec4 outColor;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization errors.
// HLG OETF for one channel.
highp float hlgOetfSingleChannel(highp float hlgChannel) {
// HLG EOTF for one channel.
highp float hlgEotfSingleChannel(highp float hlgChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return hlgChannel <= 1.0 / 12.0 ? sqrt(3.0 * hlgChannel) :
a * log(12.0 * hlgChannel - b) + c;
return hlgChannel <= 0.5 ? hlgChannel * hlgChannel / 3.0 :
(b + exp((hlgChannel - c) / a)) / 12.0;
}
// BT.2100-0 HLG OETF. Converts nonlinear relative display light to linear
// signal values, both normalized to [0, 1].
highp vec4 hlgOetf(highp vec4 hlgColor) {
return vec4(
hlgOetfSingleChannel(hlgColor.r),
hlgOetfSingleChannel(hlgColor.g),
hlgOetfSingleChannel(hlgColor.b),
hlgColor.a
// BT.2100 / BT.2020 HLG EOTF.
highp vec3 hlgEotf(highp vec3 hlgColor) {
return vec3(
hlgEotfSingleChannel(hlgColor.r),
hlgEotfSingleChannel(hlgColor.g),
hlgEotfSingleChannel(hlgColor.b)
);
}
/** Convert YUV to RGBA. */
vec4 yuvToRgba(vec3 yuv) {
// BT.2100 / BT.2020 PQ EOTF.
highp vec3 pqEotf(highp vec3 pqColor) {
// Specification:
// https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=250-263;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float m1 = (2610.0 / 16384.0);
const highp float m2 = (2523.0 / 4096.0) * 128.0;
const highp float c1 = (3424.0 / 4096.0);
const highp float c2 = (2413.0 / 4096.0) * 32.0;
const highp float c3 = (2392.0 / 4096.0) * 32.0;
highp vec3 temp = pow(clamp(pqColor, 0.0, 1.0), 1.0 / vec3(m2));
temp = max(temp - c1, 0.0) / (c2 - c3 * temp);
return pow(temp, 1.0 / vec3(m1));
}
// Applies the appropriate EOTF to convert nonlinear electrical values to linear
// optical values. Input and output are both normalized to [0, 1].
highp vec3 getOpticalColor(highp vec3 electricalColor) {
// LINT.IfChange(color_transfer)
const int COLOR_TRANSFER_ST2084 = 6;
const int COLOR_TRANSFER_HLG = 7;
if (uEotfGlColorTransfer == COLOR_TRANSFER_ST2084) {
return pqEotf(electricalColor);
} else if (uEotfGlColorTransfer == COLOR_TRANSFER_HLG) {
return hlgEotf(electricalColor);
} else {
return electricalColor;
}
}
vec3 yuvToRgb(vec3 yuv) {
const vec3 yuvOffset = vec3(0.0625, 0.5, 0.5);
vec3 rgb = clamp(uYuvToRgbColorTransform * (yuv - yuvOffset), 0.0, 1.0);
return vec4(rgb, 1.0);
return clamp(uYuvToRgbColorTransform * (yuv - yuvOffset), 0.0, 1.0);
}
void main() {
vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
outColor = yuvToRgba(srcYuv);
outColor = (uApplyHlgOetf == 1.0) ? hlgOetf(outColor) : outColor;
vec3 rgb = yuvToRgb(srcYuv);
outColor = vec4(getOpticalColor(rgb), 1.0);
}

55
libraries/effect/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl
View File

@ -1,55 +0,0 @@
#version 300 es
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 fragment shader that:
// 1. samples linear BT.2020 RGB from a (non-external) texture with uTexSampler,
// 2. applies the HLG OETF to yield HLG BT.2020 RGB, and
// 3. copies this converted texture color to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
in vec2 vTexSamplingCoord;
out vec4 outColor;
uniform mat3 uColorTransform;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization.
// HLG OETF for one channel.
highp float hlgEotfSingleChannel(highp float linearChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return linearChannel <= 0.5 ? linearChannel * linearChannel / 3.0 :
(b + exp((linearChannel - c) / a)) / 12.0;
}
// BT.2100-0 HLG EOTF. Converts nonlinear signal values to linear relative
// display light, both normalized to [0,1].
highp vec4 hlgEotf(highp vec4 linearColor) {
return vec4(
hlgEotfSingleChannel(linearColor.r),
hlgEotfSingleChannel(linearColor.g),
hlgEotfSingleChannel(linearColor.b),
linearColor.a
);
}
void main() {
outColor = hlgEotf(texture(uTexSampler, vTexSamplingCoord));
}

87
libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl Normal file
View File

@ -0,0 +1,87 @@
#version 300 es
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 fragment shader that:
// 1. samples optical linear BT.2020 RGB from a (non-external) texture with
// uTexSampler,
// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB,
// and
// 3. copies this converted texture color to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
in vec2 vTexSamplingCoord;
out vec4 outColor;
// C.java#GlColorTransfer value.
// Only GL_COLOR_TRANSFER_ST2084 and GL_COLOR_TRANSFER_HLG are allowed.
uniform int uOetfGlColorTransfer;
uniform mat3 uColorTransform;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization.
// HLG OETF for one channel.
highp float hlgOetfSingleChannel(highp float linearChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return linearChannel <= 1.0 / 12.0 ? sqrt(3.0 * linearChannel) :
a * log(12.0 * linearChannel - b) + c;
}
// BT.2100 / BT.2020 HLG OETF.
highp vec3 hlgOetf(highp vec3 linearColor) {
return vec3(
hlgOetfSingleChannel(linearColor.r),
hlgOetfSingleChannel(linearColor.g),
hlgOetfSingleChannel(linearColor.b)
);
}
// BT.2100 / BT.2020, PQ / ST2084 OETF.
highp vec3 pqOetf(highp vec3 linearColor) {
// Specification:
// https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=514-527;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float m1 = (2610.0 / 16384.0);
const highp float m2 = (2523.0 / 4096.0) * 128.0;
const highp float c1 = (3424.0 / 4096.0);
const highp float c2 = (2413.0 / 4096.0) * 32.0;
const highp float c3 = (2392.0 / 4096.0) * 32.0;
highp vec3 temp = pow(linearColor, vec3(m1));
temp = (c1 + c2 * temp) / (1.0 + c3 * temp);
return pow(temp, vec3(m2));
}
// Applies the appropriate OETF to convert linear optical signals to nonlinear
// electrical signals. Input and output are both normalzied to [0, 1].
highp vec3 getElectricalColor(highp vec3 linearColor) {
// LINT.IfChange(color_transfer)
const int GL_COLOR_TRANSFER_ST2084 = 6;
return (uOetfGlColorTransfer == GL_COLOR_TRANSFER_ST2084) ?
pqOetf(linearColor) : hlgOetf(linearColor);
}
void main() {
vec4 inputColor = texture(uTexSampler, vTexSamplingCoord);
outColor = vec4(getElectricalColor(inputColor.rgb), inputColor.a);
}

2
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
View File

@ -374,7 +374,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
matrixTransformationListBuilder.build(),
sampleFromExternalTexture,
colorInfo,
/* outputOpticalColors= */ true);
/* outputElectricalColors= */ true);
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);

4
libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java
View File

@ -37,10 +37,6 @@ public interface GlEffect extends Effect {
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
*/
// TODO(b/227624622): PQ input files will actually have the incorrect HLG OETF applied, so that
// the intermediate color space will be PQ with the HLG OETF applied. This means intermediate
// GlEffects affecting color will look incorrect on PQ input. Fix this by implementing proper PQ
// OETF / EOTF support.
GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException;
}

4
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
View File

@ -212,7 +212,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
matrixTransformations,
sampleFromExternalTexture,
colorInfo,
/* outputOpticalColors= */ false));
/* outputElectricalColors= */ false));
matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
}
@ -242,7 +242,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
ImmutableList.of(),
sampleFromExternalTexture,
colorInfo,
/* outputOpticalColors= */ false));
/* outputElectricalColors= */ false));
sampleFromExternalTexture = false;
}
textureProcessorListBuilder.add(

85
libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
@ -23,6 +24,7 @@ import android.opengl.Matrix;
import android.util.Pair;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
@ -53,8 +55,8 @@ import java.util.Arrays;
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_HLG_EOTF_ES3_PATH =
"shaders/fragment_shader_hlg_eotf_es3.glsl";
private static final String FRAGMENT_SHADER_OETF_ES3_PATH =
"shaders/fragment_shader_oetf_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
@ -123,9 +125,9 @@ import java.util.Arrays;
this(
createGlProgram(
context,
/* inputOpticalColorsFromExternalTexture= */ false,
/* inputElectricalColorsFromExternalTexture= */ false,
useHdr,
/* outputOpticalColors= */ false),
/* outputElectricalColors= */ false),
ImmutableList.of(matrixTransformation),
useHdr);
}
@ -146,9 +148,9 @@ import java.util.Arrays;
this(
createGlProgram(
context,
/* inputOpticalColorsFromExternalTexture= */ false,
/* inputElectricalColorsFromExternalTexture= */ false,
useHdr,
/* outputOpticalColors= */ false),
/* outputElectricalColors= */ false),
ImmutableList.of(matrixTransformation),
useHdr);
}
@ -156,48 +158,51 @@ import java.util.Arrays;
/**
* Creates a new instance.
*
* <p>Able to convert optical {@link ColorInfo} inputs and outputs to and from the intermediate
* {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma RGB BT.709 for SDR.
* <p>Able to convert nonlinear electrical {@link ColorInfo} inputs and outputs to and from the
* intermediate optical {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma
* RGB BT.709 for SDR.
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* @param inputOpticalColorsFromExternalTexture Whether optical color input will be provided using
* an external texture. If {@code true}, the caller should use {@link
* @param inputElectricalColorsFromExternalTexture Whether electrical color input will be provided
* using an external texture. If {@code true}, the caller should use {@link
* #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with
* the external texture.
* @param opticalColorInfo The optical {@link ColorInfo}, only used to transform between color
* spaces and transfers, when {@code inputOpticalColorsFromExternalTexture} or {@code
* outputOpticalColors} are {@code true}. If it {@link ColorInfo#isTransferHdr(ColorInfo)},
* intermediate {@link GlTextureProcessor} colors will be in linear RGB BT.2020. Otherwise,
* these colors will be in gamma RGB BT.709.
* @param outputOpticalColors If {@code true}, outputs {@code opticalColorInfo}. If {@code false},
* outputs intermediate colors of linear RGB BT.2020 if {@code opticalColorInfo} {@link
* ColorInfo#isTransferHdr(ColorInfo)}, and gamma RGB BT.709 otherwise.
* @param electricalColorInfo The electrical {@link ColorInfo}, only used to transform between
* color spaces and transfers, when {@code inputElectricalColorsFromExternalTexture} or {@code
* outputElectricalColors} are {@code true}. If it is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, intermediate {@link GlTextureProcessor} colors
* will be in linear RGB BT.2020. Otherwise, these colors will be in gamma RGB BT.709.
* @param outputElectricalColors If {@code true}, outputs {@code electricalColorInfo}. If {@code
* false}, outputs intermediate colors of linear RGB BT.2020 if {@code electricalColorInfo} is
* {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 otherwise.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public MatrixTransformationProcessor(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean inputOpticalColorsFromExternalTexture,
ColorInfo opticalColorInfo,
boolean outputOpticalColors)
boolean inputElectricalColorsFromExternalTexture,
ColorInfo electricalColorInfo,
boolean outputElectricalColors)
throws FrameProcessingException {
this(
createGlProgram(
context,
inputOpticalColorsFromExternalTexture,
ColorInfo.isTransferHdr(opticalColorInfo),
outputOpticalColors),
inputElectricalColorsFromExternalTexture,
ColorInfo.isTransferHdr(electricalColorInfo),
outputElectricalColors),
matrixTransformations,
ColorInfo.isTransferHdr(opticalColorInfo));
if (!ColorInfo.isTransferHdr(opticalColorInfo) || !inputOpticalColorsFromExternalTexture) {
ColorInfo.isTransferHdr(electricalColorInfo));
if (!ColorInfo.isTransferHdr(electricalColorInfo)) {
return;
}
// TODO(b/227624622): Implement YUV to RGB conversions in COLOR_RANGE_LIMITED as well, using
// opticalColorInfo.colorRange to select between them.
@C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
checkArgument(
colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
if (inputElectricalColorsFromExternalTexture) {
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
@ -205,16 +210,18 @@ import java.util.Arrays;
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
opticalColorInfo.colorRange == C.COLOR_RANGE_FULL
electricalColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
// TODO(b/227624622): Implement PQ and gamma TFs, and use an @IntDef to select between HLG,
// PQ, and gamma, coming from opticalColorInfo.colorTransfer.
// TODO(b/241902517): Implement gamma transfer functions.
// Applying the OETF will output a linear signal. Not applying the OETF will output an optical
// signal.
glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f);
// If electrical colors are both input and output, no EOTF is needed.
glProgram.setIntUniform(
"uEotfGlColorTransfer", outputElectricalColors ? Format.NO_VALUE : colorTransfer);
} else if (outputElectricalColors) {
glProgram.setIntUniform("uOetfGlColorTransfer", colorTransfer);
}
}
/**
@ -243,14 +250,14 @@ import java.util.Arrays;
private static GlProgram createGlProgram(
Context context,
boolean inputOpticalColorsFromExternalTexture,
boolean inputElectricalColorsFromExternalTexture,
boolean useHdr,
boolean outputOpticalColors)
boolean outputElectricalColors)
throws FrameProcessingException {
String vertexShaderFilePath;
String fragmentShaderFilePath;
if (inputOpticalColorsFromExternalTexture) {
if (inputElectricalColorsFromExternalTexture) {
if (useHdr) {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH;
@ -258,9 +265,9 @@ import java.util.Arrays;
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
}
} else if (outputOpticalColors && useHdr) {
} else if (outputElectricalColors && useHdr) {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_HLG_EOTF_ES3_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_OETF_ES3_PATH;
} else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;