loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDR: Add PQ support.

Browse Source 
Use the PQ OETF and EOTF to ensure that intermediate fragment shader operations
using PQ are in linear BT.2020 rather than PQ and HLG-1 BT.2020.

Also, swap the OETF and EOTF in shaders, as they were used incorrectly before

Manually tested by verifying transformer demo HLG and PQ videos look the same with and without this CL, including with a BitmapOverlayProcessor enabled to test flows both with one MatrixTransformationProcessor that skips HDR TFs, and with one that doesn't.

PiperOrigin-RevId: 469736067
This commit is contained in:
huangdarwin 2022-08-24 15:44:22 +00:00 committed by Marc Baechinger
parent b9bcf5224f
commit 2ad07e88ee
8 changed files with 239 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java
View File

@ -183,12 +183,17 @@ public final class GlProgram {
checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, texUnitIndex); checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, texUnitIndex);
} }
/** Sets a float type uniform. */ /** Sets an {@code int} type uniform. */
public void setIntUniform(String name, int value) {
checkNotNull(uniformByName.get(name)).setInt(value);
}
/** Sets a {@code float} type uniform. */
public void setFloatUniform(String name, float value) { public void setFloatUniform(String name, float value) {
checkNotNull(uniformByName.get(name)).setFloat(value); checkNotNull(uniformByName.get(name)).setFloat(value);
} }
/** Sets a float array type uniform. */ /** Sets a {@code float[]} type uniform. */
public void setFloatsUniform(String name, float[] value) { public void setFloatsUniform(String name, float[] value) {
checkNotNull(uniformByName.get(name)).setFloats(value); checkNotNull(uniformByName.get(name)).setFloats(value);
} }
@ -322,16 +327,17 @@ public final class GlProgram {
private final int location; private final int location;
private final int type; private final int type;
private final float[] value; private final float[] floatValue;
private int texId; private int intValue;
private int texIdValue;
private int texUnitIndex; private int texUnitIndex;
private Uniform(String name, int location, int type) { private Uniform(String name, int location, int type) {
this.name = name; this.name = name;
this.location = location; this.location = location;
this.type = type; this.type = type;
this.value = new float[16]; this.floatValue = new float[16];
} }
/** /**
@ -341,18 +347,22 @@ public final class GlProgram {
* @param texUnitIndex The GL texture unit index. * @param texUnitIndex The GL texture unit index.
*/ */
public void setSamplerTexId(int texId, int texUnitIndex) { public void setSamplerTexId(int texId, int texUnitIndex) {
this.texId = texId; this.texIdValue = texId;
this.texUnitIndex = texUnitIndex; this.texUnitIndex = texUnitIndex;
} }
/** Configures {@link #bind()} to use the specified {@code int} {@code value}. */
/** Configures {@link #bind()} to use the specified float {@code value} for this uniform. */ public void setInt(int value) {
public void setFloat(float value) { this.intValue = value;
this.value[0] = value;
} }
/** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */ /** Configures {@link #bind()} to use the specified {@code float} {@code value}. */
public void setFloat(float value) {
this.floatValue[0] = value;
}
/** Configures {@link #bind()} to use the specified {@code float[]} {@code value}. */
public void setFloats(float[] value) { public void setFloats(float[] value) {
System.arraycopy(value, /* srcPos= */ 0, this.value, /* destPos= */ 0, value.length); System.arraycopy(value, /* srcPos= */ 0, this.floatValue, /* destPos= */ 0, value.length);
} }
/** /**
@ -363,32 +373,35 @@ public final class GlProgram {
*/ */
public void bind() throws GlUtil.GlException { public void bind() throws GlUtil.GlException {
switch (type) { switch (type) {
case GLES20.GL_INT:
GLES20.glUniform1i(location, intValue);
break;
case GLES20.GL_FLOAT: case GLES20.GL_FLOAT:
GLES20.glUniform1fv(location, /* count= */ 1, value, /* offset= */ 0); GLES20.glUniform1fv(location, /* count= */ 1, floatValue, /* offset= */ 0);
GlUtil.checkGlError(); GlUtil.checkGlError();
break; break;
case GLES20.GL_FLOAT_VEC2: case GLES20.GL_FLOAT_VEC2:
GLES20.glUniform2fv(location, /* count= */ 1, value, /* offset= */ 0); GLES20.glUniform2fv(location, /* count= */ 1, floatValue, /* offset= */ 0);
GlUtil.checkGlError(); GlUtil.checkGlError();
break; break;
case GLES20.GL_FLOAT_VEC3: case GLES20.GL_FLOAT_VEC3:
GLES20.glUniform3fv(location, /* count= */ 1, value, /* offset= */ 0); GLES20.glUniform3fv(location, /* count= */ 1, floatValue, /* offset= */ 0);
GlUtil.checkGlError(); GlUtil.checkGlError();
break; break;
case GLES20.GL_FLOAT_MAT3: case GLES20.GL_FLOAT_MAT3:
GLES20.glUniformMatrix3fv( GLES20.glUniformMatrix3fv(
location, /* count= */ 1, /* transpose= */ false, value, /* offset= */ 0); location, /* count= */ 1, /* transpose= */ false, floatValue, /* offset= */ 0);
GlUtil.checkGlError(); GlUtil.checkGlError();
break; break;
case GLES20.GL_FLOAT_MAT4: case GLES20.GL_FLOAT_MAT4:
GLES20.glUniformMatrix4fv( GLES20.glUniformMatrix4fv(
location, /* count= */ 1, /* transpose= */ false, value, /* offset= */ 0); location, /* count= */ 1, /* transpose= */ false, floatValue, /* offset= */ 0);
GlUtil.checkGlError(); GlUtil.checkGlError();
break; break;
case GLES20.GL_SAMPLER_2D: case GLES20.GL_SAMPLER_2D:
case GLES11Ext.GL_SAMPLER_EXTERNAL_OES: case GLES11Ext.GL_SAMPLER_EXTERNAL_OES:
case GL_SAMPLER_EXTERNAL_2D_Y2Y_EXT: case GL_SAMPLER_EXTERNAL_2D_Y2Y_EXT:
if (texId == 0) { if (texIdValue == 0) {
throw new IllegalStateException("No call to setSamplerTexId() before bind."); throw new IllegalStateException("No call to setSamplerTexId() before bind.");
} }
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + texUnitIndex); GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + texUnitIndex);
@ -397,7 +410,7 @@ public final class GlProgram {
type == GLES20.GL_SAMPLER_2D type == GLES20.GL_SAMPLER_2D
? GLES20.GL_TEXTURE_2D ? GLES20.GL_TEXTURE_2D
: GLES11Ext.GL_TEXTURE_EXTERNAL_OES, : GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
texId); texIdValue);
GLES20.glUniform1i(location, texUnitIndex); GLES20.glUniform1i(location, texUnitIndex);
GlUtil.checkGlError(); GlUtil.checkGlError();
break; break;

91
libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl
View File

@ -14,61 +14,92 @@
// limitations under the License. // limitations under the License.
// ES 3 fragment shader that: // ES 3 fragment shader that:
// 1. samples HLG BT.2020 YUV from an external texture with uTexSampler, where // 1. Samples electrical (HLG or PQ) BT.2020 YUV from an external texture with
// the sampler uses the EXT_YUV_target extension specified at // uTexSampler, where the sampler uses the EXT_YUV_target extension specified
// at
// https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt, // https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt,
// 2. Applies a YUV to RGB conversion using the specified color transform // 2. Applies a YUV to RGB conversion using the specified color transform
// uYuvToRgbColorTransform, yielding HLG BT.2020 RGB, // uYuvToRgbColorTransform, yielding electrical (HLG or PQ) BT.2020 RGB,
// 3. If uApplyHlgOetf is 1, outputs HLG BT.2020 RGB. If 0, outputs // 3. If uEotfGlColorTransfer is GL_COLOR_TRANSFER_NO_VALUE, outputs electrical
// linear BT.2020 RGB for intermediate shaders by applying the HLG OETF. // (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for
// 4. Copies this converted texture color to the current output. // intermediate shaders by applying the HLG or PQ EOTF.
// 4. Copies this converted texture color to the current output, with alpha = 1.
#extension GL_OES_EGL_image_external : require #extension GL_OES_EGL_image_external : require
#extension GL_EXT_YUV_target : require #extension GL_EXT_YUV_target : require
precision mediump float; precision mediump float;
uniform __samplerExternal2DY2YEXT uTexSampler; uniform __samplerExternal2DY2YEXT uTexSampler;
uniform mat3 uYuvToRgbColorTransform; uniform mat3 uYuvToRgbColorTransform;
uniform float uApplyHlgOetf; // C.java#GlColorTransfer value.
uniform int uEotfGlColorTransfer;
in vec2 vTexSamplingCoord; in vec2 vTexSamplingCoord;
out vec4 outColor; out vec4 outColor;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead // TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization errors. // to noticeable quantization errors.
// HLG OETF for one channel. // HLG EOTF for one channel.
highp float hlgOetfSingleChannel(highp float hlgChannel) { highp float hlgEotfSingleChannel(highp float hlgChannel) {
// Specification: // Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation: // Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277; const highp float a = 0.17883277;
const highp float b = 0.28466892; const highp float b = 0.28466892;
const highp float c = 0.55991073; const highp float c = 0.55991073;
return hlgChannel <= 0.5 ? hlgChannel * hlgChannel / 3.0 :
return hlgChannel <= 1.0 / 12.0 ? sqrt(3.0 * hlgChannel) : (b + exp((hlgChannel - c) / a)) / 12.0;
a * log(12.0 * hlgChannel - b) + c;
} }
// BT.2100-0 HLG OETF. Converts nonlinear relative display light to linear // BT.2100 / BT.2020 HLG EOTF.
// signal values, both normalized to [0, 1]. highp vec3 hlgEotf(highp vec3 hlgColor) {
highp vec4 hlgOetf(highp vec4 hlgColor) { return vec3(
return vec4( hlgEotfSingleChannel(hlgColor.r),
hlgOetfSingleChannel(hlgColor.r), hlgEotfSingleChannel(hlgColor.g),
hlgOetfSingleChannel(hlgColor.g), hlgEotfSingleChannel(hlgColor.b)
hlgOetfSingleChannel(hlgColor.b),
hlgColor.a
); );
} }
/** Convert YUV to RGBA. */ // BT.2100 / BT.2020 PQ EOTF.
vec4 yuvToRgba(vec3 yuv) { highp vec3 pqEotf(highp vec3 pqColor) {
// Specification:
// https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=250-263;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float m1 = (2610.0 / 16384.0);
const highp float m2 = (2523.0 / 4096.0) * 128.0;
const highp float c1 = (3424.0 / 4096.0);
const highp float c2 = (2413.0 / 4096.0) * 32.0;
const highp float c3 = (2392.0 / 4096.0) * 32.0;
highp vec3 temp = pow(clamp(pqColor, 0.0, 1.0), 1.0 / vec3(m2));
temp = max(temp - c1, 0.0) / (c2 - c3 * temp);
return pow(temp, 1.0 / vec3(m1));
}
// Applies the appropriate EOTF to convert nonlinear electrical values to linear
// optical values. Input and output are both normalized to [0, 1].
highp vec3 getOpticalColor(highp vec3 electricalColor) {
// LINT.IfChange(color_transfer)
const int COLOR_TRANSFER_ST2084 = 6;
const int COLOR_TRANSFER_HLG = 7;
if (uEotfGlColorTransfer == COLOR_TRANSFER_ST2084) {
return pqEotf(electricalColor);
} else if (uEotfGlColorTransfer == COLOR_TRANSFER_HLG) {
return hlgEotf(electricalColor);
} else {
return electricalColor;
}
}
vec3 yuvToRgb(vec3 yuv) {
const vec3 yuvOffset = vec3(0.0625, 0.5, 0.5); const vec3 yuvOffset = vec3(0.0625, 0.5, 0.5);
vec3 rgb = clamp(uYuvToRgbColorTransform * (yuv - yuvOffset), 0.0, 1.0); return clamp(uYuvToRgbColorTransform * (yuv - yuvOffset), 0.0, 1.0);
return vec4(rgb, 1.0);
} }
void main() { void main() {
vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz; vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
outColor = yuvToRgba(srcYuv); vec3 rgb = yuvToRgb(srcYuv);
outColor = (uApplyHlgOetf == 1.0) ? hlgOetf(outColor) : outColor; outColor = vec4(getOpticalColor(rgb), 1.0);
} }

55
libraries/effect/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl
View File

@ -1,55 +0,0 @@
#version 300 es
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 fragment shader that:
// 1. samples linear BT.2020 RGB from a (non-external) texture with uTexSampler,
// 2. applies the HLG OETF to yield HLG BT.2020 RGB, and
// 3. copies this converted texture color to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
in vec2 vTexSamplingCoord;
out vec4 outColor;
uniform mat3 uColorTransform;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization.
// HLG OETF for one channel.
highp float hlgEotfSingleChannel(highp float linearChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return linearChannel <= 0.5 ? linearChannel * linearChannel / 3.0 :
(b + exp((linearChannel - c) / a)) / 12.0;
}
// BT.2100-0 HLG EOTF. Converts nonlinear signal values to linear relative
// display light, both normalized to [0,1].
highp vec4 hlgEotf(highp vec4 linearColor) {
return vec4(
hlgEotfSingleChannel(linearColor.r),
hlgEotfSingleChannel(linearColor.g),
hlgEotfSingleChannel(linearColor.b),
linearColor.a
);
}
void main() {
outColor = hlgEotf(texture(uTexSampler, vTexSamplingCoord));
}

87
libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl Normal file
View File

@ -0,0 +1,87 @@
#version 300 es
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 fragment shader that:
// 1. samples optical linear BT.2020 RGB from a (non-external) texture with
// uTexSampler,
// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB,
// and
// 3. copies this converted texture color to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
in vec2 vTexSamplingCoord;
out vec4 outColor;
// C.java#GlColorTransfer value.
// Only GL_COLOR_TRANSFER_ST2084 and GL_COLOR_TRANSFER_HLG are allowed.
uniform int uOetfGlColorTransfer;
uniform mat3 uColorTransform;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization.
// HLG OETF for one channel.
highp float hlgOetfSingleChannel(highp float linearChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return linearChannel <= 1.0 / 12.0 ? sqrt(3.0 * linearChannel) :
a * log(12.0 * linearChannel - b) + c;
}
// BT.2100 / BT.2020 HLG OETF.
highp vec3 hlgOetf(highp vec3 linearColor) {
return vec3(
hlgOetfSingleChannel(linearColor.r),
hlgOetfSingleChannel(linearColor.g),
hlgOetfSingleChannel(linearColor.b)
);
}
// BT.2100 / BT.2020, PQ / ST2084 OETF.
highp vec3 pqOetf(highp vec3 linearColor) {
// Specification:
// https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=514-527;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float m1 = (2610.0 / 16384.0);
const highp float m2 = (2523.0 / 4096.0) * 128.0;
const highp float c1 = (3424.0 / 4096.0);
const highp float c2 = (2413.0 / 4096.0) * 32.0;
const highp float c3 = (2392.0 / 4096.0) * 32.0;
highp vec3 temp = pow(linearColor, vec3(m1));
temp = (c1 + c2 * temp) / (1.0 + c3 * temp);
return pow(temp, vec3(m2));
}
// Applies the appropriate OETF to convert linear optical signals to nonlinear
// electrical signals. Input and output are both normalzied to [0, 1].
highp vec3 getElectricalColor(highp vec3 linearColor) {
// LINT.IfChange(color_transfer)
const int GL_COLOR_TRANSFER_ST2084 = 6;
return (uOetfGlColorTransfer == GL_COLOR_TRANSFER_ST2084) ?
pqOetf(linearColor) : hlgOetf(linearColor);
}
void main() {
vec4 inputColor = texture(uTexSampler, vTexSamplingCoord);
outColor = vec4(getElectricalColor(inputColor.rgb), inputColor.a);
}

2
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
View File

@ -374,7 +374,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
matrixTransformationListBuilder.build(), matrixTransformationListBuilder.build(),
sampleFromExternalTexture, sampleFromExternalTexture,
colorInfo, colorInfo,
/* outputOpticalColors= */ true); /* outputElectricalColors= */ true);
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
Pair<Integer, Integer> outputSize = Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight); matrixTransformationProcessor.configure(inputWidth, inputHeight);

4
libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java
View File

@ -37,10 +37,6 @@ public interface GlEffect extends Effect {
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
*/ */
// TODO(b/227624622): PQ input files will actually have the incorrect HLG OETF applied, so that
// the intermediate color space will be PQ with the HLG OETF applied. This means intermediate
// GlEffects affecting color will look incorrect on PQ input. Fix this by implementing proper PQ
// OETF / EOTF support.
GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr) GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException; throws FrameProcessingException;
} }

4
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
View File

@ -212,7 +212,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
matrixTransformations, matrixTransformations,
sampleFromExternalTexture, sampleFromExternalTexture,
colorInfo, colorInfo,
/* outputOpticalColors= */ false)); /* outputElectricalColors= */ false));
matrixTransformationListBuilder = new ImmutableList.Builder<>(); matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false; sampleFromExternalTexture = false;
} }
@ -242,7 +242,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
ImmutableList.of(), ImmutableList.of(),
sampleFromExternalTexture, sampleFromExternalTexture,
colorInfo, colorInfo,
/* outputOpticalColors= */ false)); /* outputElectricalColors= */ false));
sampleFromExternalTexture = false; sampleFromExternalTexture = false;
} }
textureProcessorListBuilder.add( textureProcessorListBuilder.add(

105
libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
View File

@ -15,6 +15,7 @@
*/ */
package androidx.media3.effect; package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context; import android.content.Context;
@ -23,6 +24,7 @@ import android.opengl.Matrix;
import android.util.Pair; import android.util.Pair;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.ColorInfo; import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.FrameProcessingException; import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
@ -53,8 +55,8 @@ import java.util.Arrays;
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH = private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl"; "shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl"; private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_HLG_EOTF_ES3_PATH = private static final String FRAGMENT_SHADER_OETF_ES3_PATH =
"shaders/fragment_shader_hlg_eotf_es3.glsl"; "shaders/fragment_shader_oetf_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl"; "shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
@ -123,9 +125,9 @@ import java.util.Arrays;
this( this(
createGlProgram( createGlProgram(
context, context,
/* inputOpticalColorsFromExternalTexture= */ false, /* inputElectricalColorsFromExternalTexture= */ false,
useHdr, useHdr,
/* outputOpticalColors= */ false), /* outputElectricalColors= */ false),
ImmutableList.of(matrixTransformation), ImmutableList.of(matrixTransformation),
useHdr); useHdr);
} }
@ -146,9 +148,9 @@ import java.util.Arrays;
this( this(
createGlProgram( createGlProgram(
context, context,
/* inputOpticalColorsFromExternalTexture= */ false, /* inputElectricalColorsFromExternalTexture= */ false,
useHdr, useHdr,
/* outputOpticalColors= */ false), /* outputElectricalColors= */ false),
ImmutableList.of(matrixTransformation), ImmutableList.of(matrixTransformation),
useHdr); useHdr);
} }
@ -156,65 +158,70 @@ import java.util.Arrays;
/** /**
* Creates a new instance. * Creates a new instance.
* *
* <p>Able to convert optical {@link ColorInfo} inputs and outputs to and from the intermediate * <p>Able to convert nonlinear electrical {@link ColorInfo} inputs and outputs to and from the
* {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma RGB BT.709 for SDR. * intermediate optical {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma
* RGB BT.709 for SDR.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. * apply to each frame in order.
* @param inputOpticalColorsFromExternalTexture Whether optical color input will be provided using * @param inputElectricalColorsFromExternalTexture Whether electrical color input will be provided
* an external texture. If {@code true}, the caller should use {@link * using an external texture. If {@code true}, the caller should use {@link
* #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with
* the external texture. * the external texture.
* @param opticalColorInfo The optical {@link ColorInfo}, only used to transform between color * @param electricalColorInfo The electrical {@link ColorInfo}, only used to transform between
* spaces and transfers, when {@code inputOpticalColorsFromExternalTexture} or {@code * color spaces and transfers, when {@code inputElectricalColorsFromExternalTexture} or {@code
* outputOpticalColors} are {@code true}. If it {@link ColorInfo#isTransferHdr(ColorInfo)}, * outputElectricalColors} are {@code true}. If it is {@linkplain
* intermediate {@link GlTextureProcessor} colors will be in linear RGB BT.2020. Otherwise, * ColorInfo#isTransferHdr(ColorInfo) HDR}, intermediate {@link GlTextureProcessor} colors
* these colors will be in gamma RGB BT.709. * will be in linear RGB BT.2020. Otherwise, these colors will be in gamma RGB BT.709.
* @param outputOpticalColors If {@code true}, outputs {@code opticalColorInfo}. If {@code false}, * @param outputElectricalColors If {@code true}, outputs {@code electricalColorInfo}. If {@code
* outputs intermediate colors of linear RGB BT.2020 if {@code opticalColorInfo} {@link * false}, outputs intermediate colors of linear RGB BT.2020 if {@code electricalColorInfo} is
* ColorInfo#isTransferHdr(ColorInfo)}, and gamma RGB BT.709 otherwise. * {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 otherwise.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported. * operation fails or is unsupported.
*/ */
public MatrixTransformationProcessor( public MatrixTransformationProcessor(
Context context, Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations, ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean inputOpticalColorsFromExternalTexture, boolean inputElectricalColorsFromExternalTexture,
ColorInfo opticalColorInfo, ColorInfo electricalColorInfo,
boolean outputOpticalColors) boolean outputElectricalColors)
throws FrameProcessingException { throws FrameProcessingException {
this( this(
createGlProgram( createGlProgram(
context, context,
inputOpticalColorsFromExternalTexture, inputElectricalColorsFromExternalTexture,
ColorInfo.isTransferHdr(opticalColorInfo), ColorInfo.isTransferHdr(electricalColorInfo),
outputOpticalColors), outputElectricalColors),
matrixTransformations, matrixTransformations,
ColorInfo.isTransferHdr(opticalColorInfo)); ColorInfo.isTransferHdr(electricalColorInfo));
if (!ColorInfo.isTransferHdr(opticalColorInfo) || !inputOpticalColorsFromExternalTexture) { if (!ColorInfo.isTransferHdr(electricalColorInfo)) {
return; return;
} }
// TODO(b/227624622): Implement YUV to RGB conversions in COLOR_RANGE_LIMITED as well, using
// opticalColorInfo.colorRange to select between them.
// In HDR editing mode the decoder output is sampled in YUV. @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
if (!GlUtil.isYuvTargetExtensionSupported()) { checkArgument(
throw new FrameProcessingException( colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
"The EXT_YUV_target extension is required for HDR editing input."); if (inputElectricalColorsFromExternalTexture) {
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing input.");
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
electricalColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
// TODO(b/241902517): Implement gamma transfer functions.
// If electrical colors are both input and output, no EOTF is needed.
glProgram.setIntUniform(
"uEotfGlColorTransfer", outputElectricalColors ? Format.NO_VALUE : colorTransfer);
} else if (outputElectricalColors) {
glProgram.setIntUniform("uOetfGlColorTransfer", colorTransfer);
} }
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
opticalColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
// TODO(b/227624622): Implement PQ and gamma TFs, and use an @IntDef to select between HLG,
// PQ, and gamma, coming from opticalColorInfo.colorTransfer.
// Applying the OETF will output a linear signal. Not applying the OETF will output an optical
// signal.
glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f);
} }
/** /**
@ -243,14 +250,14 @@ import java.util.Arrays;
private static GlProgram createGlProgram( private static GlProgram createGlProgram(
Context context, Context context,
boolean inputOpticalColorsFromExternalTexture, boolean inputElectricalColorsFromExternalTexture,
boolean useHdr, boolean useHdr,
boolean outputOpticalColors) boolean outputElectricalColors)
throws FrameProcessingException { throws FrameProcessingException {
String vertexShaderFilePath; String vertexShaderFilePath;
String fragmentShaderFilePath; String fragmentShaderFilePath;
if (inputOpticalColorsFromExternalTexture) { if (inputElectricalColorsFromExternalTexture) {
if (useHdr) { if (useHdr) {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH; vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH;
@ -258,9 +265,9 @@ import java.util.Arrays;
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
} }
} else if (outputOpticalColors && useHdr) { } else if (outputElectricalColors && useHdr) {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH; vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_HLG_EOTF_ES3_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_OETF_ES3_PATH;
} else { } else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;