loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDR: Implement HLG EOTF and OETF.

Browse Source 
This allows us to use BT.2020 RGB linear for intermediate shaders, which also
allows us to re-enable PeriodicVignetteProcessor, which should work properly in
linear color-spaces.

Manually tested by adding a GlEffectsWrapper, and confirming that HLG HDR editing still looks correct.

PiperOrigin-RevId: 462265821
(cherry picked from commit 2f977eeec93be5cc71fda57f80362c1bc639c041)
This commit is contained in:
huangdarwin 2022-07-21 00:24:41 +00:00 committed by microkatz
parent e959af40f1
commit 06d3c07a1c
10 changed files with 146 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java
View File

@ -67,7 +67,7 @@ import java.util.Locale;
*
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files.
*/
public BitmapOverlayProcessor(Context context, boolean useHdr) throws FrameProcessingException {

3
demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java
View File

@ -53,7 +53,7 @@ import java.io.IOException;
*
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param centerX The x-coordinate of the center of the effect.
* @param centerY The y-coordinate of the center of the effect.
* @param minInnerRadius The lower bound of the radius that is unaffected by the effect.
@ -71,7 +71,6 @@ import java.io.IOException;
float outerRadius)
throws FrameProcessingException {
super(useHdr);
checkArgument(!useHdr, "PeriodicVignetteProcessor does not support HDR color spaces.");
checkArgument(minInnerRadius <= maxInnerRadius);
checkArgument(maxInnerRadius <= outerRadius);
this.minInnerRadius = minInnerRadius;

2
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
View File

@ -72,7 +72,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph.

58
libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl
View File

@ -13,24 +13,62 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 fragment shader that samples from an external texture with uTexSampler,
// copying from this texture to the current output while applying the specified
// color transform uColorTransform, which should be a YUV to RGB conversion
// matrix. The sampler uses the using the EXT_YUV_target extension:
// https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt.
// ES 3 fragment shader that:
// 1. samples HLG BT.2020 YUV from an external texture with uTexSampler, where
// the sampler uses the EXT_YUV_target extension specified at
// https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt,
// 2. Applies a YUV to RGB conversion using the specified color transform
// uColorTransform, yielding HLG BT.2020 RGB,
// 3. If uApplyHlgOetf is 1, outputs HLG BT.2020 RGB. If 0, outputs
// linear BT.2020 RGB for intermediate shaders by applying the HLG OETF.
// 4. Copies this converted texture color to the current output.
#extension GL_OES_EGL_image_external : require
#extension GL_EXT_YUV_target : require
precision mediump float;
uniform __samplerExternal2DY2YEXT uTexSampler;
// YUV to RGB conversion matrix.
uniform mat3 uColorTransform;
uniform float uApplyHlgOetf;
in vec2 vTexSamplingCoord;
out vec4 outColor;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization errors.
// HLG OETF for one channel.
highp float hlgOetfSingleChannel(highp float hlgChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return hlgChannel <= 1.0 / 12.0 ? sqrt(3.0 * hlgChannel) :
a * log(12.0 * hlgChannel - b) + c;
}
// BT.2100-0 HLG OETF. Converts nonlinear relative display light to linear
// signal values, both normalized to [0, 1].
highp vec4 hlgOetf(highp vec4 hlgColor) {
return vec4(
hlgOetfSingleChannel(hlgColor.r),
hlgOetfSingleChannel(hlgColor.g),
hlgOetfSingleChannel(hlgColor.b),
hlgColor.a
);
}
/** Convert YUV to RGBA. */
vec4 yuvToRgba(vec3 yuv) {
vec3 yuvOffset = vec3(yuv.x - 0.0625, yuv.y - 0.5, yuv.z - 0.5);
return vec4(uColorTransform * yuvOffset, 1.0);
}
void main() {
vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
vec3 yuvOffset;
yuvOffset.x = srcYuv.r - 0.0625;
yuvOffset.y = srcYuv.g - 0.5;
yuvOffset.z = srcYuv.b - 0.5;
outColor = vec4(uColorTransform * yuvOffset, 1.0);
outColor = yuvToRgba(srcYuv);
outColor = (uApplyHlgOetf == 1.0) ? hlgOetf(outColor) : outColor;
}

55
libraries/transformer/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl Normal file
View File

@ -0,0 +1,55 @@
#version 300 es
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 fragment shader that:
// 1. samples linear BT.2020 RGB from a (non-external) texture with uTexSampler,
// 2. applies the HLG OETF to yield HLG BT.2020 RGB, and
// 3. copies this converted texture color to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
in vec2 vTexSamplingCoord;
out vec4 outColor;
uniform mat3 uColorTransform;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization.
// HLG OETF for one channel.
highp float hlgEotfSingleChannel(highp float linearChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return linearChannel <= 0.5 ? linearChannel * linearChannel / 3.0 :
(b + exp((linearChannel - c) / a)) / 12.0;
}
// BT.2100-0 HLG EOTF. Converts nonlinear signal values to linear relative
// display light, both normalized to [0,1].
highp vec4 hlgEotf(highp vec4 linearColor) {
return vec4(
hlgEotfSingleChannel(linearColor.r),
hlgEotfSingleChannel(linearColor.g),
hlgEotfSingleChannel(linearColor.b),
linearColor.a
);
}
void main() {
outColor = hlgEotf(texture(uTexSampler, vTexSamplingCoord));
}

6
libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java
View File

@ -251,7 +251,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
MatrixTransformationProcessor matrixTransformationProcessor =
new MatrixTransformationProcessor(
context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr);
context,
matrixTransformationListBuilder.build(),
sampleFromExternalTexture,
useHdr,
/* outputOpticalColors= */ true);
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight);
checkState(outputSize.getWidth() == outputSurfaceInfo.width);

6
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java
View File

@ -33,8 +33,12 @@ public interface GlEffect {
*
* @param context A {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
*/
// TODO(b/227624622): PQ input files will actually have the incorrect HLG OETF applied, so that
// the intermediate color space will be PQ with the HLG OETF applied. This means intermediate
// GlEffects affecting color will look incorrect on PQ input. Fix this by implementing proper PQ
// OETF / EOTF support.
GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException;
}

6
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
View File

@ -185,7 +185,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) {
textureProcessorListBuilder.add(
new MatrixTransformationProcessor(
context, matrixTransformations, sampleFromExternalTexture, useHdr));
context,
matrixTransformations,
sampleFromExternalTexture,
useHdr,
/* outputOpticalColors= */ false));
matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
}

30
libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java
View File

@ -50,6 +50,8 @@ import java.util.Arrays;
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_HLG_EOTF_ES3_PATH =
"shaders/fragment_shader_hlg_eotf_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
@ -100,7 +102,7 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation
* matrix to use for each frame.
* @throws FrameProcessingException If a problem occurs while reading shader files.
@ -112,7 +114,8 @@ import java.util.Arrays;
context,
ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false,
useHdr);
useHdr,
/* outputOpticalColors= */ false);
}
/**
@ -120,7 +123,7 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation
* matrix to use for each frame.
* @throws FrameProcessingException If a problem occurs while reading shader files.
@ -132,7 +135,8 @@ import java.util.Arrays;
context,
ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false,
useHdr);
useHdr,
/* outputOpticalColors= */ false);
}
/**
@ -146,6 +150,9 @@ import java.util.Arrays;
* provide the transformation matrix associated with the external texture.
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
* EXT_YUV_target} OpenGL extension.
* @param outputOpticalColors If {@code true} and {@code useHdr} is also {@code true}, outputs a
* non-linear optical, or display light colors, possibly by applying the EOTF (Electro-optical
* transfer function). Otherwise, outputs linear electrical colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
@ -153,12 +160,13 @@ import java.util.Arrays;
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean sampleFromExternalTexture,
boolean useHdr)
boolean useHdr,
boolean outputOpticalColors)
throws FrameProcessingException {
super(useHdr);
if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing.");
"The EXT_YUV_target extension is required for HDR editing input.");
}
this.matrixTransformations = matrixTransformations;
@ -176,6 +184,11 @@ import java.util.Arrays;
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
} else if (outputOpticalColors) {
vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_HLG_EOTF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH;
} else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
@ -190,6 +203,11 @@ import java.util.Arrays;
if (useHdr && sampleFromExternalTexture) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
// TODO(b/227624622): Implement PQ, and use an @IntDef to select between HLG, PQ, and no
// transfer function.
// Applying the OETF will output a linear signal. Not applying the OETF will output an optical
// signal.
glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f);
}
float[] identityMatrix = new float[16];
Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);

2
libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java
View File

@ -46,7 +46,7 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
* Creates a {@code SingleFrameGlTextureProcessor} instance.
*
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
*/
public SingleFrameGlTextureProcessor(boolean useHdr) {
this.useHdr = useHdr;