loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDR: Add effect interface support for different in/out color transfers

Browse Source 
Modify FrameProcessor and MatrixTextureProcessor interfaces to support
different input and output color transfers. Does not implement conversion between
color ranges (ex. HDR and SDR), but should allow for conversion between color
transfers of the same color range (ex. HLG and PQ).

This supports in-app tone mapping, where we need a single FrameProcessor to
input HDR color transfers (ex. HLG/PQ) and output SDR (ex. gamma2.2). This also
supports previewing, where we need a single FrameProcessor to be able to input HLG
and output PQ.

Manually tested by confirming colors still look right on SDR and HDR videos
with a rotation and color affect applied.

PiperOrigin-RevId: 493108678
This commit is contained in:
huangdarwin 2022-12-05 22:10:39 +00:00 committed by Ian Baker
parent 88a7d67f4a
commit cf61e219ef
9 changed files with 162 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java
View File

@ -44,9 +44,11 @@ public interface FrameProcessor {
* Creates a new {@link FrameProcessor} instance.
*
* @param context A {@link Context}.
* @param effects The {@link Effect} instances to apply to each frame.
* @param effects The {@link Effect} instances to apply to each frame. Applied on the {@code
* outputColorInfo}'s color space.
* @param debugViewProvider A {@link DebugViewProvider}.
* @param colorInfo The {@link ColorInfo} for input and output frames.
* @param inputColorInfo The {@link ColorInfo} for input frames.
* @param outputColorInfo The {@link ColorInfo} for output frames.
* @param releaseFramesAutomatically If {@code true}, the {@link FrameProcessor} will render
* output frames to the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface}
* automatically as {@link FrameProcessor} is done processing them. If {@code false}, the
@ -62,7 +64,8 @@ public interface FrameProcessor {
Context context,
List<Effect> effects,
DebugViewProvider debugViewProvider,
ColorInfo colorInfo,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean releaseFramesAutomatically,
Executor executor,
Listener listener)

3
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java
View File

@ -286,7 +286,8 @@ public final class GlEffectsFrameProcessorFrameReleaseTest {
(context, useHdr) ->
new BlankFrameProducer(inputPresentationTimesUs, useHdr)),
DebugViewProvider.NONE,
ColorInfo.SDR_BT709_LIMITED,
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
releaseFramesAutomatically,
MoreExecutors.directExecutor(),
new FrameProcessor.Listener() {

3
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java
View File

@ -418,7 +418,8 @@ public final class GlEffectsFrameProcessorPixelTest {
getApplicationContext(),
effects,
DebugViewProvider.NONE,
ColorInfo.SDR_BT709_LIMITED,
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new FrameProcessor.Listener() {

7
library/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl
View File

@ -23,9 +23,10 @@
// 3. Applies an EOTF based on uEotfColorTransfer, yielding optical linear
// BT.2020 RGB.
// 4. Applies a 4x4 RGB color matrix to change the pixel colors.
// 5. If uOetfColorTransfer is COLOR_TRANSFER_LINEAR, outputs linear colors as
// is to intermediate shaders. Otherwise, applies the HLG or PQ OETF, based
// on uOetfColorTransfer, to provide the corresponding output electrical color.
// 5. Output as requested by uOetfColorTransfer. Use COLOR_TRANSFER_LINEAR for
// outputting to intermediate shaders, or COLOR_TRANSFER_ST2084 /
// COLOR_TRANSFER_HLG to output electrical colors via an OETF (e.g. to an
// encoder).
// The output will be red if an error has occurred.
#extension GL_OES_EGL_image_external : require

33
library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl
View File

@ -20,15 +20,18 @@
// 2. Transforms the electrical colors to optical colors using the SMPTE 170M
// EOTF.
// 3. Applies a 4x4 RGB color matrix to change the pixel colors.
// 4. Transforms the optical colors back to electrical ones if uApplyOetf == 1
// using the SMPTE 170M OETF.
// 4. Output as requested by uOetfColorTransfer. Use COLOR_TRANSFER_LINEAR for
// outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to output
// electrical colors via an OETF (e.g. to an encoder).
#extension GL_OES_EGL_image_external : require
precision mediump float;
uniform samplerExternalOES uTexSampler;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;
uniform int uApplyOetf;
// C.java#ColorTransfer value.
// Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
uniform int uOetfColorTransfer;
const float inverseGamma = 0.4500;
const float gamma = 1.0 / inverseGamma;
@ -61,12 +64,26 @@ float sdrOetfSingleChannel(float opticalChannel) {
// Transforms optical SDR colors to electrical SDR using the SMPTE 170M OETF.
vec3 sdrOetf(vec3 opticalColor) {
return uApplyOetf == 1
? vec3(
return vec3(
sdrOetfSingleChannel(opticalColor.r),
sdrOetfSingleChannel(opticalColor.g),
sdrOetfSingleChannel(opticalColor.b))
: opticalColor;
sdrOetfSingleChannel(opticalColor.b));
}
// Applies the appropriate OETF to convert linear optical signals to nonlinear
// electrical signals. Input and output are both normalized to [0, 1].
highp vec3 applyOetf(highp vec3 linearColor) {
// LINT.IfChange(color_transfer_oetf)
const int COLOR_TRANSFER_LINEAR = 1;
const int COLOR_TRANSFER_SDR_VIDEO = 3;
if (uOetfColorTransfer == COLOR_TRANSFER_LINEAR) {
return linearColor;
} else if(uOetfColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return sdrOetf(linearColor);
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
}
void main() {
@ -75,5 +92,5 @@ void main() {
vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);
gl_FragColor = vec4(sdrOetf(transformedColors.rgb), inputColor.a);
gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a);
}

25
library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalMatrixTextureProcessorWrapper.java
View File

@ -70,7 +70,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final EGLContext eglContext;
private final DebugViewProvider debugViewProvider;
private final boolean sampleFromExternalTexture;
private final ColorInfo colorInfo;
private final ColorInfo inputColorInfo;
private final ColorInfo outputColorInfo;
private final boolean releaseFramesAutomatically;
private final Executor frameProcessorListenerExecutor;
private final FrameProcessor.Listener frameProcessorListener;
@ -104,7 +105,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ImmutableList<RgbMatrix> rgbMatrices,
DebugViewProvider debugViewProvider,
boolean sampleFromExternalTexture,
ColorInfo colorInfo,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean releaseFramesAutomatically,
Executor frameProcessorListenerExecutor,
FrameProcessor.Listener frameProcessorListener) {
@ -115,7 +117,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglContext = eglContext;
this.debugViewProvider = debugViewProvider;
this.sampleFromExternalTexture = sampleFromExternalTexture;
this.colorInfo = colorInfo;
this.inputColorInfo = inputColorInfo;
this.outputColorInfo = outputColorInfo;
this.releaseFramesAutomatically = releaseFramesAutomatically;
this.frameProcessorListenerExecutor = frameProcessorListenerExecutor;
this.frameProcessorListener = frameProcessorListener;
@ -336,13 +339,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
@Nullable EGLSurface outputEglSurface = this.outputEglSurface;
if (outputEglSurface == null) {
boolean colorInfoIsHdr = ColorInfo.isTransferHdr(colorInfo);
boolean outputTransferIsHdr = ColorInfo.isTransferHdr(outputColorInfo);
outputEglSurface =
GlUtil.createEglSurface(
eglDisplay,
outputSurfaceInfo.surface,
colorInfoIsHdr
outputTransferIsHdr
? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102
: GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888);
@ -352,7 +355,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSurfaceInfo.width, outputSurfaceInfo.height);
if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
debugSurfaceViewWrapper =
new SurfaceViewWrapper(eglDisplay, eglContext, colorInfoIsHdr, debugSurfaceView);
new SurfaceViewWrapper(eglDisplay, eglContext, outputTransferIsHdr, debugSurfaceView);
}
this.debugSurfaceView = debugSurfaceView;
}
@ -390,12 +393,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
matrixTransformationListBuilder.build();
if (sampleFromExternalTexture) {
matrixTextureProcessor =
MatrixTextureProcessor.createWithExternalSamplerApplyingEotfThenOetf(
context, expandedMatrixTransformations, rgbMatrices, colorInfo);
MatrixTextureProcessor.createWithExternalSampler(
context,
expandedMatrixTransformations,
rgbMatrices,
/* inputColorInfo= */ inputColorInfo,
/* outputColorInfo= */ outputColorInfo);
} else {
matrixTextureProcessor =
MatrixTextureProcessor.createApplyingOetf(
context, expandedMatrixTransformations, rgbMatrices, colorInfo);
context, expandedMatrixTransformations, rgbMatrices, outputColorInfo);
}
matrixTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);

51
library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java
View File

@ -68,12 +68,24 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
Context context,
List<Effect> effects,
DebugViewProvider debugViewProvider,
ColorInfo colorInfo,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean releaseFramesAutomatically,
Executor listenerExecutor,
Listener listener)
throws FrameProcessingException {
// TODO(b/261188041) Add tests to verify the Listener is invoked on the given Executor.
// TODO(b/239735341): Reduce the scope of these checks by implementing GL tone-mapping.
checkArgument(
inputColorInfo.colorSpace == outputColorInfo.colorSpace,
"Conversion between HDR and SDR color spaces is not yet supported.");
checkArgument(
ColorInfo.isTransferHdr(inputColorInfo) == ColorInfo.isTransferHdr(outputColorInfo),
"Conversion between HDR and SDR color transfers is not yet supported.");
checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
Future<GlEffectsFrameProcessor> glFrameProcessorFuture =
@ -83,7 +95,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
context,
effects,
debugViewProvider,
colorInfo,
inputColorInfo,
outputColorInfo,
releaseFramesAutomatically,
singleThreadExecutorService,
listenerExecutor,
@ -115,7 +128,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
Context context,
List<Effect> effects,
DebugViewProvider debugViewProvider,
ColorInfo colorInfo,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean releaseFramesAutomatically,
ExecutorService singleThreadExecutorService,
Executor executor,
@ -125,10 +139,11 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
// TODO(b/237674316): Delay initialization of things requiring the colorInfo, to
// configure based on the color info from the decoder output media format instead.
boolean useHdr = ColorInfo.isTransferHdr(colorInfo);
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
int[] configAttributes =
useHdr ? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102 : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888;
ColorInfo.isTransferHdr(outputColorInfo)
? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102
: GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888;
EGLContext eglContext = GlUtil.createEglContext(eglDisplay, configAttributes);
GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay, configAttributes);
@ -139,7 +154,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
eglDisplay,
eglContext,
debugViewProvider,
colorInfo,
inputColorInfo,
outputColorInfo,
releaseFramesAutomatically,
executor,
listener);
@ -173,7 +189,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
EGLDisplay eglDisplay,
EGLContext eglContext,
DebugViewProvider debugViewProvider,
ColorInfo colorInfo,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean releaseFramesAutomatically,
Executor executor,
Listener listener)
@ -184,6 +201,9 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
new ImmutableList.Builder<>();
ImmutableList.Builder<RgbMatrix> rgbMatrixListBuilder = new ImmutableList.Builder<>();
boolean sampleFromExternalTexture = true;
ColorInfo linearColorInfo =
new ColorInfo(
inputColorInfo.colorSpace, inputColorInfo.colorRange, C.COLOR_TRANSFER_LINEAR, null);
for (int i = 0; i < effects.size(); i++) {
Effect effect = effects.get(i);
checkArgument(effect instanceof GlEffect, "GlEffectsFrameProcessor only supports GlEffects");
@ -203,24 +223,28 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build();
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo);
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromExternalTexture) {
MatrixTextureProcessor matrixTextureProcessor;
if (sampleFromExternalTexture) {
matrixTextureProcessor =
MatrixTextureProcessor.createWithExternalSamplerApplyingEotf(
context, matrixTransformations, rgbMatrices, colorInfo);
MatrixTextureProcessor.createWithExternalSampler(
context,
matrixTransformations,
rgbMatrices,
/* inputColorInfo= */ inputColorInfo,
/* outputColorInfo= */ linearColorInfo);
} else {
matrixTextureProcessor =
MatrixTextureProcessor.create(
context, matrixTransformations, rgbMatrices, ColorInfo.isTransferHdr(colorInfo));
context, matrixTransformations, rgbMatrices, isOutputTransferHdr);
}
textureProcessorListBuilder.add(matrixTextureProcessor);
matrixTransformationListBuilder = new ImmutableList.Builder<>();
rgbMatrixListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
}
textureProcessorListBuilder.add(
glEffect.toGlTextureProcessor(context, ColorInfo.isTransferHdr(colorInfo)));
textureProcessorListBuilder.add(glEffect.toGlTextureProcessor(context, isOutputTransferHdr));
}
textureProcessorListBuilder.add(
@ -232,7 +256,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
rgbMatrixListBuilder.build(),
debugViewProvider,
sampleFromExternalTexture,
colorInfo,
/* inputColorInfo= */ sampleFromExternalTexture ? inputColorInfo : linearColorInfo,
outputColorInfo,
releaseFramesAutomatically,
executor,
listener));

158
library/effect/src/main/java/com/google/android/exoplayer2/effect/MatrixTextureProcessor.java
View File

@ -165,38 +165,50 @@ import java.util.List;
* #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the
* external texture.
*
* <p>Applies the {@code electricalColorInfo} EOTF to convert from electrical color input, to
* intermediate optical {@link GlTextureProcessor} color output, before {@code
* matrixTransformations} and {@code rgbMatrices} are applied.
*
* <p>Intermediate optical/linear colors are RGB BT.2020 if {@code electricalColorInfo} is
* {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
* <p>Applies the {@linkplain ColorInfo#colorTransfer inputColorInfo EOTF} to convert from
* electrical color input, to intermediate optical {@link GlTextureProcessor} color output, before
* {@code matrixTransformations} and {@code rgbMatrices} are applied. Also applies the {@linkplain
* ColorInfo#colorTransfer outputColorInfo OETF}, if needed, to convert back to an electrical
* color output.
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input colors.
* @param inputColorInfo The input electrical (nonlinear) {@link ColorInfo}.
* @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
* If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public static MatrixTextureProcessor createWithExternalSamplerApplyingEotf(
public static MatrixTextureProcessor createWithExternalSampler(
Context context,
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
ColorInfo inputColorInfo,
ColorInfo outputColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
isInputTransferHdr
? VERTEX_SHADER_TRANSFORMATION_ES3_PATH
: VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr
isInputTransferHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
if (useHdr) {
@C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer;
if (isInputTransferHdr) {
// TODO(b/239735341): Remove this after implementing in-app tone-mapping.
checkArgument(
outputColorInfo.colorSpace == C.COLOR_SPACE_BT2020,
"Converting from HDR to SDR is not yet supported.");
checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
@ -204,43 +216,57 @@ import java.util.List;
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
electricalColorInfo.colorRange == C.COLOR_RANGE_FULL
inputColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
@C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
@C.ColorTransfer int inputColorTransfer = inputColorInfo.colorTransfer;
checkArgument(
colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
glProgram.setIntUniform("uEotfColorTransfer", colorTransfer);
// No OETF needed, because the intended output here is optical colors.
glProgram.setIntUniform("uOetfColorTransfer", C.COLOR_TRANSFER_LINEAR);
inputColorTransfer == C.COLOR_TRANSFER_HLG
|| inputColorTransfer == C.COLOR_TRANSFER_ST2084);
glProgram.setIntUniform("uEotfColorTransfer", inputColorTransfer);
checkArgument(
outputColorTransfer == C.COLOR_TRANSFER_HLG
|| outputColorTransfer == C.COLOR_TRANSFER_ST2084
|| outputColorTransfer == C.COLOR_TRANSFER_LINEAR);
glProgram.setIntUniform("uOetfColorTransfer", outputColorTransfer);
} else {
glProgram.setIntUniform("uApplyOetf", 0);
checkArgument(
outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020,
"Converting from SDR to HDR is not supported.");
checkArgument(
inputColorInfo.colorSpace == C.COLOR_SPACE_BT709
|| inputColorInfo.colorSpace == C.COLOR_SPACE_BT601);
checkArgument(
outputColorTransfer == C.COLOR_TRANSFER_SDR
|| outputColorTransfer == C.COLOR_TRANSFER_LINEAR);
// The SDR shader automatically applies an COLOR_TRANSFER_SDR EOTF.
glProgram.setIntUniform("uOetfColorTransfer", outputColorTransfer);
}
return new MatrixTextureProcessor(
glProgram,
ImmutableList.copyOf(matrixTransformations),
ImmutableList.copyOf(rgbMatrices),
useHdr);
isInputTransferHdr);
}
/**
* Creates a new instance.
*
* <p>Applies the {@code electricalColorInfo} OETF to convert from intermediate optical {@link
* GlTextureProcessor} color input, to electrical color output, after {@code
* matrixTransformations} and {@code rgbMatrices} are applied.
* <p>Applies the {@linkplain ColorInfo#colorTransfer outputColorInfo OETF} to convert from
* intermediate optical {@link GlTextureProcessor} color input, to electrical color output, after
* {@code matrixTransformations} and {@code rgbMatrices} are applied.
*
* <p>Intermediate optical/linear colors are RGB BT.2020 if {@code electricalColorInfo} is
* {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
* <p>Intermediate optical/linear colors are RGB BT.2020 if {@code outputColorInfo} is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing output colors.
* @param outputColorInfo The electrical (non-linear) {@link ColorInfo} describing output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
@ -248,17 +274,19 @@ import java.util.List;
Context context,
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
ColorInfo outputColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
boolean outputIsHdr = ColorInfo.isTransferHdr(outputColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
outputIsHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH;
outputIsHdr
? FRAGMENT_SHADER_OETF_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
if (useHdr) {
@C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
if (outputIsHdr) {
@C.ColorTransfer int colorTransfer = outputColorInfo.colorTransfer;
checkArgument(
colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
glProgram.setIntUniform("uOetfColorTransfer", colorTransfer);
@ -268,69 +296,7 @@ import java.util.List;
glProgram,
ImmutableList.copyOf(matrixTransformations),
ImmutableList.copyOf(rgbMatrices),
useHdr);
}
/**
* Creates a new instance.
*
* <p>Input will be sampled from an external texture. The caller should use {@link
* #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the
* external texture.
*
* <p>Applies the EOTF, {@code matrixTransformations}, {@code rgbMatrices}, then the OETF, to
* convert from and to input and output electrical colors.
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public static MatrixTextureProcessor createWithExternalSamplerApplyingEotfThenOetf(
Context context,
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing input.");
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
electricalColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
@C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
checkArgument(
colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
glProgram.setIntUniform("uEotfColorTransfer", colorTransfer);
glProgram.setIntUniform("uOetfColorTransfer", colorTransfer);
} else {
glProgram.setIntUniform("uApplyOetf", 1);
}
return new MatrixTextureProcessor(
glProgram,
ImmutableList.copyOf(matrixTransformations),
ImmutableList.copyOf(rgbMatrices),
useHdr);
outputIsHdr);
}
/**

12
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
View File

@ -155,17 +155,19 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest,
fallbackListener);
// HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing.
// This implies that the OpenGL EXT_YUV_target extension is supported and hence the
// default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
// mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
ColorInfo encoderSupportedInputColor = encoderWrapper.getSupportedInputColor();
try {
frameProcessor =
frameProcessorFactory.create(
context,
effectsListBuilder.build(),
debugViewProvider,
// HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing.
// This implies that the OpenGL EXT_YUV_target extension is supported and hence the
// default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
// mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
encoderWrapper.getSupportedInputColor(),
/* inputColorInfo= */ encoderSupportedInputColor,
/* outputColorInfo= */ encoderSupportedInputColor,
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new FrameProcessor.Listener() {