diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java b/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java index 70f1a58496..3ebbf008ed 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java @@ -44,9 +44,11 @@ public interface FrameProcessor { * Creates a new {@link FrameProcessor} instance. * * @param context A {@link Context}. - * @param effects The {@link Effect} instances to apply to each frame. + * @param effects The {@link Effect} instances to apply to each frame. Applied on the {@code + * outputColorInfo}'s color space. * @param debugViewProvider A {@link DebugViewProvider}. - * @param colorInfo The {@link ColorInfo} for input and output frames. + * @param inputColorInfo The {@link ColorInfo} for input frames. + * @param outputColorInfo The {@link ColorInfo} for output frames. * @param releaseFramesAutomatically If {@code true}, the {@link FrameProcessor} will render * output frames to the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} * automatically as {@link FrameProcessor} is done processing them. If {@code false}, the @@ -62,7 +64,8 @@ public interface FrameProcessor { Context context, List effects, DebugViewProvider debugViewProvider, - ColorInfo colorInfo, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, boolean releaseFramesAutomatically, Executor executor, Listener listener) diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java index af2033e593..4d5ede72bf 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java @@ -286,7 +286,8 @@ public final class GlEffectsFrameProcessorFrameReleaseTest { (context, useHdr) -> new BlankFrameProducer(inputPresentationTimesUs, useHdr)), DebugViewProvider.NONE, - ColorInfo.SDR_BT709_LIMITED, + /* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, releaseFramesAutomatically, MoreExecutors.directExecutor(), new FrameProcessor.Listener() { diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java index 66f33ecf50..f8459b28a7 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java @@ -418,7 +418,8 @@ public final class GlEffectsFrameProcessorPixelTest { getApplicationContext(), effects, DebugViewProvider.NONE, - ColorInfo.SDR_BT709_LIMITED, + /* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, /* releaseFramesAutomatically= */ true, MoreExecutors.directExecutor(), new FrameProcessor.Listener() { diff --git a/library/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl b/library/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl index aaabf57ac4..055ce99660 100644 --- a/library/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl +++ b/library/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl @@ -23,9 +23,10 @@ // 3. Applies an EOTF based on uEotfColorTransfer, yielding optical linear // BT.2020 RGB. // 4. Applies a 4x4 RGB color matrix to change the pixel colors. -// 5. If uOetfColorTransfer is COLOR_TRANSFER_LINEAR, outputs linear colors as -// is to intermediate shaders. Otherwise, applies the HLG or PQ OETF, based -// on uOetfColorTransfer, to provide the corresponding output electrical color. +// 5. Output as requested by uOetfColorTransfer. Use COLOR_TRANSFER_LINEAR for +// outputting to intermediate shaders, or COLOR_TRANSFER_ST2084 / +// COLOR_TRANSFER_HLG to output electrical colors via an OETF (e.g. to an +// encoder). // The output will be red if an error has occurred. #extension GL_OES_EGL_image_external : require diff --git a/library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl b/library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl index 6f8dc5489b..d2006c845d 100644 --- a/library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl +++ b/library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl @@ -20,15 +20,18 @@ // 2. Transforms the electrical colors to optical colors using the SMPTE 170M // EOTF. // 3. Applies a 4x4 RGB color matrix to change the pixel colors. -// 4. Transforms the optical colors back to electrical ones if uApplyOetf == 1 -// using the SMPTE 170M OETF. +// 4. Output as requested by uOetfColorTransfer. Use COLOR_TRANSFER_LINEAR for +// outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to output +// electrical colors via an OETF (e.g. to an encoder). #extension GL_OES_EGL_image_external : require precision mediump float; uniform samplerExternalOES uTexSampler; uniform mat4 uRgbMatrix; varying vec2 vTexSamplingCoord; -uniform int uApplyOetf; +// C.java#ColorTransfer value. +// Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed. +uniform int uOetfColorTransfer; const float inverseGamma = 0.4500; const float gamma = 1.0 / inverseGamma; @@ -61,12 +64,26 @@ float sdrOetfSingleChannel(float opticalChannel) { // Transforms optical SDR colors to electrical SDR using the SMPTE 170M OETF. vec3 sdrOetf(vec3 opticalColor) { - return uApplyOetf == 1 - ? vec3( + return vec3( sdrOetfSingleChannel(opticalColor.r), sdrOetfSingleChannel(opticalColor.g), - sdrOetfSingleChannel(opticalColor.b)) - : opticalColor; + sdrOetfSingleChannel(opticalColor.b)); +} + +// Applies the appropriate OETF to convert linear optical signals to nonlinear +// electrical signals. Input and output are both normalized to [0, 1]. +highp vec3 applyOetf(highp vec3 linearColor) { + // LINT.IfChange(color_transfer_oetf) + const int COLOR_TRANSFER_LINEAR = 1; + const int COLOR_TRANSFER_SDR_VIDEO = 3; + if (uOetfColorTransfer == COLOR_TRANSFER_LINEAR) { + return linearColor; + } else if(uOetfColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { + return sdrOetf(linearColor); + } else { + // Output red as an obviously visible error. + return vec3(1.0, 0.0, 0.0); + } } void main() { @@ -75,5 +92,5 @@ void main() { vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1); - gl_FragColor = vec4(sdrOetf(transformedColors.rgb), inputColor.a); + gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a); } diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalMatrixTextureProcessorWrapper.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalMatrixTextureProcessorWrapper.java index 8af28fba1b..ad3789f4de 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalMatrixTextureProcessorWrapper.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalMatrixTextureProcessorWrapper.java @@ -70,7 +70,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final EGLContext eglContext; private final DebugViewProvider debugViewProvider; private final boolean sampleFromExternalTexture; - private final ColorInfo colorInfo; + private final ColorInfo inputColorInfo; + private final ColorInfo outputColorInfo; private final boolean releaseFramesAutomatically; private final Executor frameProcessorListenerExecutor; private final FrameProcessor.Listener frameProcessorListener; @@ -104,7 +105,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ImmutableList rgbMatrices, DebugViewProvider debugViewProvider, boolean sampleFromExternalTexture, - ColorInfo colorInfo, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, boolean releaseFramesAutomatically, Executor frameProcessorListenerExecutor, FrameProcessor.Listener frameProcessorListener) { @@ -115,7 +117,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.eglContext = eglContext; this.debugViewProvider = debugViewProvider; this.sampleFromExternalTexture = sampleFromExternalTexture; - this.colorInfo = colorInfo; + this.inputColorInfo = inputColorInfo; + this.outputColorInfo = outputColorInfo; this.releaseFramesAutomatically = releaseFramesAutomatically; this.frameProcessorListenerExecutor = frameProcessorListenerExecutor; this.frameProcessorListener = frameProcessorListener; @@ -336,13 +339,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; @Nullable EGLSurface outputEglSurface = this.outputEglSurface; if (outputEglSurface == null) { - boolean colorInfoIsHdr = ColorInfo.isTransferHdr(colorInfo); + boolean outputTransferIsHdr = ColorInfo.isTransferHdr(outputColorInfo); outputEglSurface = GlUtil.createEglSurface( eglDisplay, outputSurfaceInfo.surface, - colorInfoIsHdr + outputTransferIsHdr ? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102 : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888); @@ -352,7 +355,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputSurfaceInfo.width, outputSurfaceInfo.height); if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) { debugSurfaceViewWrapper = - new SurfaceViewWrapper(eglDisplay, eglContext, colorInfoIsHdr, debugSurfaceView); + new SurfaceViewWrapper(eglDisplay, eglContext, outputTransferIsHdr, debugSurfaceView); } this.debugSurfaceView = debugSurfaceView; } @@ -390,12 +393,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; matrixTransformationListBuilder.build(); if (sampleFromExternalTexture) { matrixTextureProcessor = - MatrixTextureProcessor.createWithExternalSamplerApplyingEotfThenOetf( - context, expandedMatrixTransformations, rgbMatrices, colorInfo); + MatrixTextureProcessor.createWithExternalSampler( + context, + expandedMatrixTransformations, + rgbMatrices, + /* inputColorInfo= */ inputColorInfo, + /* outputColorInfo= */ outputColorInfo); } else { matrixTextureProcessor = MatrixTextureProcessor.createApplyingOetf( - context, expandedMatrixTransformations, rgbMatrices, colorInfo); + context, expandedMatrixTransformations, rgbMatrices, outputColorInfo); } matrixTextureProcessor.setTextureTransformMatrix(textureTransformMatrix); diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java index 2918b2f627..0fa1e514f4 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java @@ -68,12 +68,24 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { Context context, List effects, DebugViewProvider debugViewProvider, - ColorInfo colorInfo, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, boolean releaseFramesAutomatically, Executor listenerExecutor, Listener listener) throws FrameProcessingException { // TODO(b/261188041) Add tests to verify the Listener is invoked on the given Executor. + + // TODO(b/239735341): Reduce the scope of these checks by implementing GL tone-mapping. + checkArgument( + inputColorInfo.colorSpace == outputColorInfo.colorSpace, + "Conversion between HDR and SDR color spaces is not yet supported."); + checkArgument( + ColorInfo.isTransferHdr(inputColorInfo) == ColorInfo.isTransferHdr(outputColorInfo), + "Conversion between HDR and SDR color transfers is not yet supported."); + checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR); + checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR); + ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); Future glFrameProcessorFuture = @@ -83,7 +95,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { context, effects, debugViewProvider, - colorInfo, + inputColorInfo, + outputColorInfo, releaseFramesAutomatically, singleThreadExecutorService, listenerExecutor, @@ -115,7 +128,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { Context context, List effects, DebugViewProvider debugViewProvider, - ColorInfo colorInfo, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, boolean releaseFramesAutomatically, ExecutorService singleThreadExecutorService, Executor executor, @@ -125,10 +139,11 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { // TODO(b/237674316): Delay initialization of things requiring the colorInfo, to // configure based on the color info from the decoder output media format instead. - boolean useHdr = ColorInfo.isTransferHdr(colorInfo); EGLDisplay eglDisplay = GlUtil.createEglDisplay(); int[] configAttributes = - useHdr ? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102 : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888; + ColorInfo.isTransferHdr(outputColorInfo) + ? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102 + : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888; EGLContext eglContext = GlUtil.createEglContext(eglDisplay, configAttributes); GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay, configAttributes); @@ -139,7 +154,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { eglDisplay, eglContext, debugViewProvider, - colorInfo, + inputColorInfo, + outputColorInfo, releaseFramesAutomatically, executor, listener); @@ -173,7 +189,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { EGLDisplay eglDisplay, EGLContext eglContext, DebugViewProvider debugViewProvider, - ColorInfo colorInfo, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, boolean releaseFramesAutomatically, Executor executor, Listener listener) @@ -184,6 +201,9 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { new ImmutableList.Builder<>(); ImmutableList.Builder rgbMatrixListBuilder = new ImmutableList.Builder<>(); boolean sampleFromExternalTexture = true; + ColorInfo linearColorInfo = + new ColorInfo( + inputColorInfo.colorSpace, inputColorInfo.colorRange, C.COLOR_TRANSFER_LINEAR, null); for (int i = 0; i < effects.size(); i++) { Effect effect = effects.get(i); checkArgument(effect instanceof GlEffect, "GlEffectsFrameProcessor only supports GlEffects"); @@ -203,24 +223,28 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); ImmutableList rgbMatrices = rgbMatrixListBuilder.build(); + boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo); if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromExternalTexture) { MatrixTextureProcessor matrixTextureProcessor; if (sampleFromExternalTexture) { matrixTextureProcessor = - MatrixTextureProcessor.createWithExternalSamplerApplyingEotf( - context, matrixTransformations, rgbMatrices, colorInfo); + MatrixTextureProcessor.createWithExternalSampler( + context, + matrixTransformations, + rgbMatrices, + /* inputColorInfo= */ inputColorInfo, + /* outputColorInfo= */ linearColorInfo); } else { matrixTextureProcessor = MatrixTextureProcessor.create( - context, matrixTransformations, rgbMatrices, ColorInfo.isTransferHdr(colorInfo)); + context, matrixTransformations, rgbMatrices, isOutputTransferHdr); } textureProcessorListBuilder.add(matrixTextureProcessor); matrixTransformationListBuilder = new ImmutableList.Builder<>(); rgbMatrixListBuilder = new ImmutableList.Builder<>(); sampleFromExternalTexture = false; } - textureProcessorListBuilder.add( - glEffect.toGlTextureProcessor(context, ColorInfo.isTransferHdr(colorInfo))); + textureProcessorListBuilder.add(glEffect.toGlTextureProcessor(context, isOutputTransferHdr)); } textureProcessorListBuilder.add( @@ -232,7 +256,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { rgbMatrixListBuilder.build(), debugViewProvider, sampleFromExternalTexture, - colorInfo, + /* inputColorInfo= */ sampleFromExternalTexture ? inputColorInfo : linearColorInfo, + outputColorInfo, releaseFramesAutomatically, executor, listener)); diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/MatrixTextureProcessor.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/MatrixTextureProcessor.java index f48c017a7c..eda5e42248 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/MatrixTextureProcessor.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/MatrixTextureProcessor.java @@ -165,38 +165,50 @@ import java.util.List; * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the * external texture. * - *

Applies the {@code electricalColorInfo} EOTF to convert from electrical color input, to - * intermediate optical {@link GlTextureProcessor} color output, before {@code - * matrixTransformations} and {@code rgbMatrices} are applied. - * - *

Intermediate optical/linear colors are RGB BT.2020 if {@code electricalColorInfo} is - * {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not. + *

Applies the {@linkplain ColorInfo#colorTransfer inputColorInfo EOTF} to convert from + * electrical color input, to intermediate optical {@link GlTextureProcessor} color output, before + * {@code matrixTransformations} and {@code rgbMatrices} are applied. Also applies the {@linkplain + * ColorInfo#colorTransfer outputColorInfo OETF}, if needed, to convert back to an electrical + * color output. * * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * apply to each frame in order. Can be empty to apply no vertex transformations. * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be * empty to apply no color transformations. - * @param electricalColorInfo The electrical {@link ColorInfo} describing input colors. + * @param inputColorInfo The input electrical (nonlinear) {@link ColorInfo}. + * @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}. + * If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain + * ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. */ - public static MatrixTextureProcessor createWithExternalSamplerApplyingEotf( + public static MatrixTextureProcessor createWithExternalSampler( Context context, List matrixTransformations, List rgbMatrices, - ColorInfo electricalColorInfo) + ColorInfo inputColorInfo, + ColorInfo outputColorInfo) throws FrameProcessingException { - boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); + boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo); String vertexShaderFilePath = - useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; + isInputTransferHdr + ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH + : VERTEX_SHADER_TRANSFORMATION_PATH; String fragmentShaderFilePath = - useHdr + isInputTransferHdr ? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH; GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); - if (useHdr) { + @C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer; + if (isInputTransferHdr) { + // TODO(b/239735341): Remove this after implementing in-app tone-mapping. + checkArgument( + outputColorInfo.colorSpace == C.COLOR_SPACE_BT2020, + "Converting from HDR to SDR is not yet supported."); + checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020); + // In HDR editing mode the decoder output is sampled in YUV. if (!GlUtil.isYuvTargetExtensionSupported()) { throw new FrameProcessingException( @@ -204,43 +216,57 @@ import java.util.List; } glProgram.setFloatsUniform( "uYuvToRgbColorTransform", - electricalColorInfo.colorRange == C.COLOR_RANGE_FULL + inputColorInfo.colorRange == C.COLOR_RANGE_FULL ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); - @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; + @C.ColorTransfer int inputColorTransfer = inputColorInfo.colorTransfer; checkArgument( - colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); - glProgram.setIntUniform("uEotfColorTransfer", colorTransfer); - // No OETF needed, because the intended output here is optical colors. - glProgram.setIntUniform("uOetfColorTransfer", C.COLOR_TRANSFER_LINEAR); + inputColorTransfer == C.COLOR_TRANSFER_HLG + || inputColorTransfer == C.COLOR_TRANSFER_ST2084); + glProgram.setIntUniform("uEotfColorTransfer", inputColorTransfer); + checkArgument( + outputColorTransfer == C.COLOR_TRANSFER_HLG + || outputColorTransfer == C.COLOR_TRANSFER_ST2084 + || outputColorTransfer == C.COLOR_TRANSFER_LINEAR); + glProgram.setIntUniform("uOetfColorTransfer", outputColorTransfer); } else { - glProgram.setIntUniform("uApplyOetf", 0); + checkArgument( + outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020, + "Converting from SDR to HDR is not supported."); + checkArgument( + inputColorInfo.colorSpace == C.COLOR_SPACE_BT709 + || inputColorInfo.colorSpace == C.COLOR_SPACE_BT601); + checkArgument( + outputColorTransfer == C.COLOR_TRANSFER_SDR + || outputColorTransfer == C.COLOR_TRANSFER_LINEAR); + // The SDR shader automatically applies an COLOR_TRANSFER_SDR EOTF. + glProgram.setIntUniform("uOetfColorTransfer", outputColorTransfer); } return new MatrixTextureProcessor( glProgram, ImmutableList.copyOf(matrixTransformations), ImmutableList.copyOf(rgbMatrices), - useHdr); + isInputTransferHdr); } /** * Creates a new instance. * - *

Applies the {@code electricalColorInfo} OETF to convert from intermediate optical {@link - * GlTextureProcessor} color input, to electrical color output, after {@code - * matrixTransformations} and {@code rgbMatrices} are applied. + *

Applies the {@linkplain ColorInfo#colorTransfer outputColorInfo OETF} to convert from + * intermediate optical {@link GlTextureProcessor} color input, to electrical color output, after + * {@code matrixTransformations} and {@code rgbMatrices} are applied. * - *

Intermediate optical/linear colors are RGB BT.2020 if {@code electricalColorInfo} is - * {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not. + *

Intermediate optical/linear colors are RGB BT.2020 if {@code outputColorInfo} is {@linkplain + * ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not. * * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * apply to each frame in order. Can be empty to apply no vertex transformations. * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be * empty to apply no color transformations. - * @param electricalColorInfo The electrical {@link ColorInfo} describing output colors. + * @param outputColorInfo The electrical (non-linear) {@link ColorInfo} describing output colors. * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * operation fails or is unsupported. */ @@ -248,17 +274,19 @@ import java.util.List; Context context, List matrixTransformations, List rgbMatrices, - ColorInfo electricalColorInfo) + ColorInfo outputColorInfo) throws FrameProcessingException { - boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); + boolean outputIsHdr = ColorInfo.isTransferHdr(outputColorInfo); String vertexShaderFilePath = - useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; + outputIsHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; String fragmentShaderFilePath = - useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH; + outputIsHdr + ? FRAGMENT_SHADER_OETF_ES3_PATH + : FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH; GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); - if (useHdr) { - @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; + if (outputIsHdr) { + @C.ColorTransfer int colorTransfer = outputColorInfo.colorTransfer; checkArgument( colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); glProgram.setIntUniform("uOetfColorTransfer", colorTransfer); @@ -268,69 +296,7 @@ import java.util.List; glProgram, ImmutableList.copyOf(matrixTransformations), ImmutableList.copyOf(rgbMatrices), - useHdr); - } - - /** - * Creates a new instance. - * - *

Input will be sampled from an external texture. The caller should use {@link - * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the - * external texture. - * - *

Applies the EOTF, {@code matrixTransformations}, {@code rgbMatrices}, then the OETF, to - * convert from and to input and output electrical colors. - * - * @param context The {@link Context}. - * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to - * apply to each frame in order. Can be empty to apply no vertex transformations. - * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be - * empty to apply no color transformations. - * @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors. - * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL - * operation fails or is unsupported. - */ - public static MatrixTextureProcessor createWithExternalSamplerApplyingEotfThenOetf( - Context context, - List matrixTransformations, - List rgbMatrices, - ColorInfo electricalColorInfo) - throws FrameProcessingException { - boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo); - String vertexShaderFilePath = - useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; - String fragmentShaderFilePath = - useHdr - ? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH - : FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH; - GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); - - if (useHdr) { - // In HDR editing mode the decoder output is sampled in YUV. - if (!GlUtil.isYuvTargetExtensionSupported()) { - throw new FrameProcessingException( - "The EXT_YUV_target extension is required for HDR editing input."); - } - glProgram.setFloatsUniform( - "uYuvToRgbColorTransform", - electricalColorInfo.colorRange == C.COLOR_RANGE_FULL - ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX - : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX); - - @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; - checkArgument( - colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); - glProgram.setIntUniform("uEotfColorTransfer", colorTransfer); - glProgram.setIntUniform("uOetfColorTransfer", colorTransfer); - } else { - glProgram.setIntUniform("uApplyOetf", 1); - } - - return new MatrixTextureProcessor( - glProgram, - ImmutableList.copyOf(matrixTransformations), - ImmutableList.copyOf(rgbMatrices), - useHdr); + outputIsHdr); } /** diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index e5a5c26985..e8c7249dd3 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -155,17 +155,19 @@ import org.checkerframework.dataflow.qual.Pure; transformationRequest, fallbackListener); + // HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing. + // This implies that the OpenGL EXT_YUV_target extension is supported and hence the + // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone + // mapping is applied, which ensures the decoder outputs SDR output for an HDR input. + ColorInfo encoderSupportedInputColor = encoderWrapper.getSupportedInputColor(); try { frameProcessor = frameProcessorFactory.create( context, effectsListBuilder.build(), debugViewProvider, - // HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing. - // This implies that the OpenGL EXT_YUV_target extension is supported and hence the - // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone - // mapping is applied, which ensures the decoder outputs SDR output for an HDR input. - encoderWrapper.getSupportedInputColor(), + /* inputColorInfo= */ encoderSupportedInputColor, + /* outputColorInfo= */ encoderSupportedInputColor, /* releaseFramesAutomatically= */ true, MoreExecutors.directExecutor(), new FrameProcessor.Listener() {