From a5ff4ef17f78b64735d82c6b16f0b69af5a570a3 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Tue, 12 Jul 2022 11:04:05 +0000 Subject: [PATCH] HDR: Check whether EXT_YUV_target extension is supported. This extension is needed for editing HDR input with OpenGL, as the ExternalTextureProcessor samples raw YUV values from the external texture for HDR and converts them to RGB itself rather than relying on the OpenGL driver to do this automatically as for SDR. PiperOrigin-RevId: 460424154 --- .../androidx/media3/common/util/GlUtil.java | 37 +++++++++++++++++++ .../transformer/GlEffectsFrameProcessor.java | 3 +- .../MatrixTransformationProcessor.java | 27 ++++++++------ .../VideoTranscodingSamplePipeline.java | 5 ++- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index aa6abcad39..1c488a2459 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -63,6 +63,8 @@ public final class GlUtil { private static final String EXTENSION_PROTECTED_CONTENT = "EGL_EXT_protected_content"; // https://www.khronos.org/registry/EGL/extensions/KHR/EGL_KHR_surfaceless_context.txt private static final String EXTENSION_SURFACELESS_CONTEXT = "EGL_KHR_surfaceless_context"; + // https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt + private static final String EXTENSION_YUV_TARGET = "GL_EXT_YUV_target"; private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_NONE = new int[] {EGL14.EGL_NONE}; private static final int[] EGL_CONFIG_ATTRIBUTES_RGBA_8888 = @@ -170,6 +172,41 @@ public final class GlUtil { return eglExtensions != null && eglExtensions.contains(EXTENSION_SURFACELESS_CONTEXT); } + /** + * Returns whether the {@value #EXTENSION_YUV_TARGET} extension is supported. + * + *

This extension allows sampling raw YUV values from an external texture, which is required + * for HDR. + */ + public static boolean isYuvTargetExtensionSupported() { + if (Util.SDK_INT < 17) { + return false; + } + + @Nullable String glExtensions; + if (Util.areEqual(EGL14.eglGetCurrentContext(), EGL14.EGL_NO_CONTEXT)) { + // Create a placeholder context and make it current to allow calling GLES20.glGetString(). + try { + EGLDisplay eglDisplay = createEglDisplay(); + EGLContext eglContext = createEglContext(eglDisplay); + if (GlUtil.isSurfacelessContextExtensionSupported()) { + focusEglSurface( + eglDisplay, eglContext, EGL14.EGL_NO_SURFACE, /* width= */ 1, /* height= */ 1); + } else { + focusPlaceholderEglSurface(eglContext, eglDisplay); + } + glExtensions = GLES20.glGetString(GLES20.GL_EXTENSIONS); + destroyEglContext(eglDisplay, eglContext); + } catch (GlException e) { + return false; + } + } else { + glExtensions = GLES20.glGetString(GLES20.GL_EXTENSIONS); + } + + return glExtensions != null && glExtensions.contains(EXTENSION_YUV_TARGET); + } + /** Returns an initialized default {@link EGLDisplay}. */ @RequiresApi(17) public static EGLDisplay createEglDisplay() throws GlException { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index 3856a6bdee..46e4c298ed 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -53,7 +53,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param listener A {@link Listener}. * @param effects The {@link GlEffect GlEffects} to apply to each frame. * @param debugViewProvider A {@link DebugViewProvider}. - * @param useHdr Whether to process the input as an HDR signal. + * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code + * EXT_YUV_target} OpenGL extension. * @return A new instance. * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while * creating and configuring the OpenGL components. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java index c85bf100f1..7379cdf974 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java @@ -109,7 +109,7 @@ import java.util.Arrays; context, ImmutableList.of(matrixTransformation), /* sampleFromExternalTexture= */ false, - /* enableExperimentalHdrEditing= */ false); + /* useHdr= */ false); } /** @@ -126,7 +126,7 @@ import java.util.Arrays; context, ImmutableList.of(matrixTransformation), /* sampleFromExternalTexture= */ false, - /* enableExperimentalHdrEditing= */ false); + /* useHdr= */ false); } /** @@ -138,15 +138,22 @@ import java.util.Arrays; * @param sampleFromExternalTexture Whether the input will be provided using an external texture. * If {@code true}, the caller should use {@link #setTextureTransformMatrix(float[])} to * provide the transformation matrix associated with the external texture. - * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. - * @throws FrameProcessingException If a problem occurs while reading shader files. + * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code + * EXT_YUV_target} OpenGL extension. + * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL + * operation fails or is unsupported. */ public MatrixTransformationProcessor( Context context, ImmutableList matrixTransformations, boolean sampleFromExternalTexture, - boolean enableExperimentalHdrEditing) + boolean useHdr) throws FrameProcessingException { + if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) { + throw new FrameProcessingException( + "The EXT_YUV_target extension is required for HDR editing."); + } + this.matrixTransformations = matrixTransformations; transformationMatrixCache = new float[matrixTransformations.size()][16]; @@ -159,13 +166,9 @@ import java.util.Arrays; String fragmentShaderFilePath; if (sampleFromExternalTexture) { vertexShaderFilePath = - enableExperimentalHdrEditing - ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH - : VERTEX_SHADER_TRANSFORMATION_PATH; + useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; fragmentShaderFilePath = - enableExperimentalHdrEditing - ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH - : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; } else { vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH; @@ -177,7 +180,7 @@ import java.util.Arrays; throw new FrameProcessingException(e); } - if (enableExperimentalHdrEditing && sampleFromExternalTexture) { + if (useHdr && sampleFromExternalTexture) { // In HDR editing mode the decoder output is sampled in YUV. glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 8b3ebe6bb2..0c97b1268f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -111,8 +111,6 @@ import org.checkerframework.dataflow.qual.Pure; boolean useHdr = transformationRequest.enableHdrEditing && ColorInfo.isHdr(inputFormat.colorInfo); if (useHdr && !encoderWrapper.supportsHdr()) { - // TODO(b/236316454): Also check whether GlEffectsFrameProcessor supports HDR, i.e., whether - // EXT_YUV_target is supported. useHdr = false; enableRequestSdrToneMapping = true; encoderWrapper.signalFallbackToSdr(); @@ -152,6 +150,9 @@ import org.checkerframework.dataflow.qual.Pure; streamOffsetUs, effectsListBuilder.build(), debugViewProvider, + // HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This + // implies that the OpenGL EXT_YUV_target extension is supported and hence the + // GlEffectsFrameProcessor also supports HDR. useHdr); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessingException(