From a0870a42be5336b313f79f596e94411563e14720 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Fri, 1 Jul 2022 16:57:43 +0000 Subject: [PATCH] HDR: Configure GL shaders and encoder. Configure the GL shaders and encoder to take in HDR metadata. This mostly just consists of passing the Format.colorInfo through the VideoTranscodingSamplePipeline down to the encoder, rather than passing the PQ-ness down to the GL step. Due to b/237674316, this will remove HDR10+ support temporarily to introduce support for HLG10. Manually tested to confirm that HLG10 operations that don't affect color display correctly after this CL with "HDR editing" in the demo checked, and continue to display incorrectly (as before this CL) without the option unchecked. PiperOrigin-RevId: 458490810 --- .../androidx/media3/common/util/GlUtil.java | 18 +++++-------- .../transformer/DefaultEncoderFactory.java | 2 ++ .../transformer/ExternalTextureProcessor.java | 15 ++++------- ...lMatrixTransformationProcessorWrapper.java | 27 ++++++++----------- .../transformer/GlEffectsFrameProcessor.java | 23 ++++++++-------- .../VideoTranscodingSamplePipeline.java | 13 ++++++++- 6 files changed, 48 insertions(+), 50 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index adbb780e7a..1d623ef416 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -66,12 +66,8 @@ public final class GlUtil { // https://www.khronos.org/registry/EGL/extensions/KHR/EGL_KHR_gl_colorspace.txt private static final int EGL_GL_COLORSPACE_KHR = 0x309D; - // https://www.khronos.org/registry/EGL/extensions/EXT/EGL_EXT_gl_colorspace_bt2020_linear.txt - private static final int EGL_GL_COLORSPACE_BT2020_PQ_EXT = 0x3340; private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_NONE = new int[] {EGL14.EGL_NONE}; - private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ = - new int[] {EGL_GL_COLORSPACE_KHR, EGL_GL_COLORSPACE_BT2020_PQ_EXT, EGL14.EGL_NONE}; private static final int[] EGL_CONFIG_ATTRIBUTES_RGBA_8888 = new int[] { EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, @@ -213,19 +209,19 @@ public final class GlUtil { /** * Returns a new {@link EGLSurface} wrapping the specified {@code surface}, for HDR rendering with - * Rec. 2020 color primaries and using the PQ transfer function. + * Rec. 2020 color primaries. * * @param eglDisplay The {@link EGLDisplay} to attach the surface to. * @param surface The surface to wrap; must be a surface, surface texture or surface holder. */ @RequiresApi(17) - public static EGLSurface getEglSurfaceBt2020Pq(EGLDisplay eglDisplay, Object surface) + public static EGLSurface getEglSurfaceBt2020(EGLDisplay eglDisplay, Object surface) throws GlException { return Api17.getEglSurface( eglDisplay, surface, EGL_CONFIG_ATTRIBUTES_RGBA_1010102, - EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ); + EGL_WINDOW_SURFACE_ATTRIBUTES_NONE); } /** @@ -277,22 +273,22 @@ public final class GlUtil { /** * Creates and focuses a new {@link EGLSurface} wrapping a 1x1 pixel buffer, for HDR rendering - * with Rec. 2020 color primaries and using the PQ transfer function. + * with Rec. 2020 color primaries. * * @param eglContext The {@link EGLContext} to make current. * @param eglDisplay The {@link EGLDisplay} to attach the surface to. */ @RequiresApi(17) - public static void focusPlaceholderEglSurfaceBt2020Pq( - EGLContext eglContext, EGLDisplay eglDisplay) throws GlException { + public static void focusPlaceholderEglSurfaceBt2020(EGLContext eglContext, EGLDisplay eglDisplay) + throws GlException { int[] pbufferAttributes = new int[] { EGL14.EGL_WIDTH, /* width= */ 1, EGL14.EGL_HEIGHT, /* height= */ 1, + // TODO(b/227624622): Figure out if we can remove the EGL_GL_COLORSPACE_KHR item. EGL_GL_COLORSPACE_KHR, - EGL_GL_COLORSPACE_BT2020_PQ_EXT, EGL14.EGL_NONE }; EGLSurface eglSurface = diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index c3048f4368..df4bcb1393 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -34,6 +34,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Log; +import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; @@ -254,6 +255,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo); } + MediaFormatUtil.maybeSetColorInfo(mediaFormat, format.colorInfo); mediaFormat.setInteger( MediaFormat.KEY_COLOR_FORMAT, supportedVideoEncoderSettings.colorProfile); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java index 693f5533bc..a2f96ef63f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java @@ -49,19 +49,14 @@ import java.io.IOException; /** * Creates a new instance. * - * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. + * @param useHdr Whether to process the input as an HDR signal. * @throws FrameProcessingException If a problem occurs while reading shader files. */ - public ExternalTextureProcessor(Context context, boolean enableExperimentalHdrEditing) - throws FrameProcessingException { + public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException { String vertexShaderFilePath = - enableExperimentalHdrEditing - ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH - : VERTEX_SHADER_TEX_TRANSFORM_PATH; + useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH; String fragmentShaderFilePath = - enableExperimentalHdrEditing - ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH - : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; try { glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); } catch (IOException | GlUtil.GlException e) { @@ -72,7 +67,7 @@ import java.io.IOException; "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - if (enableExperimentalHdrEditing) { + if (useHdr) { // In HDR editing mode the decoder output is sampled in YUV. glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java index 44b890d903..1be2c7ea98 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java @@ -61,7 +61,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final long streamOffsetUs; private final DebugViewProvider debugViewProvider; private final FrameProcessor.Listener frameProcessorListener; - private final boolean enableExperimentalHdrEditing; + private final boolean useHdr; private int inputWidth; private int inputHeight; @@ -86,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, FrameProcessor.Listener frameProcessorListener, DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing) { + boolean useHdr) { this.context = context; this.matrixTransformations = matrixTransformations; this.eglDisplay = eglDisplay; @@ -94,7 +94,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.streamOffsetUs = streamOffsetUs; this.debugViewProvider = debugViewProvider; this.frameProcessorListener = frameProcessorListener; - this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; + this.useHdr = useHdr; } /** @@ -194,9 +194,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; @Nullable EGLSurface outputEglSurface = this.outputEglSurface; if (outputEglSurface == null) { // This means that outputSurfaceInfo changed. - if (enableExperimentalHdrEditing) { - // TODO(b/227624622): Don't assume BT.2020 PQ input/output. - outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface); + if (useHdr) { + outputEglSurface = GlUtil.getEglSurfaceBt2020(eglDisplay, outputSurfaceInfo.surface); } else { outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface); } @@ -207,8 +206,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputSurfaceInfo.width, outputSurfaceInfo.height); if (debugSurfaceView != null) { debugSurfaceViewWrapper = - new SurfaceViewWrapper( - eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView); + new SurfaceViewWrapper(eglDisplay, eglContext, useHdr, debugSurfaceView); } if (matrixTransformationProcessor != null) { matrixTransformationProcessor.release(); @@ -281,7 +279,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final class SurfaceViewWrapper implements SurfaceHolder.Callback { private final EGLDisplay eglDisplay; private final EGLContext eglContext; - private final boolean enableExperimentalHdrEditing; + private final boolean useHdr; @GuardedBy("this") @Nullable @@ -295,13 +293,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private int height; public SurfaceViewWrapper( - EGLDisplay eglDisplay, - EGLContext eglContext, - boolean enableExperimentalHdrEditing, - SurfaceView surfaceView) { + EGLDisplay eglDisplay, EGLContext eglContext, boolean useHdr, SurfaceView surfaceView) { this.eglDisplay = eglDisplay; this.eglContext = eglContext; - this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; + this.useHdr = useHdr; surfaceView.getHolder().addCallback(this); surface = surfaceView.getHolder().getSurface(); width = surfaceView.getWidth(); @@ -321,8 +316,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } if (eglSurface == null) { - if (enableExperimentalHdrEditing) { - eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, surface); + if (useHdr) { + eglSurface = GlUtil.getEglSurfaceBt2020(eglDisplay, surface); } else { eglSurface = GlUtil.getEglSurface(eglDisplay, surface); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index ce418d8281..d01d690c6e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -53,7 +53,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param listener A {@link Listener}. * @param effects The {@link GlEffect GlEffects} to apply to each frame. * @param debugViewProvider A {@link DebugViewProvider}. - * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. + * @param useHdr Whether to process the input as an HDR signal. * @return A new instance. * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while * creating and configuring the OpenGL components. @@ -64,7 +64,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, List effects, DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing) + boolean useHdr) throws FrameProcessingException { ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); @@ -78,7 +78,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, effects, debugViewProvider, - enableExperimentalHdrEditing, + useHdr, singleThreadExecutorService)); try { @@ -106,23 +106,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, List effects, DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing, + boolean useHdr, ExecutorService singleThreadExecutorService) throws GlUtil.GlException, FrameProcessingException { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); EGLDisplay eglDisplay = GlUtil.createEglDisplay(); EGLContext eglContext = - enableExperimentalHdrEditing + useHdr ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay) : GlUtil.createEglContext(eglDisplay); if (GlUtil.isSurfacelessContextExtensionSupported()) { GlUtil.focusEglSurface( eglDisplay, eglContext, EGL14.EGL_NO_SURFACE, /* width= */ 1, /* height= */ 1); - } else if (enableExperimentalHdrEditing) { - // TODO(b/227624622): Don't assume BT.2020 PQ input/output. - GlUtil.focusPlaceholderEglSurfaceBt2020Pq(eglContext, eglDisplay); + } else if (useHdr) { + GlUtil.focusPlaceholderEglSurfaceBt2020(eglContext, eglDisplay); } else { GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); } @@ -137,13 +136,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, listener, debugViewProvider, - enableExperimentalHdrEditing); + useHdr); ImmutableList intermediateTextureProcessors = textureProcessors.first; FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper = textureProcessors.second; ExternalTextureProcessor externalTextureProcessor = - new ExternalTextureProcessor(context, enableExperimentalHdrEditing); + new ExternalTextureProcessor(context, useHdr); FrameProcessingTaskExecutor frameProcessingTaskExecutor = new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); chainTextureProcessorsWithListeners( @@ -182,7 +181,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, FrameProcessor.Listener listener, DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing) + boolean useHdr) throws FrameProcessingException { ImmutableList.Builder textureProcessorListBuilder = new ImmutableList.Builder<>(); @@ -213,7 +212,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, listener, debugViewProvider, - enableExperimentalHdrEditing)); + useHdr)); } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 2bd3311656..5e1b6ddd12 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -24,6 +24,7 @@ import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; @@ -97,6 +98,10 @@ import org.checkerframework.dataflow.qual.Pure; transformationRequest, fallbackListener); + // TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly + // processed as SDR, because the inputFormat.colorInfo reports the wrong value. + boolean useHdr = transformationRequest.enableHdrEditing && isHdr(inputFormat.colorInfo); + try { frameProcessor = GlEffectsFrameProcessor.create( @@ -131,7 +136,7 @@ import org.checkerframework.dataflow.qual.Pure; streamOffsetUs, effectsListBuilder.build(), debugViewProvider, - transformationRequest.enableHdrEditing); + useHdr); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessingException( e, TransformationException.ERROR_CODE_GL_INIT_FAILED); @@ -147,6 +152,11 @@ import org.checkerframework.dataflow.qual.Pure; maxPendingFrameCount = decoder.getMaxPendingFrameCount(); } + /** Whether this is a supported HDR format. */ + private static boolean isHdr(@Nullable ColorInfo colorInfo) { + return colorInfo != null && colorInfo.colorTransfer != C.COLOR_TRANSFER_SDR; + } + @Override @Nullable public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { @@ -352,6 +362,7 @@ import org.checkerframework.dataflow.qual.Pure; transformationRequest.videoMimeType != null ? transformationRequest.videoMimeType : inputFormat.sampleMimeType) + .setColorInfo(inputFormat.colorInfo) .build(); encoder =