diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index adbb780e7a..1d623ef416 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -66,12 +66,8 @@ public final class GlUtil { // https://www.khronos.org/registry/EGL/extensions/KHR/EGL_KHR_gl_colorspace.txt private static final int EGL_GL_COLORSPACE_KHR = 0x309D; - // https://www.khronos.org/registry/EGL/extensions/EXT/EGL_EXT_gl_colorspace_bt2020_linear.txt - private static final int EGL_GL_COLORSPACE_BT2020_PQ_EXT = 0x3340; private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_NONE = new int[] {EGL14.EGL_NONE}; - private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ = - new int[] {EGL_GL_COLORSPACE_KHR, EGL_GL_COLORSPACE_BT2020_PQ_EXT, EGL14.EGL_NONE}; private static final int[] EGL_CONFIG_ATTRIBUTES_RGBA_8888 = new int[] { EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, @@ -213,19 +209,19 @@ public final class GlUtil { /** * Returns a new {@link EGLSurface} wrapping the specified {@code surface}, for HDR rendering with - * Rec. 2020 color primaries and using the PQ transfer function. + * Rec. 2020 color primaries. * * @param eglDisplay The {@link EGLDisplay} to attach the surface to. * @param surface The surface to wrap; must be a surface, surface texture or surface holder. */ @RequiresApi(17) - public static EGLSurface getEglSurfaceBt2020Pq(EGLDisplay eglDisplay, Object surface) + public static EGLSurface getEglSurfaceBt2020(EGLDisplay eglDisplay, Object surface) throws GlException { return Api17.getEglSurface( eglDisplay, surface, EGL_CONFIG_ATTRIBUTES_RGBA_1010102, - EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ); + EGL_WINDOW_SURFACE_ATTRIBUTES_NONE); } /** @@ -277,22 +273,22 @@ public final class GlUtil { /** * Creates and focuses a new {@link EGLSurface} wrapping a 1x1 pixel buffer, for HDR rendering - * with Rec. 2020 color primaries and using the PQ transfer function. + * with Rec. 2020 color primaries. * * @param eglContext The {@link EGLContext} to make current. * @param eglDisplay The {@link EGLDisplay} to attach the surface to. */ @RequiresApi(17) - public static void focusPlaceholderEglSurfaceBt2020Pq( - EGLContext eglContext, EGLDisplay eglDisplay) throws GlException { + public static void focusPlaceholderEglSurfaceBt2020(EGLContext eglContext, EGLDisplay eglDisplay) + throws GlException { int[] pbufferAttributes = new int[] { EGL14.EGL_WIDTH, /* width= */ 1, EGL14.EGL_HEIGHT, /* height= */ 1, + // TODO(b/227624622): Figure out if we can remove the EGL_GL_COLORSPACE_KHR item. EGL_GL_COLORSPACE_KHR, - EGL_GL_COLORSPACE_BT2020_PQ_EXT, EGL14.EGL_NONE }; EGLSurface eglSurface = diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index c3048f4368..df4bcb1393 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -34,6 +34,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Log; +import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; @@ -254,6 +255,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo); } + MediaFormatUtil.maybeSetColorInfo(mediaFormat, format.colorInfo); mediaFormat.setInteger( MediaFormat.KEY_COLOR_FORMAT, supportedVideoEncoderSettings.colorProfile); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java index 693f5533bc..a2f96ef63f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java @@ -49,19 +49,14 @@ import java.io.IOException; /** * Creates a new instance. * - * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. + * @param useHdr Whether to process the input as an HDR signal. * @throws FrameProcessingException If a problem occurs while reading shader files. */ - public ExternalTextureProcessor(Context context, boolean enableExperimentalHdrEditing) - throws FrameProcessingException { + public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException { String vertexShaderFilePath = - enableExperimentalHdrEditing - ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH - : VERTEX_SHADER_TEX_TRANSFORM_PATH; + useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH; String fragmentShaderFilePath = - enableExperimentalHdrEditing - ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH - : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; try { glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); } catch (IOException | GlUtil.GlException e) { @@ -72,7 +67,7 @@ import java.io.IOException; "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - if (enableExperimentalHdrEditing) { + if (useHdr) { // In HDR editing mode the decoder output is sampled in YUV. glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java index 44b890d903..1be2c7ea98 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java @@ -61,7 +61,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final long streamOffsetUs; private final DebugViewProvider debugViewProvider; private final FrameProcessor.Listener frameProcessorListener; - private final boolean enableExperimentalHdrEditing; + private final boolean useHdr; private int inputWidth; private int inputHeight; @@ -86,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, FrameProcessor.Listener frameProcessorListener, DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing) { + boolean useHdr) { this.context = context; this.matrixTransformations = matrixTransformations; this.eglDisplay = eglDisplay; @@ -94,7 +94,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.streamOffsetUs = streamOffsetUs; this.debugViewProvider = debugViewProvider; this.frameProcessorListener = frameProcessorListener; - this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; + this.useHdr = useHdr; } /** @@ -194,9 +194,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; @Nullable EGLSurface outputEglSurface = this.outputEglSurface; if (outputEglSurface == null) { // This means that outputSurfaceInfo changed. - if (enableExperimentalHdrEditing) { - // TODO(b/227624622): Don't assume BT.2020 PQ input/output. - outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface); + if (useHdr) { + outputEglSurface = GlUtil.getEglSurfaceBt2020(eglDisplay, outputSurfaceInfo.surface); } else { outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface); } @@ -207,8 +206,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputSurfaceInfo.width, outputSurfaceInfo.height); if (debugSurfaceView != null) { debugSurfaceViewWrapper = - new SurfaceViewWrapper( - eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView); + new SurfaceViewWrapper(eglDisplay, eglContext, useHdr, debugSurfaceView); } if (matrixTransformationProcessor != null) { matrixTransformationProcessor.release(); @@ -281,7 +279,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final class SurfaceViewWrapper implements SurfaceHolder.Callback { private final EGLDisplay eglDisplay; private final EGLContext eglContext; - private final boolean enableExperimentalHdrEditing; + private final boolean useHdr; @GuardedBy("this") @Nullable @@ -295,13 +293,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private int height; public SurfaceViewWrapper( - EGLDisplay eglDisplay, - EGLContext eglContext, - boolean enableExperimentalHdrEditing, - SurfaceView surfaceView) { + EGLDisplay eglDisplay, EGLContext eglContext, boolean useHdr, SurfaceView surfaceView) { this.eglDisplay = eglDisplay; this.eglContext = eglContext; - this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; + this.useHdr = useHdr; surfaceView.getHolder().addCallback(this); surface = surfaceView.getHolder().getSurface(); width = surfaceView.getWidth(); @@ -321,8 +316,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } if (eglSurface == null) { - if (enableExperimentalHdrEditing) { - eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, surface); + if (useHdr) { + eglSurface = GlUtil.getEglSurfaceBt2020(eglDisplay, surface); } else { eglSurface = GlUtil.getEglSurface(eglDisplay, surface); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index ce418d8281..d01d690c6e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -53,7 +53,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param listener A {@link Listener}. * @param effects The {@link GlEffect GlEffects} to apply to each frame. * @param debugViewProvider A {@link DebugViewProvider}. - * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. + * @param useHdr Whether to process the input as an HDR signal. * @return A new instance. * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while * creating and configuring the OpenGL components. @@ -64,7 +64,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, List effects, DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing) + boolean useHdr) throws FrameProcessingException { ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); @@ -78,7 +78,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, effects, debugViewProvider, - enableExperimentalHdrEditing, + useHdr, singleThreadExecutorService)); try { @@ -106,23 +106,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, List effects, DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing, + boolean useHdr, ExecutorService singleThreadExecutorService) throws GlUtil.GlException, FrameProcessingException { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); EGLDisplay eglDisplay = GlUtil.createEglDisplay(); EGLContext eglContext = - enableExperimentalHdrEditing + useHdr ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay) : GlUtil.createEglContext(eglDisplay); if (GlUtil.isSurfacelessContextExtensionSupported()) { GlUtil.focusEglSurface( eglDisplay, eglContext, EGL14.EGL_NO_SURFACE, /* width= */ 1, /* height= */ 1); - } else if (enableExperimentalHdrEditing) { - // TODO(b/227624622): Don't assume BT.2020 PQ input/output. - GlUtil.focusPlaceholderEglSurfaceBt2020Pq(eglContext, eglDisplay); + } else if (useHdr) { + GlUtil.focusPlaceholderEglSurfaceBt2020(eglContext, eglDisplay); } else { GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); } @@ -137,13 +136,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, listener, debugViewProvider, - enableExperimentalHdrEditing); + useHdr); ImmutableList intermediateTextureProcessors = textureProcessors.first; FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper = textureProcessors.second; ExternalTextureProcessor externalTextureProcessor = - new ExternalTextureProcessor(context, enableExperimentalHdrEditing); + new ExternalTextureProcessor(context, useHdr); FrameProcessingTaskExecutor frameProcessingTaskExecutor = new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); chainTextureProcessorsWithListeners( @@ -182,7 +181,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, FrameProcessor.Listener listener, DebugViewProvider debugViewProvider, - boolean enableExperimentalHdrEditing) + boolean useHdr) throws FrameProcessingException { ImmutableList.Builder textureProcessorListBuilder = new ImmutableList.Builder<>(); @@ -213,7 +212,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, listener, debugViewProvider, - enableExperimentalHdrEditing)); + useHdr)); } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 2bd3311656..5e1b6ddd12 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -24,6 +24,7 @@ import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; @@ -97,6 +98,10 @@ import org.checkerframework.dataflow.qual.Pure; transformationRequest, fallbackListener); + // TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly + // processed as SDR, because the inputFormat.colorInfo reports the wrong value. + boolean useHdr = transformationRequest.enableHdrEditing && isHdr(inputFormat.colorInfo); + try { frameProcessor = GlEffectsFrameProcessor.create( @@ -131,7 +136,7 @@ import org.checkerframework.dataflow.qual.Pure; streamOffsetUs, effectsListBuilder.build(), debugViewProvider, - transformationRequest.enableHdrEditing); + useHdr); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessingException( e, TransformationException.ERROR_CODE_GL_INIT_FAILED); @@ -147,6 +152,11 @@ import org.checkerframework.dataflow.qual.Pure; maxPendingFrameCount = decoder.getMaxPendingFrameCount(); } + /** Whether this is a supported HDR format. */ + private static boolean isHdr(@Nullable ColorInfo colorInfo) { + return colorInfo != null && colorInfo.colorTransfer != C.COLOR_TRANSFER_SDR; + } + @Override @Nullable public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { @@ -352,6 +362,7 @@ import org.checkerframework.dataflow.qual.Pure; transformationRequest.videoMimeType != null ? transformationRequest.videoMimeType : inputFormat.sampleMimeType) + .setColorInfo(inputFormat.colorInfo) .build(); encoder =