loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDR: Configure GL shaders and encoder.

Browse Source 
Configure the GL shaders and encoder to take in HDR metadata.

This mostly just consists of passing the Format.colorInfo through
the VideoTranscodingSamplePipeline down to the encoder, rather than passing
the PQ-ness down to the GL step.

Due to b/237674316, this will remove HDR10+ support temporarily to introduce
support for HLG10.

Manually tested to confirm that HLG10 operations that don't affect color display
correctly after this CL with "HDR editing" in the demo checked, and continue to display incorrectly (as before this CL) without the option unchecked.

PiperOrigin-RevId: 458490810
This commit is contained in:
huangdarwin 2022-07-01 16:57:43 +00:00 committed by Marc Baechinger
parent deea5c927a
commit a0870a42be
6 changed files with 48 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
View File

@ -66,12 +66,8 @@ public final class GlUtil {
// https://www.khronos.org/registry/EGL/extensions/KHR/EGL_KHR_gl_colorspace.txt // https://www.khronos.org/registry/EGL/extensions/KHR/EGL_KHR_gl_colorspace.txt
private static final int EGL_GL_COLORSPACE_KHR = 0x309D; private static final int EGL_GL_COLORSPACE_KHR = 0x309D;
// https://www.khronos.org/registry/EGL/extensions/EXT/EGL_EXT_gl_colorspace_bt2020_linear.txt
private static final int EGL_GL_COLORSPACE_BT2020_PQ_EXT = 0x3340;
private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_NONE = new int[] {EGL14.EGL_NONE}; private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_NONE = new int[] {EGL14.EGL_NONE};
private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ =
new int[] {EGL_GL_COLORSPACE_KHR, EGL_GL_COLORSPACE_BT2020_PQ_EXT, EGL14.EGL_NONE};
private static final int[] EGL_CONFIG_ATTRIBUTES_RGBA_8888 = private static final int[] EGL_CONFIG_ATTRIBUTES_RGBA_8888 =
new int[] { new int[] {
EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
@ -213,19 +209,19 @@ public final class GlUtil {
/** /**
* Returns a new {@link EGLSurface} wrapping the specified {@code surface}, for HDR rendering with * Returns a new {@link EGLSurface} wrapping the specified {@code surface}, for HDR rendering with
* Rec. 2020 color primaries and using the PQ transfer function. * Rec. 2020 color primaries.
* *
* @param eglDisplay The {@link EGLDisplay} to attach the surface to. * @param eglDisplay The {@link EGLDisplay} to attach the surface to.
* @param surface The surface to wrap; must be a surface, surface texture or surface holder. * @param surface The surface to wrap; must be a surface, surface texture or surface holder.
*/ */
@RequiresApi(17) @RequiresApi(17)
public static EGLSurface getEglSurfaceBt2020Pq(EGLDisplay eglDisplay, Object surface) public static EGLSurface getEglSurfaceBt2020(EGLDisplay eglDisplay, Object surface)
throws GlException { throws GlException {
return Api17.getEglSurface( return Api17.getEglSurface(
eglDisplay, eglDisplay,
surface, surface,
EGL_CONFIG_ATTRIBUTES_RGBA_1010102, EGL_CONFIG_ATTRIBUTES_RGBA_1010102,
EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ); EGL_WINDOW_SURFACE_ATTRIBUTES_NONE);
} }
/** /**
@ -277,22 +273,22 @@ public final class GlUtil {
/** /**
* Creates and focuses a new {@link EGLSurface} wrapping a 1x1 pixel buffer, for HDR rendering * Creates and focuses a new {@link EGLSurface} wrapping a 1x1 pixel buffer, for HDR rendering
* with Rec. 2020 color primaries and using the PQ transfer function. * with Rec. 2020 color primaries.
* *
* @param eglContext The {@link EGLContext} to make current. * @param eglContext The {@link EGLContext} to make current.
* @param eglDisplay The {@link EGLDisplay} to attach the surface to. * @param eglDisplay The {@link EGLDisplay} to attach the surface to.
*/ */
@RequiresApi(17) @RequiresApi(17)
public static void focusPlaceholderEglSurfaceBt2020Pq( public static void focusPlaceholderEglSurfaceBt2020(EGLContext eglContext, EGLDisplay eglDisplay)
EGLContext eglContext, EGLDisplay eglDisplay) throws GlException { throws GlException {
int[] pbufferAttributes = int[] pbufferAttributes =
new int[] { new int[] {
EGL14.EGL_WIDTH, EGL14.EGL_WIDTH,
/* width= */ 1, /* width= */ 1,
EGL14.EGL_HEIGHT, EGL14.EGL_HEIGHT,
/* height= */ 1, /* height= */ 1,
// TODO(b/227624622): Figure out if we can remove the EGL_GL_COLORSPACE_KHR item.
EGL_GL_COLORSPACE_KHR, EGL_GL_COLORSPACE_KHR,
EGL_GL_COLORSPACE_BT2020_PQ_EXT,
EGL14.EGL_NONE EGL14.EGL_NONE
}; };
EGLSurface eglSurface = EGLSurface eglSurface =

2
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
View File

@ -34,6 +34,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes; import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Log; import androidx.media3.common.util.Log;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
@ -254,6 +255,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo); adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo);
} }
MediaFormatUtil.maybeSetColorInfo(mediaFormat, format.colorInfo);
mediaFormat.setInteger( mediaFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, supportedVideoEncoderSettings.colorProfile); MediaFormat.KEY_COLOR_FORMAT, supportedVideoEncoderSettings.colorProfile);

15
libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java
View File

@ -49,19 +49,14 @@ import java.io.IOException;
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @param useHdr Whether to process the input as an HDR signal.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public ExternalTextureProcessor(Context context, boolean enableExperimentalHdrEditing) public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException {
throws FrameProcessingException {
String vertexShaderFilePath = String vertexShaderFilePath =
enableExperimentalHdrEditing useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH;
? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH
: VERTEX_SHADER_TEX_TRANSFORM_PATH;
String fragmentShaderFilePath = String fragmentShaderFilePath =
enableExperimentalHdrEditing useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
try { try {
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) { } catch (IOException | GlUtil.GlException e) {
@ -72,7 +67,7 @@ import java.io.IOException;
"aFramePosition", "aFramePosition",
GlUtil.getNormalizedCoordinateBounds(), GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
if (enableExperimentalHdrEditing) { if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV. // In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
} }

27
libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java
View File

@ -61,7 +61,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final long streamOffsetUs; private final long streamOffsetUs;
private final DebugViewProvider debugViewProvider; private final DebugViewProvider debugViewProvider;
private final FrameProcessor.Listener frameProcessorListener; private final FrameProcessor.Listener frameProcessorListener;
private final boolean enableExperimentalHdrEditing; private final boolean useHdr;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
@ -86,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs, long streamOffsetUs,
FrameProcessor.Listener frameProcessorListener, FrameProcessor.Listener frameProcessorListener,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing) { boolean useHdr) {
this.context = context; this.context = context;
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
@ -94,7 +94,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.streamOffsetUs = streamOffsetUs; this.streamOffsetUs = streamOffsetUs;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
this.frameProcessorListener = frameProcessorListener; this.frameProcessorListener = frameProcessorListener;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; this.useHdr = useHdr;
} }
/** /**
@ -194,9 +194,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
@Nullable EGLSurface outputEglSurface = this.outputEglSurface; @Nullable EGLSurface outputEglSurface = this.outputEglSurface;
if (outputEglSurface == null) { // This means that outputSurfaceInfo changed. if (outputEglSurface == null) { // This means that outputSurfaceInfo changed.
if (enableExperimentalHdrEditing) { if (useHdr) {
// TODO(b/227624622): Don't assume BT.2020 PQ input/output. outputEglSurface = GlUtil.getEglSurfaceBt2020(eglDisplay, outputSurfaceInfo.surface);
outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface);
} else { } else {
outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface); outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface);
} }
@ -207,8 +206,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSurfaceInfo.width, outputSurfaceInfo.height); outputSurfaceInfo.width, outputSurfaceInfo.height);
if (debugSurfaceView != null) { if (debugSurfaceView != null) {
debugSurfaceViewWrapper = debugSurfaceViewWrapper =
new SurfaceViewWrapper( new SurfaceViewWrapper(eglDisplay, eglContext, useHdr, debugSurfaceView);
eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView);
} }
if (matrixTransformationProcessor != null) { if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release(); matrixTransformationProcessor.release();
@ -281,7 +279,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final class SurfaceViewWrapper implements SurfaceHolder.Callback { private static final class SurfaceViewWrapper implements SurfaceHolder.Callback {
private final EGLDisplay eglDisplay; private final EGLDisplay eglDisplay;
private final EGLContext eglContext; private final EGLContext eglContext;
private final boolean enableExperimentalHdrEditing; private final boolean useHdr;
@GuardedBy("this") @GuardedBy("this")
@Nullable @Nullable
@ -295,13 +293,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private int height; private int height;
public SurfaceViewWrapper( public SurfaceViewWrapper(
EGLDisplay eglDisplay, EGLDisplay eglDisplay, EGLContext eglContext, boolean useHdr, SurfaceView surfaceView) {
EGLContext eglContext,
boolean enableExperimentalHdrEditing,
SurfaceView surfaceView) {
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; this.useHdr = useHdr;
surfaceView.getHolder().addCallback(this); surfaceView.getHolder().addCallback(this);
surface = surfaceView.getHolder().getSurface(); surface = surfaceView.getHolder().getSurface();
width = surfaceView.getWidth(); width = surfaceView.getWidth();
@ -321,8 +316,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
if (eglSurface == null) { if (eglSurface == null) {
if (enableExperimentalHdrEditing) { if (useHdr) {
eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, surface); eglSurface = GlUtil.getEglSurfaceBt2020(eglDisplay, surface);
} else { } else {
eglSurface = GlUtil.getEglSurface(eglDisplay, surface); eglSurface = GlUtil.getEglSurface(eglDisplay, surface);
} }

23
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
View File

@ -53,7 +53,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param listener A {@link Listener}. * @param listener A {@link Listener}.
* @param effects The {@link GlEffect GlEffects} to apply to each frame. * @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}. * @param debugViewProvider A {@link DebugViewProvider}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @param useHdr Whether to process the input as an HDR signal.
* @return A new instance. * @return A new instance.
* @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while
* creating and configuring the OpenGL components. * creating and configuring the OpenGL components.
@ -64,7 +64,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs, long streamOffsetUs,
List<GlEffect> effects, List<GlEffect> effects,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing) boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
@ -78,7 +78,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs, streamOffsetUs,
effects, effects,
debugViewProvider, debugViewProvider,
enableExperimentalHdrEditing, useHdr,
singleThreadExecutorService)); singleThreadExecutorService));
try { try {
@ -106,23 +106,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs, long streamOffsetUs,
List<GlEffect> effects, List<GlEffect> effects,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing, boolean useHdr,
ExecutorService singleThreadExecutorService) ExecutorService singleThreadExecutorService)
throws GlUtil.GlException, FrameProcessingException { throws GlUtil.GlException, FrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME)); checkState(Thread.currentThread().getName().equals(THREAD_NAME));
EGLDisplay eglDisplay = GlUtil.createEglDisplay(); EGLDisplay eglDisplay = GlUtil.createEglDisplay();
EGLContext eglContext = EGLContext eglContext =
enableExperimentalHdrEditing useHdr
? GlUtil.createEglContextEs3Rgba1010102(eglDisplay) ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay)
: GlUtil.createEglContext(eglDisplay); : GlUtil.createEglContext(eglDisplay);
if (GlUtil.isSurfacelessContextExtensionSupported()) { if (GlUtil.isSurfacelessContextExtensionSupported()) {
GlUtil.focusEglSurface( GlUtil.focusEglSurface(
eglDisplay, eglContext, EGL14.EGL_NO_SURFACE, /* width= */ 1, /* height= */ 1); eglDisplay, eglContext, EGL14.EGL_NO_SURFACE, /* width= */ 1, /* height= */ 1);
} else if (enableExperimentalHdrEditing) { } else if (useHdr) {
// TODO(b/227624622): Don't assume BT.2020 PQ input/output. GlUtil.focusPlaceholderEglSurfaceBt2020(eglContext, eglDisplay);
GlUtil.focusPlaceholderEglSurfaceBt2020Pq(eglContext, eglDisplay);
} else { } else {
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
} }
@ -137,13 +136,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs, streamOffsetUs,
listener, listener,
debugViewProvider, debugViewProvider,
enableExperimentalHdrEditing); useHdr);
ImmutableList<GlTextureProcessor> intermediateTextureProcessors = textureProcessors.first; ImmutableList<GlTextureProcessor> intermediateTextureProcessors = textureProcessors.first;
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper = FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper =
textureProcessors.second; textureProcessors.second;
ExternalTextureProcessor externalTextureProcessor = ExternalTextureProcessor externalTextureProcessor =
new ExternalTextureProcessor(context, enableExperimentalHdrEditing); new ExternalTextureProcessor(context, useHdr);
FrameProcessingTaskExecutor frameProcessingTaskExecutor = FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
chainTextureProcessorsWithListeners( chainTextureProcessorsWithListeners(
@ -182,7 +181,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs, long streamOffsetUs,
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing) boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder = ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
new ImmutableList.Builder<>(); new ImmutableList.Builder<>();
@ -213,7 +212,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs, streamOffsetUs,
listener, listener,
debugViewProvider, debugViewProvider,
enableExperimentalHdrEditing)); useHdr));
} }
/** /**

13
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -24,6 +24,7 @@ import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.decoder.DecoderInputBuffer;
@ -97,6 +98,10 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest, transformationRequest,
fallbackListener); fallbackListener);
// TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly
// processed as SDR, because the inputFormat.colorInfo reports the wrong value.
boolean useHdr = transformationRequest.enableHdrEditing && isHdr(inputFormat.colorInfo);
try { try {
frameProcessor = frameProcessor =
GlEffectsFrameProcessor.create( GlEffectsFrameProcessor.create(
@ -131,7 +136,7 @@ import org.checkerframework.dataflow.qual.Pure;
streamOffsetUs, streamOffsetUs,
effectsListBuilder.build(), effectsListBuilder.build(),
debugViewProvider, debugViewProvider,
transformationRequest.enableHdrEditing); useHdr);
} catch (FrameProcessingException e) { } catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessingException( throw TransformationException.createForFrameProcessingException(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED); e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
@ -147,6 +152,11 @@ import org.checkerframework.dataflow.qual.Pure;
maxPendingFrameCount = decoder.getMaxPendingFrameCount(); maxPendingFrameCount = decoder.getMaxPendingFrameCount();
} }
/** Whether this is a supported HDR format. */
private static boolean isHdr(@Nullable ColorInfo colorInfo) {
return colorInfo != null && colorInfo.colorTransfer != C.COLOR_TRANSFER_SDR;
}
@Override @Override
@Nullable @Nullable
public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {
@ -352,6 +362,7 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest.videoMimeType != null transformationRequest.videoMimeType != null
? transformationRequest.videoMimeType ? transformationRequest.videoMimeType
: inputFormat.sampleMimeType) : inputFormat.sampleMimeType)
.setColorInfo(inputFormat.colorInfo)
.build(); .build();
encoder = encoder =