loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDR: Configure GL shaders and encoder.

Browse Source 
Configure the GL shaders and encoder to take in HDR metadata.

This mostly just consists of passing the Format.colorInfo through
the VideoTranscodingSamplePipeline down to the encoder, rather than passing
the PQ-ness down to the GL step.

Due to b/237674316, this will remove HDR10+ support temporarily to introduce
support for HLG10.

Manually tested to confirm that HLG10 operations that don't affect color display
correctly after this CL with "HDR editing" in the demo checked, and continue to display incorrectly (as before this CL) without the option unchecked.

PiperOrigin-RevId: 458490810
This commit is contained in:
huangdarwin 2022-07-01 16:57:43 +00:00 committed by Marc Baechinger
parent deea5c927a
commit a0870a42be
6 changed files with 48 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
View File

@ -66,12 +66,8 @@ public final class GlUtil {
// https://www.khronos.org/registry/EGL/extensions/KHR/EGL_KHR_gl_colorspace.txt
private static final int EGL_GL_COLORSPACE_KHR = 0x309D;
// https://www.khronos.org/registry/EGL/extensions/EXT/EGL_EXT_gl_colorspace_bt2020_linear.txt
private static final int EGL_GL_COLORSPACE_BT2020_PQ_EXT = 0x3340;
private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_NONE = new int[] {EGL14.EGL_NONE};
private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ =
new int[] {EGL_GL_COLORSPACE_KHR, EGL_GL_COLORSPACE_BT2020_PQ_EXT, EGL14.EGL_NONE};
private static final int[] EGL_CONFIG_ATTRIBUTES_RGBA_8888 =
new int[] {
EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
@ -213,19 +209,19 @@ public final class GlUtil {
/**
* Returns a new {@link EGLSurface} wrapping the specified {@code surface}, for HDR rendering with
* Rec. 2020 color primaries and using the PQ transfer function.
* Rec. 2020 color primaries.
*
* @param eglDisplay The {@link EGLDisplay} to attach the surface to.
* @param surface The surface to wrap; must be a surface, surface texture or surface holder.
*/
@RequiresApi(17)
public static EGLSurface getEglSurfaceBt2020Pq(EGLDisplay eglDisplay, Object surface)
public static EGLSurface getEglSurfaceBt2020(EGLDisplay eglDisplay, Object surface)
throws GlException {
return Api17.getEglSurface(
eglDisplay,
surface,
EGL_CONFIG_ATTRIBUTES_RGBA_1010102,
EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ);
EGL_WINDOW_SURFACE_ATTRIBUTES_NONE);
}
/**
@ -277,22 +273,22 @@ public final class GlUtil {
/**
* Creates and focuses a new {@link EGLSurface} wrapping a 1x1 pixel buffer, for HDR rendering
* with Rec. 2020 color primaries and using the PQ transfer function.
* with Rec. 2020 color primaries.
*
* @param eglContext The {@link EGLContext} to make current.
* @param eglDisplay The {@link EGLDisplay} to attach the surface to.
*/
@RequiresApi(17)
public static void focusPlaceholderEglSurfaceBt2020Pq(
EGLContext eglContext, EGLDisplay eglDisplay) throws GlException {
public static void focusPlaceholderEglSurfaceBt2020(EGLContext eglContext, EGLDisplay eglDisplay)
throws GlException {
int[] pbufferAttributes =
new int[] {
EGL14.EGL_WIDTH,
/* width= */ 1,
EGL14.EGL_HEIGHT,
/* height= */ 1,
// TODO(b/227624622): Figure out if we can remove the EGL_GL_COLORSPACE_KHR item.
EGL_GL_COLORSPACE_KHR,
EGL_GL_COLORSPACE_BT2020_PQ_EXT,
EGL14.EGL_NONE
};
EGLSurface eglSurface =

2
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
View File

@ -34,6 +34,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
@ -254,6 +255,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo);
}
MediaFormatUtil.maybeSetColorInfo(mediaFormat, format.colorInfo);
mediaFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, supportedVideoEncoderSettings.colorProfile);

15
libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java
View File

@ -49,19 +49,14 @@ import java.io.IOException;
/**
* Creates a new instance.
*
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @param useHdr Whether to process the input as an HDR signal.
* @throws FrameProcessingException If a problem occurs while reading shader files.
*/
public ExternalTextureProcessor(Context context, boolean enableExperimentalHdrEditing)
throws FrameProcessingException {
public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException {
String vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH
: VERTEX_SHADER_TEX_TRANSFORM_PATH;
useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH;
String fragmentShaderFilePath =
enableExperimentalHdrEditing
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
try {
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) {
@ -72,7 +67,7 @@ import java.io.IOException;
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
if (enableExperimentalHdrEditing) {
if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}

27
libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java
View File

@ -61,7 +61,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final long streamOffsetUs;
private final DebugViewProvider debugViewProvider;
private final FrameProcessor.Listener frameProcessorListener;
private final boolean enableExperimentalHdrEditing;
private final boolean useHdr;
private int inputWidth;
private int inputHeight;
@ -86,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs,
FrameProcessor.Listener frameProcessorListener,
DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing) {
boolean useHdr) {
this.context = context;
this.matrixTransformations = matrixTransformations;
this.eglDisplay = eglDisplay;
@ -94,7 +94,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.streamOffsetUs = streamOffsetUs;
this.debugViewProvider = debugViewProvider;
this.frameProcessorListener = frameProcessorListener;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
this.useHdr = useHdr;
}
/**
@ -194,9 +194,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
@Nullable EGLSurface outputEglSurface = this.outputEglSurface;
if (outputEglSurface == null) { // This means that outputSurfaceInfo changed.
if (enableExperimentalHdrEditing) {
// TODO(b/227624622): Don't assume BT.2020 PQ input/output.
outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface);
if (useHdr) {
outputEglSurface = GlUtil.getEglSurfaceBt2020(eglDisplay, outputSurfaceInfo.surface);
} else {
outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface);
}
@ -207,8 +206,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSurfaceInfo.width, outputSurfaceInfo.height);
if (debugSurfaceView != null) {
debugSurfaceViewWrapper =
new SurfaceViewWrapper(
eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView);
new SurfaceViewWrapper(eglDisplay, eglContext, useHdr, debugSurfaceView);
}
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
@ -281,7 +279,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final class SurfaceViewWrapper implements SurfaceHolder.Callback {
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final boolean enableExperimentalHdrEditing;
private final boolean useHdr;
@GuardedBy("this")
@Nullable
@ -295,13 +293,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private int height;
public SurfaceViewWrapper(
EGLDisplay eglDisplay,
EGLContext eglContext,
boolean enableExperimentalHdrEditing,
SurfaceView surfaceView) {
EGLDisplay eglDisplay, EGLContext eglContext, boolean useHdr, SurfaceView surfaceView) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
this.useHdr = useHdr;
surfaceView.getHolder().addCallback(this);
surface = surfaceView.getHolder().getSurface();
width = surfaceView.getWidth();
@ -321,8 +316,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
if (eglSurface == null) {
if (enableExperimentalHdrEditing) {
eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, surface);
if (useHdr) {
eglSurface = GlUtil.getEglSurfaceBt2020(eglDisplay, surface);
} else {
eglSurface = GlUtil.getEglSurface(eglDisplay, surface);
}

23
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
View File

@ -53,7 +53,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param listener A {@link Listener}.
* @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @param useHdr Whether to process the input as an HDR signal.
* @return A new instance.
* @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while
* creating and configuring the OpenGL components.
@ -64,7 +64,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs,
List<GlEffect> effects,
DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing)
boolean useHdr)
throws FrameProcessingException {
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
@ -78,7 +78,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs,
effects,
debugViewProvider,
enableExperimentalHdrEditing,
useHdr,
singleThreadExecutorService));
try {
@ -106,23 +106,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs,
List<GlEffect> effects,
DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing,
boolean useHdr,
ExecutorService singleThreadExecutorService)
throws GlUtil.GlException, FrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
EGLContext eglContext =
enableExperimentalHdrEditing
useHdr
? GlUtil.createEglContextEs3Rgba1010102(eglDisplay)
: GlUtil.createEglContext(eglDisplay);
if (GlUtil.isSurfacelessContextExtensionSupported()) {
GlUtil.focusEglSurface(
eglDisplay, eglContext, EGL14.EGL_NO_SURFACE, /* width= */ 1, /* height= */ 1);
} else if (enableExperimentalHdrEditing) {
// TODO(b/227624622): Don't assume BT.2020 PQ input/output.
GlUtil.focusPlaceholderEglSurfaceBt2020Pq(eglContext, eglDisplay);
} else if (useHdr) {
GlUtil.focusPlaceholderEglSurfaceBt2020(eglContext, eglDisplay);
} else {
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
}
@ -137,13 +136,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs,
listener,
debugViewProvider,
enableExperimentalHdrEditing);
useHdr);
ImmutableList<GlTextureProcessor> intermediateTextureProcessors = textureProcessors.first;
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper =
textureProcessors.second;
ExternalTextureProcessor externalTextureProcessor =
new ExternalTextureProcessor(context, enableExperimentalHdrEditing);
new ExternalTextureProcessor(context, useHdr);
FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
chainTextureProcessorsWithListeners(
@ -182,7 +181,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs,
FrameProcessor.Listener listener,
DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing)
boolean useHdr)
throws FrameProcessingException {
ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
new ImmutableList.Builder<>();
@ -213,7 +212,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs,
listener,
debugViewProvider,
enableExperimentalHdrEditing));
useHdr));
}
/**

13
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -24,6 +24,7 @@ import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
@ -97,6 +98,10 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest,
fallbackListener);
// TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly
// processed as SDR, because the inputFormat.colorInfo reports the wrong value.
boolean useHdr = transformationRequest.enableHdrEditing && isHdr(inputFormat.colorInfo);
try {
frameProcessor =
GlEffectsFrameProcessor.create(
@ -131,7 +136,7 @@ import org.checkerframework.dataflow.qual.Pure;
streamOffsetUs,
effectsListBuilder.build(),
debugViewProvider,
transformationRequest.enableHdrEditing);
useHdr);
} catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessingException(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
@ -147,6 +152,11 @@ import org.checkerframework.dataflow.qual.Pure;
maxPendingFrameCount = decoder.getMaxPendingFrameCount();
}
/** Whether this is a supported HDR format. */
private static boolean isHdr(@Nullable ColorInfo colorInfo) {
return colorInfo != null && colorInfo.colorTransfer != C.COLOR_TRANSFER_SDR;
}
@Override
@Nullable
public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {
@ -352,6 +362,7 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest.videoMimeType != null
? transformationRequest.videoMimeType
: inputFormat.sampleMimeType)
.setColorInfo(inputFormat.colorInfo)
.build();
encoder =