From c6b51003d3f1bb25f9b4b93eb8efe77e7c23f7a8 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Mon, 8 Jan 2024 10:06:10 -0800 Subject: [PATCH] Effect: Allow updating inputColorInfo between streams in VFP. Whenever the inputColorInfo updates, update the samplingGlShaderProgram. Also, allow either SDR or gamma2.2 to be used for HDR->SDR tone-mapping `outputColorInfo` request. This is required because we can't update the `outputColorInfo`, but plan to always use gamma2.2 for `outputColorInfo` in the future. This allows VideoFrameProcessor to work as is for exoplayer previewing, but only when not seeking. As we haven't plumbed the per-stream inputColorInfo from ExoPlayer down to VFP.registerInputStream, follow-up CLs will be needed to properly support previewing with changing inputColorInfo. PiperOrigin-RevId: 596627890 --- .../media3/effect/DefaultShaderProgram.java | 8 +++-- .../effect/DefaultVideoFrameProcessor.java | 30 +++++++++---------- .../androidx/media3/effect/InputSwitcher.java | 27 ++++++++++++----- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java index 1ec8192eec..92f34d7a3f 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java @@ -379,8 +379,12 @@ import java.util.List; glProgram.setIntUniform( "uApplyHdrToSdrToneMapping", /* value= */ (outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020) ? GL_TRUE : GL_FALSE); - checkArgument( - outputColorTransfer != Format.NO_VALUE && outputColorTransfer != C.COLOR_TRANSFER_SDR); + checkArgument(outputColorTransfer != Format.NO_VALUE); + if (outputColorTransfer == C.COLOR_TRANSFER_SDR) { + // When tone-mapping from HDR to SDR, COLOR_TRANSFER_SDR is interpreted as + // COLOR_TRANSFER_GAMMA_2_2. + outputColorTransfer = C.COLOR_TRANSFER_GAMMA_2_2; + } glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer); } else { glProgram.setIntUniform("uEnableColorTransfer", enableColorTransfers ? GL_TRUE : GL_FALSE); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index ed926ee4ff..a3639842db 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -310,8 +310,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final boolean enableColorTransfers; private final ColorInfo outputColorInfo; - private @MonotonicNonNull ColorInfo firstInputColorInfo; - private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; private volatile boolean inputStreamEnded; @@ -438,6 +436,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * GLES30#GL_HALF_FLOAT}. Otherwise, textures will use {@link GLES20#GL_RGBA} and {@link * GLES20#GL_UNSIGNED_BYTE}. * + *

If {@linkplain FrameInfo#colorInfo input color} {@linkplain ColorInfo#isTransferHdr is HDR}, + * but {@code outputColorInfo} is SDR, then HDR to SDR tone-mapping is applied, and {@code + * outputColorInfo}'s {@link ColorInfo#colorTransfer} must be {@link C#COLOR_TRANSFER_GAMMA_2_2} + * or {@link C#COLOR_TRANSFER_SDR}. In this case, the actual output transfer function will be in + * {@link C#COLOR_TRANSFER_GAMMA_2_2}, for consistency with other tone-mapping and color behavior + * in the Android ecosystem (for example, MediaFormat's COLOR_TRANSFER_SDR_VIDEO is defined as + * SMPTE 170M, but most OEMs process it as Gamma 2.2). + * *

If either {@link FrameInfo#colorInfo} or {@code outputColorInfo} {@linkplain * ColorInfo#isTransferHdr} are HDR}, color transfers must {@linkplain * Factory.Builder#setEnableColorTransfers be enabled}. @@ -445,7 +451,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { *

The {@link FrameInfo}'s {@link ColorInfo} must not change between different calls to this * method. */ - // TODO: b/307952514: After FrameInfo.colorInfo may change between calls, remove relevant javadoc. + // TODO: b/307952514: After updating frameInfo.colorInfo works with flushing, remove relevant + // javadoc. @Override public void registerInputStream( @InputType int inputType, List effects, FrameInfo frameInfo) { @@ -805,14 +812,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { */ private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure) throws VideoFrameProcessingException { - // TODO: b/307952514 - Remove this color check, and reinitialize the InputSwitcher's - // samplingGlShaderProgram instead. - checkState( - firstInputColorInfo == null - || firstInputColorInfo.equals(inputStreamInfo.frameInfo.colorInfo)); - if (inputStreamInfo.frameInfo.colorInfo != null) { - firstInputColorInfo = inputStreamInfo.frameInfo.colorInfo; - } checkColors( /* inputColorInfo= */ inputStreamInfo.frameInfo.colorInfo, outputColorInfo, @@ -877,14 +876,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { if (inputColorInfo.colorSpace != outputColorInfo.colorSpace || ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) { - // OpenGL tone mapping is only implemented for BT2020 to BT709 and HDR to SDR (Gamma 2.2). - // Gamma 2.2 is used instead of SMPTE 170M for SDR, despite MediaFormat's - // COLOR_TRANSFER_SDR_VIDEO being defined as SMPTE 170M. This is to match - // other known tone-mapping behavior within the Android ecosystem. + // OpenGL tone mapping is only implemented for BT2020 to BT709 and HDR to SDR. checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020); checkArgument(outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020); checkArgument(ColorInfo.isTransferHdr(inputColorInfo)); - checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2); + checkArgument( + outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2 + || outputColorInfo.colorTransfer == C.COLOR_TRANSFER_SDR); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index 3c4a4ce6c8..305b578b16 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -154,7 +154,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; * @param inputFrameInfo The {@link FrameInfo} associated with the new input. */ public void switchToInput( - @VideoFrameProcessor.InputType int newInputType, FrameInfo inputFrameInfo) + @VideoFrameProcessor.InputType int newInputType, FrameInfo newInputFrameInfo) throws VideoFrameProcessingException { checkStateNotNull(downstreamShaderProgram); checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType); @@ -163,11 +163,11 @@ import org.checkerframework.checker.nullness.qual.Nullable; @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); Input input = inputs.get(inputType); if (inputType == newInputType) { - if (input.getSamplingGlShaderProgram() == null) { - // TODO: b/307952514 - When switchToInput is called, and the inputColorInfo doesn't match - // the prior inputColorInfo, recreate and reinitialize the input.samplingGlShaderProgram. + if (input.getInputColorInfo() == null + || !newInputFrameInfo.colorInfo.equals(input.getInputColorInfo())) { input.setSamplingGlShaderProgram( - createSamplingShaderProgram(inputFrameInfo.colorInfo, newInputType)); + createSamplingShaderProgram(newInputFrameInfo.colorInfo, newInputType)); + input.setInputColorInfo(newInputFrameInfo.colorInfo); } input.setChainingListener( new GatedChainingListenerWrapper( @@ -182,7 +182,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; input.setActive(false); } } - checkNotNull(activeTextureManager).setInputFrameInfo(inputFrameInfo); + checkNotNull(activeTextureManager).setInputFrameInfo(newInputFrameInfo); } /** Returns whether the {@code InputSwitcher} is connected to an active input. */ @@ -248,18 +248,27 @@ import org.checkerframework.checker.nullness.qual.Nullable; public final TextureManager textureManager; private @MonotonicNonNull ExternalShaderProgram samplingGlShaderProgram; + private @MonotonicNonNull ColorInfo inputColorInfo; private @MonotonicNonNull GatedChainingListenerWrapper gatedChainingListenerWrapper; public Input(TextureManager textureManager) { this.textureManager = textureManager; } - public void setSamplingGlShaderProgram(ExternalShaderProgram samplingGlShaderProgram) { + public void setSamplingGlShaderProgram(ExternalShaderProgram samplingGlShaderProgram) + throws VideoFrameProcessingException { + if (this.samplingGlShaderProgram != null) { + this.samplingGlShaderProgram.release(); + } this.samplingGlShaderProgram = samplingGlShaderProgram; textureManager.setSamplingGlShaderProgram(samplingGlShaderProgram); samplingGlShaderProgram.setInputListener(textureManager); } + public void setInputColorInfo(ColorInfo inputColorInfo) { + this.inputColorInfo = inputColorInfo; + } + public void setChainingListener(GatedChainingListenerWrapper gatedChainingListenerWrapper) { this.gatedChainingListenerWrapper = gatedChainingListenerWrapper; checkNotNull(samplingGlShaderProgram).setOutputListener(gatedChainingListenerWrapper); @@ -269,6 +278,10 @@ import org.checkerframework.checker.nullness.qual.Nullable; return samplingGlShaderProgram; } + public @Nullable ColorInfo getInputColorInfo() { + return inputColorInfo; + } + public void setActive(boolean active) { if (gatedChainingListenerWrapper == null) { return;