loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Effect: Allow updating inputColorInfo between streams in VFP.

Browse Source 
Whenever the inputColorInfo updates, update the samplingGlShaderProgram.

Also, allow either SDR or gamma2.2 to be used for HDR->SDR tone-mapping
`outputColorInfo` request. This is required because we can't update the
`outputColorInfo`, but plan to always use gamma2.2 for `outputColorInfo` in the
future.

This allows VideoFrameProcessor to work as is for exoplayer previewing, but
only when not seeking. As we haven't plumbed the per-stream inputColorInfo from
ExoPlayer down to VFP.registerInputStream, follow-up CLs will be needed to
properly support previewing with changing inputColorInfo.

PiperOrigin-RevId: 596627890
This commit is contained in:
huangdarwin 2024-01-08 10:06:10 -08:00 committed by Copybara-Service
parent 77f311917f
commit c6b51003d3
3 changed files with 40 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java
View File

@ -379,8 +379,12 @@ import java.util.List;
glProgram.setIntUniform(
"uApplyHdrToSdrToneMapping",
/* value= */ (outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020) ? GL_TRUE : GL_FALSE);
checkArgument(
outputColorTransfer != Format.NO_VALUE && outputColorTransfer != C.COLOR_TRANSFER_SDR);
checkArgument(outputColorTransfer != Format.NO_VALUE);
if (outputColorTransfer == C.COLOR_TRANSFER_SDR) {
// When tone-mapping from HDR to SDR, COLOR_TRANSFER_SDR is interpreted as
// COLOR_TRANSFER_GAMMA_2_2.
outputColorTransfer = C.COLOR_TRANSFER_GAMMA_2_2;
}
glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
} else {
glProgram.setIntUniform("uEnableColorTransfer", enableColorTransfers ? GL_TRUE : GL_FALSE);

30
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -310,8 +310,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private final boolean enableColorTransfers;
private final ColorInfo outputColorInfo;
private @MonotonicNonNull ColorInfo firstInputColorInfo;
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
private volatile boolean inputStreamEnded;
@ -438,6 +436,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* GLES30#GL_HALF_FLOAT}. Otherwise, textures will use {@link GLES20#GL_RGBA} and {@link
* GLES20#GL_UNSIGNED_BYTE}.
*
* <p>If {@linkplain FrameInfo#colorInfo input color} {@linkplain ColorInfo#isTransferHdr is HDR},
* but {@code outputColorInfo} is SDR, then HDR to SDR tone-mapping is applied, and {@code
* outputColorInfo}'s {@link ColorInfo#colorTransfer} must be {@link C#COLOR_TRANSFER_GAMMA_2_2}
* or {@link C#COLOR_TRANSFER_SDR}. In this case, the actual output transfer function will be in
* {@link C#COLOR_TRANSFER_GAMMA_2_2}, for consistency with other tone-mapping and color behavior
* in the Android ecosystem (for example, MediaFormat's COLOR_TRANSFER_SDR_VIDEO is defined as
* SMPTE 170M, but most OEMs process it as Gamma 2.2).
*
* <p>If either {@link FrameInfo#colorInfo} or {@code outputColorInfo} {@linkplain
* ColorInfo#isTransferHdr} are HDR}, color transfers must {@linkplain
* Factory.Builder#setEnableColorTransfers be enabled}.
@ -445,7 +451,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* <p>The {@link FrameInfo}'s {@link ColorInfo} must not change between different calls to this
* method.
*/
// TODO: b/307952514: After FrameInfo.colorInfo may change between calls, remove relevant javadoc.
// TODO: b/307952514: After updating frameInfo.colorInfo works with flushing, remove relevant
// javadoc.
@Override
public void registerInputStream(
@InputType int inputType, List<Effect> effects, FrameInfo frameInfo) {
@ -805,14 +812,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
*/
private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure)
throws VideoFrameProcessingException {
// TODO: b/307952514 - Remove this color check, and reinitialize the InputSwitcher's
// samplingGlShaderProgram instead.
checkState(
firstInputColorInfo == null
|| firstInputColorInfo.equals(inputStreamInfo.frameInfo.colorInfo));
if (inputStreamInfo.frameInfo.colorInfo != null) {
firstInputColorInfo = inputStreamInfo.frameInfo.colorInfo;
}
checkColors(
/* inputColorInfo= */ inputStreamInfo.frameInfo.colorInfo,
outputColorInfo,
@ -877,14 +876,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
|| ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
// OpenGL tone mapping is only implemented for BT2020 to BT709 and HDR to SDR (Gamma 2.2).
// Gamma 2.2 is used instead of SMPTE 170M for SDR, despite MediaFormat's
// COLOR_TRANSFER_SDR_VIDEO being defined as SMPTE 170M. This is to match
// other known tone-mapping behavior within the Android ecosystem.
// OpenGL tone mapping is only implemented for BT2020 to BT709 and HDR to SDR.
checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
checkArgument(outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020);
checkArgument(ColorInfo.isTransferHdr(inputColorInfo));
checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2);
checkArgument(
outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2
|| outputColorInfo.colorTransfer == C.COLOR_TRANSFER_SDR);
}
}

27
libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java
View File

@ -154,7 +154,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
* @param inputFrameInfo The {@link FrameInfo} associated with the new input.
*/
public void switchToInput(
@VideoFrameProcessor.InputType int newInputType, FrameInfo inputFrameInfo)
@VideoFrameProcessor.InputType int newInputType, FrameInfo newInputFrameInfo)
throws VideoFrameProcessingException {
checkStateNotNull(downstreamShaderProgram);
checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType);
@ -163,11 +163,11 @@ import org.checkerframework.checker.nullness.qual.Nullable;
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
Input input = inputs.get(inputType);
if (inputType == newInputType) {
if (input.getSamplingGlShaderProgram() == null) {
// TODO: b/307952514 - When switchToInput is called, and the inputColorInfo doesn't match
// the prior inputColorInfo, recreate and reinitialize the input.samplingGlShaderProgram.
if (input.getInputColorInfo() == null
|| !newInputFrameInfo.colorInfo.equals(input.getInputColorInfo())) {
input.setSamplingGlShaderProgram(
createSamplingShaderProgram(inputFrameInfo.colorInfo, newInputType));
createSamplingShaderProgram(newInputFrameInfo.colorInfo, newInputType));
input.setInputColorInfo(newInputFrameInfo.colorInfo);
}
input.setChainingListener(
new GatedChainingListenerWrapper(
@ -182,7 +182,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
input.setActive(false);
}
}
checkNotNull(activeTextureManager).setInputFrameInfo(inputFrameInfo);
checkNotNull(activeTextureManager).setInputFrameInfo(newInputFrameInfo);
}
/** Returns whether the {@code InputSwitcher} is connected to an active input. */
@ -248,18 +248,27 @@ import org.checkerframework.checker.nullness.qual.Nullable;
public final TextureManager textureManager;
private @MonotonicNonNull ExternalShaderProgram samplingGlShaderProgram;
private @MonotonicNonNull ColorInfo inputColorInfo;
private @MonotonicNonNull GatedChainingListenerWrapper gatedChainingListenerWrapper;
public Input(TextureManager textureManager) {
this.textureManager = textureManager;
}
public void setSamplingGlShaderProgram(ExternalShaderProgram samplingGlShaderProgram) {
public void setSamplingGlShaderProgram(ExternalShaderProgram samplingGlShaderProgram)
throws VideoFrameProcessingException {
if (this.samplingGlShaderProgram != null) {
this.samplingGlShaderProgram.release();
}
this.samplingGlShaderProgram = samplingGlShaderProgram;
textureManager.setSamplingGlShaderProgram(samplingGlShaderProgram);
samplingGlShaderProgram.setInputListener(textureManager);
}
public void setInputColorInfo(ColorInfo inputColorInfo) {
this.inputColorInfo = inputColorInfo;
}
public void setChainingListener(GatedChainingListenerWrapper gatedChainingListenerWrapper) {
this.gatedChainingListenerWrapper = gatedChainingListenerWrapper;
checkNotNull(samplingGlShaderProgram).setOutputListener(gatedChainingListenerWrapper);
@ -269,6 +278,10 @@ import org.checkerframework.checker.nullness.qual.Nullable;
return samplingGlShaderProgram;
}
public @Nullable ColorInfo getInputColorInfo() {
return inputColorInfo;
}
public void setActive(boolean active) {
if (gatedChainingListenerWrapper == null) {
return;