Effect: Move VideoFrameProcessor inputColorInfo interface to FrameInfo.

Move `inputColorInfo` from `VideoFrameProcessor`'s `Factory.create` to `FrameInfo`,
input via `registerInputStream`.

Also, manually tested on exoplayer demo that setVideoEffects still works.

PiperOrigin-RevId: 591273545
This commit is contained in:
huangdarwin 2023-12-15 09:12:22 -08:00 committed by Copybara-Service
parent f47c4e33ec
commit 7579693739
12 changed files with 85 additions and 57 deletions

View File

@ -27,6 +27,7 @@ public class FrameInfo {
/** A builder for {@link FrameInfo} instances. */ /** A builder for {@link FrameInfo} instances. */
public static final class Builder { public static final class Builder {
private ColorInfo colorInfo;
private int width; private int width;
private int height; private int height;
private float pixelWidthHeightRatio; private float pixelWidthHeightRatio;
@ -35,10 +36,12 @@ public class FrameInfo {
/** /**
* Creates an instance with default values. * Creates an instance with default values.
* *
* @param colorInfo The {@link ColorInfo}.
* @param width The frame width, in pixels. * @param width The frame width, in pixels.
* @param height The frame height, in pixels. * @param height The frame height, in pixels.
*/ */
public Builder(int width, int height) { public Builder(ColorInfo colorInfo, int width, int height) {
this.colorInfo = colorInfo;
this.width = width; this.width = width;
this.height = height; this.height = height;
pixelWidthHeightRatio = 1; pixelWidthHeightRatio = 1;
@ -46,12 +49,20 @@ public class FrameInfo {
/** Creates an instance with the values of the provided {@link FrameInfo}. */ /** Creates an instance with the values of the provided {@link FrameInfo}. */
public Builder(FrameInfo frameInfo) { public Builder(FrameInfo frameInfo) {
colorInfo = frameInfo.colorInfo;
width = frameInfo.width; width = frameInfo.width;
height = frameInfo.height; height = frameInfo.height;
pixelWidthHeightRatio = frameInfo.pixelWidthHeightRatio; pixelWidthHeightRatio = frameInfo.pixelWidthHeightRatio;
offsetToAddUs = frameInfo.offsetToAddUs; offsetToAddUs = frameInfo.offsetToAddUs;
} }
/** Sets the {@link ColorInfo}. */
@CanIgnoreReturnValue
public Builder setColorInfo(ColorInfo colorInfo) {
this.colorInfo = colorInfo;
return this;
}
/** Sets the frame width, in pixels. */ /** Sets the frame width, in pixels. */
@CanIgnoreReturnValue @CanIgnoreReturnValue
public Builder setWidth(int width) { public Builder setWidth(int width) {
@ -91,10 +102,13 @@ public class FrameInfo {
/** Builds a {@link FrameInfo} instance. */ /** Builds a {@link FrameInfo} instance. */
public FrameInfo build() { public FrameInfo build() {
return new FrameInfo(width, height, pixelWidthHeightRatio, offsetToAddUs); return new FrameInfo(colorInfo, width, height, pixelWidthHeightRatio, offsetToAddUs);
} }
} }
/** The {@link ColorInfo} of the frame. */
public final ColorInfo colorInfo;
/** The width of the frame, in pixels. */ /** The width of the frame, in pixels. */
public final int width; public final int width;
@ -112,12 +126,12 @@ public class FrameInfo {
*/ */
public final long offsetToAddUs; public final long offsetToAddUs;
// TODO(b/227624622): Add color space information for HDR. private FrameInfo(
ColorInfo colorInfo, int width, int height, float pixelWidthHeightRatio, long offsetToAddUs) {
private FrameInfo(int width, int height, float pixelWidthHeightRatio, long offsetToAddUs) {
checkArgument(width > 0, "width must be positive, but is: " + width); checkArgument(width > 0, "width must be positive, but is: " + width);
checkArgument(height > 0, "height must be positive, but is: " + height); checkArgument(height > 0, "height must be positive, but is: " + height);
this.colorInfo = colorInfo;
this.width = width; this.width = width;
this.height = height; this.height = height;
this.pixelWidthHeightRatio = pixelWidthHeightRatio; this.pixelWidthHeightRatio = pixelWidthHeightRatio;

View File

@ -82,7 +82,6 @@ public interface VideoFrameProcessor {
* *
* @param context A {@link Context}. * @param context A {@link Context}.
* @param debugViewProvider A {@link DebugViewProvider}. * @param debugViewProvider A {@link DebugViewProvider}.
* @param inputColorInfo The {@link ColorInfo} for the input frames.
* @param outputColorInfo The {@link ColorInfo} for the output frames. * @param outputColorInfo The {@link ColorInfo} for the output frames.
* @param renderFramesAutomatically If {@code true}, the instance will render output frames to * @param renderFramesAutomatically If {@code true}, the instance will render output frames to
* the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as * the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as
@ -98,7 +97,6 @@ public interface VideoFrameProcessor {
VideoFrameProcessor create( VideoFrameProcessor create(
Context context, Context context,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
Executor listenerExecutor, Executor listenerExecutor,

View File

@ -100,7 +100,8 @@ public class DefaultVideoFrameProcessorTest {
defaultVideoFrameProcessor.registerInputStream( defaultVideoFrameProcessor.registerInputStream(
VideoFrameProcessor.INPUT_TYPE_BITMAP, VideoFrameProcessor.INPUT_TYPE_BITMAP,
ImmutableList.of(), ImmutableList.of(),
new FrameInfo.Builder(/* width= */ 100, /* height= */ 100).build()); new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, /* width= */ 100, /* height= */ 100)
.build());
assertThat(defaultVideoFrameProcessor.getPendingInputFrameCount()).isEqualTo(0); assertThat(defaultVideoFrameProcessor.getPendingInputFrameCount()).isEqualTo(0);
// Unblocks configuration. // Unblocks configuration.
@ -150,17 +151,20 @@ public class DefaultVideoFrameProcessorTest {
new InputStreamInfo( new InputStreamInfo(
VideoFrameProcessor.INPUT_TYPE_BITMAP, VideoFrameProcessor.INPUT_TYPE_BITMAP,
ImmutableList.of(), ImmutableList.of(),
new FrameInfo.Builder(/* width= */ 100, /* height= */ 100).build()); new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, /* width= */ 100, /* height= */ 100)
.build());
InputStreamInfo stream2 = InputStreamInfo stream2 =
new InputStreamInfo( new InputStreamInfo(
VideoFrameProcessor.INPUT_TYPE_BITMAP, VideoFrameProcessor.INPUT_TYPE_BITMAP,
ImmutableList.of(new Contrast(.5f)), ImmutableList.of(new Contrast(.5f)),
new FrameInfo.Builder(/* width= */ 200, /* height= */ 200).build()); new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, /* width= */ 200, /* height= */ 200)
.build());
InputStreamInfo stream3 = InputStreamInfo stream3 =
new InputStreamInfo( new InputStreamInfo(
VideoFrameProcessor.INPUT_TYPE_BITMAP, VideoFrameProcessor.INPUT_TYPE_BITMAP,
ImmutableList.of(), ImmutableList.of(),
new FrameInfo.Builder(/* width= */ 300, /* height= */ 300).build()); new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, /* width= */ 300, /* height= */ 300)
.build());
registerInputStream(defaultVideoFrameProcessor, stream1); registerInputStream(defaultVideoFrameProcessor, stream1);
registerInputStream(defaultVideoFrameProcessor, stream2); registerInputStream(defaultVideoFrameProcessor, stream2);
@ -179,7 +183,6 @@ public class DefaultVideoFrameProcessorTest {
.create( .create(
getApplicationContext(), getApplicationContext(),
DebugViewProvider.NONE, DebugViewProvider.NONE,
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* renderFramesAutomatically= */ true, /* renderFramesAutomatically= */ true,
/* listenerExecutor= */ MoreExecutors.directExecutor(), /* listenerExecutor= */ MoreExecutors.directExecutor(),

View File

@ -292,7 +292,6 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
.create( .create(
getApplicationContext(), getApplicationContext(),
DebugViewProvider.NONE, DebugViewProvider.NONE,
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
renderFramesAutomatically, renderFramesAutomatically,
MoreExecutors.directExecutor(), MoreExecutors.directExecutor(),
@ -349,7 +348,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
.registerInputStream( .registerInputStream(
INPUT_TYPE_SURFACE, INPUT_TYPE_SURFACE,
/* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer), /* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer),
new FrameInfo.Builder(WIDTH, HEIGHT).build()); new FrameInfo.Builder(ColorInfo.SDR_BT709_LIMITED, WIDTH, HEIGHT).build());
videoFrameProcessorReadyCountDownLatch.await(); videoFrameProcessorReadyCountDownLatch.await();
blankFrameProducer.produceBlankFrames(inputPresentationTimesUs); blankFrameProducer.produceBlankFrames(inputPresentationTimesUs);
defaultVideoFrameProcessor.signalEndOfInput(); defaultVideoFrameProcessor.signalEndOfInput();

View File

@ -172,7 +172,6 @@ public class FrameDropTest {
.create( .create(
getApplicationContext(), getApplicationContext(),
DebugViewProvider.NONE, DebugViewProvider.NONE,
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* renderFramesAutomatically= */ true, /* renderFramesAutomatically= */ true,
MoreExecutors.directExecutor(), MoreExecutors.directExecutor(),
@ -241,7 +240,9 @@ public class FrameDropTest {
} }
})), })),
frameDropEffect), frameDropEffect),
new FrameInfo.Builder(BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT).build()); new FrameInfo.Builder(
ColorInfo.SDR_BT709_LIMITED, BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT)
.build());
videoFrameProcessorReadyCountDownLatch.await(); videoFrameProcessorReadyCountDownLatch.await();
checkNoVideoFrameProcessingExceptionIsThrown(videoFrameProcessingExceptionReference); checkNoVideoFrameProcessingExceptionIsThrown(videoFrameProcessingExceptionReference);
blankFrameProducer.produceBlankFrames(inputPresentationTimesUs); blankFrameProducer.produceBlankFrames(inputPresentationTimesUs);

View File

@ -246,7 +246,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
public DefaultVideoFrameProcessor create( public DefaultVideoFrameProcessor create(
Context context, Context context,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
Executor listenerExecutor, Executor listenerExecutor,
@ -268,7 +267,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
createOpenGlObjectsAndFrameProcessor( createOpenGlObjectsAndFrameProcessor(
context, context,
debugViewProvider, debugViewProvider,
inputColorInfo,
outputColorInfo, outputColorInfo,
enableColorTransfers, enableColorTransfers,
renderFramesAutomatically, renderFramesAutomatically,
@ -321,9 +319,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private final List<Effect> activeEffects; private final List<Effect> activeEffects;
private final Object lock; private final Object lock;
private final boolean enableColorTransfers; private final boolean enableColorTransfers;
private final ColorInfo firstInputColorInfo;
private final ColorInfo outputColorInfo; private final ColorInfo outputColorInfo;
private @MonotonicNonNull ColorInfo firstInputColorInfo;
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
private volatile boolean inputStreamEnded; private volatile boolean inputStreamEnded;
@ -339,7 +338,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
FinalShaderProgramWrapper finalShaderProgramWrapper, FinalShaderProgramWrapper finalShaderProgramWrapper,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
boolean enableColorTransfers, boolean enableColorTransfers,
ColorInfo firstInputColorInfo,
ColorInfo outputColorInfo) { ColorInfo outputColorInfo) {
this.context = context; this.context = context;
this.glObjectsProvider = glObjectsProvider; this.glObjectsProvider = glObjectsProvider;
@ -353,7 +351,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
this.activeEffects = new ArrayList<>(); this.activeEffects = new ArrayList<>();
this.lock = new Object(); this.lock = new Object();
this.enableColorTransfers = enableColorTransfers; this.enableColorTransfers = enableColorTransfers;
this.firstInputColorInfo = firstInputColorInfo;
this.outputColorInfo = outputColorInfo; this.outputColorInfo = outputColorInfo;
this.finalShaderProgramWrapper = finalShaderProgramWrapper; this.finalShaderProgramWrapper = finalShaderProgramWrapper;
this.intermediateGlShaderPrograms = new ArrayList<>(); this.intermediateGlShaderPrograms = new ArrayList<>();
@ -438,6 +435,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return inputSwitcher.getInputSurface(); return inputSwitcher.getInputSurface();
} }
/**
* {@inheritDoc}
*
* <p>The {@link FrameInfo}'s {@link ColorInfo} must not change between different calls to this
* method.
*/
// TODO: b/307952514: Remove this javadoc after FrameInfo.colorInfo may change between calls.
@Override @Override
public void registerInputStream( public void registerInputStream(
@InputType int inputType, List<Effect> effects, FrameInfo frameInfo) { @InputType int inputType, List<Effect> effects, FrameInfo frameInfo) {
@ -463,10 +467,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
synchronized (lock) { synchronized (lock) {
// An input stream is pending until its effects are configured. // An input stream is pending until its effects are configured.
// TODO: b/307952514 - Move inputColorInfo's API from Factory.create into registerInputStream, InputStreamInfo pendingInputStreamInfo = new InputStreamInfo(inputType, effects, frameInfo);
// and from StreamInfo into FrameInfo.
InputStreamInfo pendingInputStreamInfo =
new InputStreamInfo(inputType, effects, this.firstInputColorInfo, frameInfo);
if (!registeredFirstInputStream) { if (!registeredFirstInputStream) {
registeredFirstInputStream = true; registeredFirstInputStream = true;
inputStreamRegisteredCondition.close(); inputStreamRegisteredCondition.close();
@ -611,7 +612,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor( private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor(
Context context, Context context,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
boolean enableColorTransfers, boolean enableColorTransfers,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
@ -685,7 +685,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
finalShaderProgramWrapper, finalShaderProgramWrapper,
renderFramesAutomatically, renderFramesAutomatically,
enableColorTransfers, enableColorTransfers,
/* firstInputColorInfo= */ inputColorInfo,
outputColorInfo); outputColorInfo);
} }
@ -802,7 +801,18 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
*/ */
private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure) private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
checkColors(firstInputColorInfo, outputColorInfo, enableColorTransfers); // TODO: b/307952514 - Remove this color check, and reinitialize the InputSwitcher's
// samplingGlShaderProgram instead.
checkState(
firstInputColorInfo == null
|| firstInputColorInfo.equals(inputStreamInfo.frameInfo.colorInfo));
if (inputStreamInfo.frameInfo.colorInfo != null) {
firstInputColorInfo = inputStreamInfo.frameInfo.colorInfo;
}
checkColors(
/* inputColorInfo= */ inputStreamInfo.frameInfo.colorInfo,
outputColorInfo,
enableColorTransfers);
if (forceReconfigure || !activeEffects.equals(inputStreamInfo.effects)) { if (forceReconfigure || !activeEffects.equals(inputStreamInfo.effects)) {
if (!intermediateGlShaderPrograms.isEmpty()) { if (!intermediateGlShaderPrograms.isEmpty()) {
for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) { for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) {
@ -830,8 +840,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
activeEffects.addAll(inputStreamInfo.effects); activeEffects.addAll(inputStreamInfo.effects);
} }
inputSwitcher.switchToInput( inputSwitcher.switchToInput(inputStreamInfo.inputType, inputStreamInfo.frameInfo);
inputStreamInfo.inputType, inputStreamInfo.frameInfo, inputStreamInfo.colorInfo);
inputStreamRegisteredCondition.open(); inputStreamRegisteredCondition.open();
listenerExecutor.execute( listenerExecutor.execute(
() -> () ->
@ -940,14 +949,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private static final class InputStreamInfo { private static final class InputStreamInfo {
public final @InputType int inputType; public final @InputType int inputType;
public final List<Effect> effects; public final List<Effect> effects;
public final ColorInfo colorInfo;
public final FrameInfo frameInfo; public final FrameInfo frameInfo;
public InputStreamInfo( public InputStreamInfo(@InputType int inputType, List<Effect> effects, FrameInfo frameInfo) {
@InputType int inputType, List<Effect> effects, ColorInfo colorInfo, FrameInfo frameInfo) {
this.inputType = inputType; this.inputType = inputType;
this.effects = effects; this.effects = effects;
this.colorInfo = colorInfo;
this.frameInfo = frameInfo; this.frameInfo = frameInfo;
} }
} }

View File

@ -152,12 +152,9 @@ import org.checkerframework.checker.nullness.qual.Nullable;
* *
* @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to. * @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to.
* @param inputFrameInfo The {@link FrameInfo} associated with the new input. * @param inputFrameInfo The {@link FrameInfo} associated with the new input.
* @param inputColorInfo The {@link ColorInfo} associated with the new input.
*/ */
public void switchToInput( public void switchToInput(
@VideoFrameProcessor.InputType int newInputType, @VideoFrameProcessor.InputType int newInputType, FrameInfo inputFrameInfo)
FrameInfo inputFrameInfo,
ColorInfo inputColorInfo)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
checkStateNotNull(downstreamShaderProgram); checkStateNotNull(downstreamShaderProgram);
checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType); checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType);
@ -170,7 +167,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
// TODO: b/307952514 - When switchToInput is called, and the inputColorInfo doesn't match // TODO: b/307952514 - When switchToInput is called, and the inputColorInfo doesn't match
// the prior inputColorInfo, recreate and reinitialize the input.samplingGlShaderProgram. // the prior inputColorInfo, recreate and reinitialize the input.samplingGlShaderProgram.
input.setSamplingGlShaderProgram( input.setSamplingGlShaderProgram(
createSamplingShaderProgram(inputColorInfo, newInputType)); createSamplingShaderProgram(inputFrameInfo.colorInfo, newInputType));
} }
input.setChainingListener( input.setChainingListener(
new GatedChainingListenerWrapper( new GatedChainingListenerWrapper(

View File

@ -147,9 +147,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
videoFrameProcessorFactory.create( videoFrameProcessorFactory.create(
context, context,
debugViewProvider, debugViewProvider,
// Pre-processing VideoFrameProcessors have converted the inputColor to outputColor
// already.
/* inputColorInfo= */ outputColorInfo,
outputColorInfo, outputColorInfo,
/* renderFramesAutomatically= */ true, /* renderFramesAutomatically= */ true,
/* listenerExecutor= */ MoreExecutors.directExecutor(), /* listenerExecutor= */ MoreExecutors.directExecutor(),
@ -233,7 +230,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
.create( .create(
context, context,
DebugViewProvider.NONE, DebugViewProvider.NONE,
inputColorInfo,
outputColorInfo, outputColorInfo,
// Pre-processors render frames as soon as available, to VideoCompositor. // Pre-processors render frames as soon as available, to VideoCompositor.
/* renderFramesAutomatically= */ true, /* renderFramesAutomatically= */ true,
@ -361,7 +357,11 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
.registerInputStream( .registerInputStream(
INPUT_TYPE_TEXTURE_ID, INPUT_TYPE_TEXTURE_ID,
compositionEffects, compositionEffects,
new FrameInfo.Builder(outputTexture.width, outputTexture.height).build()); // Pre-processing VideoFrameProcessors have converted the inputColor to outputColor
// already, so use outputColorInfo for the input color to the
// compositionVideoFrameProcessor.
new FrameInfo.Builder(outputColorInfo, outputTexture.width, outputTexture.height)
.build());
compositionVideoFrameProcessorInputStreamRegistered = true; compositionVideoFrameProcessorInputStreamRegistered = true;
// Return as the VideoFrameProcessor rejects input textures until the input is registered. // Return as the VideoFrameProcessor rejects input textures until the input is registered.
return; return;

View File

@ -63,6 +63,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph {
* *
* <p>{@code videoCompositorSettings} must be {@link VideoCompositorSettings#DEFAULT}. * <p>{@code videoCompositorSettings} must be {@link VideoCompositorSettings#DEFAULT}.
*/ */
// TODO: b/307952514 - Remove inputColorInfo reference in VideoGraph constructor.
public SingleInputVideoGraph( public SingleInputVideoGraph(
Context context, Context context,
VideoFrameProcessor.Factory videoFrameProcessorFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory,
@ -109,7 +110,6 @@ public abstract class SingleInputVideoGraph implements VideoGraph {
videoFrameProcessorFactory.create( videoFrameProcessorFactory.create(
context, context,
debugViewProvider, debugViewProvider,
inputColorInfo,
outputColorInfo, outputColorInfo,
renderFramesAutomatically, renderFramesAutomatically,
/* listenerExecutor= */ MoreExecutors.directExecutor(), /* listenerExecutor= */ MoreExecutors.directExecutor(),

View File

@ -259,10 +259,7 @@ public final class CompositingVideoSinkProvider
// Lazily initialize the handler here so it's initialized on the playback looper. // Lazily initialize the handler here so it's initialized on the playback looper.
handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null); handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null);
ColorInfo inputColorInfo = ColorInfo inputColorInfo = getAdjustedInputColorInfo(sourceFormat.colorInfo);
sourceFormat.colorInfo != null && ColorInfo.isTransferHdr(sourceFormat.colorInfo)
? sourceFormat.colorInfo
: ColorInfo.SDR_BT709_LIMITED;
ColorInfo outputColorInfo = inputColorInfo; ColorInfo outputColorInfo = inputColorInfo;
if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) { if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) {
// SurfaceView only supports BT2020 PQ input. Therefore, convert HLG to PQ. // SurfaceView only supports BT2020 PQ input. Therefore, convert HLG to PQ.
@ -552,6 +549,12 @@ public final class CompositingVideoSinkProvider
videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs); videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs);
} }
private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) {
return inputColorInfo != null && ColorInfo.isTransferHdr(inputColorInfo)
? inputColorInfo
: ColorInfo.SDR_BT709_LIMITED;
}
/** Receives input from an ExoPlayer renderer and forwards it to the video graph. */ /** Receives input from an ExoPlayer renderer and forwards it to the video graph. */
private static final class VideoSinkImpl implements VideoSink { private static final class VideoSinkImpl implements VideoSink {
private final Context context; private final Context context;
@ -562,6 +565,7 @@ public final class CompositingVideoSinkProvider
@Nullable private Effect rotationEffect; @Nullable private Effect rotationEffect;
@Nullable private Format inputFormat; @Nullable private Format inputFormat;
private @MonotonicNonNull ColorInfo firstInputColorInfo;
@InputType int inputType; @InputType int inputType;
private long inputStreamOffsetUs; private long inputStreamOffsetUs;
private boolean pendingInputStreamOffsetChange; private boolean pendingInputStreamOffsetChange;
@ -779,10 +783,15 @@ public final class CompositingVideoSinkProvider
} }
effects.addAll(videoEffects); effects.addAll(videoEffects);
Format inputFormat = checkNotNull(this.inputFormat); Format inputFormat = checkNotNull(this.inputFormat);
if (firstInputColorInfo == null) {
// TODO: b/307952514 - Get inputColorInfo from inputFormat for each stream, after this value
// can change per-stream.
firstInputColorInfo = getAdjustedInputColorInfo(inputFormat.colorInfo);
}
videoFrameProcessor.registerInputStream( videoFrameProcessor.registerInputStream(
inputType, inputType,
effects, effects,
new FrameInfo.Builder(inputFormat.width, inputFormat.height) new FrameInfo.Builder(firstInputColorInfo, inputFormat.width, inputFormat.height)
.setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio)
.build()); .build());
} }
@ -908,7 +917,6 @@ public final class CompositingVideoSinkProvider
public VideoFrameProcessor create( public VideoFrameProcessor create(
Context context, Context context,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
Executor listenerExecutor, Executor listenerExecutor,
@ -919,7 +927,6 @@ public final class CompositingVideoSinkProvider
.create( .create(
context, context,
debugViewProvider, debugViewProvider,
inputColorInfo,
outputColorInfo, outputColorInfo,
renderFramesAutomatically, renderFramesAutomatically,
listenerExecutor, listenerExecutor,

View File

@ -262,6 +262,7 @@ public final class VideoFrameProcessorTestRunner {
private final AtomicReference<VideoFrameProcessingException> videoFrameProcessingException; private final AtomicReference<VideoFrameProcessingException> videoFrameProcessingException;
private final VideoFrameProcessor videoFrameProcessor; private final VideoFrameProcessor videoFrameProcessor;
private final ImmutableList<Effect> effects; private final ImmutableList<Effect> effects;
private final ColorInfo inputColorInfo;
private final @MonotonicNonNull BitmapReader bitmapReader; private final @MonotonicNonNull BitmapReader bitmapReader;
private VideoFrameProcessorTestRunner( private VideoFrameProcessorTestRunner(
@ -290,7 +291,6 @@ public final class VideoFrameProcessorTestRunner {
videoFrameProcessorFactory.create( videoFrameProcessorFactory.create(
getApplicationContext(), getApplicationContext(),
DebugViewProvider.NONE, DebugViewProvider.NONE,
inputColorInfo,
outputColorInfo, outputColorInfo,
/* renderFramesAutomatically= */ true, /* renderFramesAutomatically= */ true,
/* listenerExecutor= */ MoreExecutors.directExecutor(), /* listenerExecutor= */ MoreExecutors.directExecutor(),
@ -335,6 +335,7 @@ public final class VideoFrameProcessorTestRunner {
} }
}); });
this.effects = effects; this.effects = effects;
this.inputColorInfo = inputColorInfo;
} }
public void processFirstFrameAndEnd() throws Exception { public void processFirstFrameAndEnd() throws Exception {
@ -348,6 +349,7 @@ public final class VideoFrameProcessorTestRunner {
INPUT_TYPE_SURFACE, INPUT_TYPE_SURFACE,
effects, effects,
new FrameInfo.Builder( new FrameInfo.Builder(
inputColorInfo,
mediaFormat.getInteger(MediaFormat.KEY_WIDTH), mediaFormat.getInteger(MediaFormat.KEY_WIDTH),
mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)) mediaFormat.getInteger(MediaFormat.KEY_HEIGHT))
.setPixelWidthHeightRatio(pixelWidthHeightRatio) .setPixelWidthHeightRatio(pixelWidthHeightRatio)
@ -377,7 +379,7 @@ public final class VideoFrameProcessorTestRunner {
videoFrameProcessor.registerInputStream( videoFrameProcessor.registerInputStream(
INPUT_TYPE_BITMAP, INPUT_TYPE_BITMAP,
effects, effects,
new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight()) new FrameInfo.Builder(inputColorInfo, inputBitmap.getWidth(), inputBitmap.getHeight())
.setPixelWidthHeightRatio(pixelWidthHeightRatio) .setPixelWidthHeightRatio(pixelWidthHeightRatio)
.setOffsetToAddUs(offsetToAddUs) .setOffsetToAddUs(offsetToAddUs)
.build()); .build());
@ -393,7 +395,7 @@ public final class VideoFrameProcessorTestRunner {
videoFrameProcessor.registerInputStream( videoFrameProcessor.registerInputStream(
INPUT_TYPE_BITMAP, INPUT_TYPE_BITMAP,
effects, effects,
new FrameInfo.Builder(width, height) new FrameInfo.Builder(inputColorInfo, width, height)
.setPixelWidthHeightRatio(pixelWidthHeightRatio) .setPixelWidthHeightRatio(pixelWidthHeightRatio)
.build()); .build());
videoFrameProcessorReadyCondition.block(); videoFrameProcessorReadyCondition.block();
@ -406,7 +408,7 @@ public final class VideoFrameProcessorTestRunner {
videoFrameProcessor.registerInputStream( videoFrameProcessor.registerInputStream(
INPUT_TYPE_TEXTURE_ID, INPUT_TYPE_TEXTURE_ID,
effects, effects,
new FrameInfo.Builder(inputTexture.width, inputTexture.height) new FrameInfo.Builder(inputColorInfo, inputTexture.width, inputTexture.height)
.setPixelWidthHeightRatio(pixelWidthHeightRatio) .setPixelWidthHeightRatio(pixelWidthHeightRatio)
.build()); .build());
videoFrameProcessor.setOnInputFrameProcessedListener( videoFrameProcessor.setOnInputFrameProcessedListener(

View File

@ -54,6 +54,7 @@ import java.util.concurrent.atomic.AtomicLong;
long initialTimestampOffsetUs) { long initialTimestampOffsetUs) {
this.videoFrameProcessor = videoFrameProcessor; this.videoFrameProcessor = videoFrameProcessor;
this.mediaItemOffsetUs = new AtomicLong(); this.mediaItemOffsetUs = new AtomicLong();
// TODO: b/307952514 - Remove inputColorInfo reference.
this.inputColorInfo = inputColorInfo; this.inputColorInfo = inputColorInfo;
this.initialTimestampOffsetUs = initialTimestampOffsetUs; this.initialTimestampOffsetUs = initialTimestampOffsetUs;
this.presentation = presentation; this.presentation = presentation;
@ -70,7 +71,7 @@ import java.util.concurrent.atomic.AtomicLong;
videoFrameProcessor.registerInputStream( videoFrameProcessor.registerInputStream(
getInputType(checkNotNull(trackFormat.sampleMimeType)), getInputType(checkNotNull(trackFormat.sampleMimeType)),
createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation), createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation),
new FrameInfo.Builder(decodedSize.getWidth(), decodedSize.getHeight()) new FrameInfo.Builder(inputColorInfo, decodedSize.getWidth(), decodedSize.getHeight())
.setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio) .setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio)
.setOffsetToAddUs(initialTimestampOffsetUs + mediaItemOffsetUs.get()) .setOffsetToAddUs(initialTimestampOffsetUs + mediaItemOffsetUs.get())
.build()); .build());