Effect: Move VideoFrameProcessor inputColorInfo interface to FrameInfo.
Move `inputColorInfo` from `VideoFrameProcessor`'s `Factory.create` to `FrameInfo`, input via `registerInputStream`. Also, manually tested on exoplayer demo that setVideoEffects still works. PiperOrigin-RevId: 591273545
This commit is contained in:
parent
f47c4e33ec
commit
7579693739
@ -27,6 +27,7 @@ public class FrameInfo {
|
|||||||
/** A builder for {@link FrameInfo} instances. */
|
/** A builder for {@link FrameInfo} instances. */
|
||||||
public static final class Builder {
|
public static final class Builder {
|
||||||
|
|
||||||
|
private ColorInfo colorInfo;
|
||||||
private int width;
|
private int width;
|
||||||
private int height;
|
private int height;
|
||||||
private float pixelWidthHeightRatio;
|
private float pixelWidthHeightRatio;
|
||||||
@ -35,10 +36,12 @@ public class FrameInfo {
|
|||||||
/**
|
/**
|
||||||
* Creates an instance with default values.
|
* Creates an instance with default values.
|
||||||
*
|
*
|
||||||
|
* @param colorInfo The {@link ColorInfo}.
|
||||||
* @param width The frame width, in pixels.
|
* @param width The frame width, in pixels.
|
||||||
* @param height The frame height, in pixels.
|
* @param height The frame height, in pixels.
|
||||||
*/
|
*/
|
||||||
public Builder(int width, int height) {
|
public Builder(ColorInfo colorInfo, int width, int height) {
|
||||||
|
this.colorInfo = colorInfo;
|
||||||
this.width = width;
|
this.width = width;
|
||||||
this.height = height;
|
this.height = height;
|
||||||
pixelWidthHeightRatio = 1;
|
pixelWidthHeightRatio = 1;
|
||||||
@ -46,12 +49,20 @@ public class FrameInfo {
|
|||||||
|
|
||||||
/** Creates an instance with the values of the provided {@link FrameInfo}. */
|
/** Creates an instance with the values of the provided {@link FrameInfo}. */
|
||||||
public Builder(FrameInfo frameInfo) {
|
public Builder(FrameInfo frameInfo) {
|
||||||
|
colorInfo = frameInfo.colorInfo;
|
||||||
width = frameInfo.width;
|
width = frameInfo.width;
|
||||||
height = frameInfo.height;
|
height = frameInfo.height;
|
||||||
pixelWidthHeightRatio = frameInfo.pixelWidthHeightRatio;
|
pixelWidthHeightRatio = frameInfo.pixelWidthHeightRatio;
|
||||||
offsetToAddUs = frameInfo.offsetToAddUs;
|
offsetToAddUs = frameInfo.offsetToAddUs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Sets the {@link ColorInfo}. */
|
||||||
|
@CanIgnoreReturnValue
|
||||||
|
public Builder setColorInfo(ColorInfo colorInfo) {
|
||||||
|
this.colorInfo = colorInfo;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
/** Sets the frame width, in pixels. */
|
/** Sets the frame width, in pixels. */
|
||||||
@CanIgnoreReturnValue
|
@CanIgnoreReturnValue
|
||||||
public Builder setWidth(int width) {
|
public Builder setWidth(int width) {
|
||||||
@ -91,10 +102,13 @@ public class FrameInfo {
|
|||||||
|
|
||||||
/** Builds a {@link FrameInfo} instance. */
|
/** Builds a {@link FrameInfo} instance. */
|
||||||
public FrameInfo build() {
|
public FrameInfo build() {
|
||||||
return new FrameInfo(width, height, pixelWidthHeightRatio, offsetToAddUs);
|
return new FrameInfo(colorInfo, width, height, pixelWidthHeightRatio, offsetToAddUs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The {@link ColorInfo} of the frame. */
|
||||||
|
public final ColorInfo colorInfo;
|
||||||
|
|
||||||
/** The width of the frame, in pixels. */
|
/** The width of the frame, in pixels. */
|
||||||
public final int width;
|
public final int width;
|
||||||
|
|
||||||
@ -112,12 +126,12 @@ public class FrameInfo {
|
|||||||
*/
|
*/
|
||||||
public final long offsetToAddUs;
|
public final long offsetToAddUs;
|
||||||
|
|
||||||
// TODO(b/227624622): Add color space information for HDR.
|
private FrameInfo(
|
||||||
|
ColorInfo colorInfo, int width, int height, float pixelWidthHeightRatio, long offsetToAddUs) {
|
||||||
private FrameInfo(int width, int height, float pixelWidthHeightRatio, long offsetToAddUs) {
|
|
||||||
checkArgument(width > 0, "width must be positive, but is: " + width);
|
checkArgument(width > 0, "width must be positive, but is: " + width);
|
||||||
checkArgument(height > 0, "height must be positive, but is: " + height);
|
checkArgument(height > 0, "height must be positive, but is: " + height);
|
||||||
|
|
||||||
|
this.colorInfo = colorInfo;
|
||||||
this.width = width;
|
this.width = width;
|
||||||
this.height = height;
|
this.height = height;
|
||||||
this.pixelWidthHeightRatio = pixelWidthHeightRatio;
|
this.pixelWidthHeightRatio = pixelWidthHeightRatio;
|
||||||
|
@ -82,7 +82,6 @@ public interface VideoFrameProcessor {
|
|||||||
*
|
*
|
||||||
* @param context A {@link Context}.
|
* @param context A {@link Context}.
|
||||||
* @param debugViewProvider A {@link DebugViewProvider}.
|
* @param debugViewProvider A {@link DebugViewProvider}.
|
||||||
* @param inputColorInfo The {@link ColorInfo} for the input frames.
|
|
||||||
* @param outputColorInfo The {@link ColorInfo} for the output frames.
|
* @param outputColorInfo The {@link ColorInfo} for the output frames.
|
||||||
* @param renderFramesAutomatically If {@code true}, the instance will render output frames to
|
* @param renderFramesAutomatically If {@code true}, the instance will render output frames to
|
||||||
* the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as
|
* the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as
|
||||||
@ -98,7 +97,6 @@ public interface VideoFrameProcessor {
|
|||||||
VideoFrameProcessor create(
|
VideoFrameProcessor create(
|
||||||
Context context,
|
Context context,
|
||||||
DebugViewProvider debugViewProvider,
|
DebugViewProvider debugViewProvider,
|
||||||
ColorInfo inputColorInfo,
|
|
||||||
ColorInfo outputColorInfo,
|
ColorInfo outputColorInfo,
|
||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
Executor listenerExecutor,
|
Executor listenerExecutor,
|
||||||
|
@ -100,7 +100,8 @@ public class DefaultVideoFrameProcessorTest {
|
|||||||
defaultVideoFrameProcessor.registerInputStream(
|
defaultVideoFrameProcessor.registerInputStream(
|
||||||
VideoFrameProcessor.INPUT_TYPE_BITMAP,
|
VideoFrameProcessor.INPUT_TYPE_BITMAP,
|
||||||
ImmutableList.of(),
|
ImmutableList.of(),
|
||||||
new FrameInfo.Builder(/* width= */ 100, /* height= */ 100).build());
|
new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, /* width= */ 100, /* height= */ 100)
|
||||||
|
.build());
|
||||||
|
|
||||||
assertThat(defaultVideoFrameProcessor.getPendingInputFrameCount()).isEqualTo(0);
|
assertThat(defaultVideoFrameProcessor.getPendingInputFrameCount()).isEqualTo(0);
|
||||||
// Unblocks configuration.
|
// Unblocks configuration.
|
||||||
@ -150,17 +151,20 @@ public class DefaultVideoFrameProcessorTest {
|
|||||||
new InputStreamInfo(
|
new InputStreamInfo(
|
||||||
VideoFrameProcessor.INPUT_TYPE_BITMAP,
|
VideoFrameProcessor.INPUT_TYPE_BITMAP,
|
||||||
ImmutableList.of(),
|
ImmutableList.of(),
|
||||||
new FrameInfo.Builder(/* width= */ 100, /* height= */ 100).build());
|
new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, /* width= */ 100, /* height= */ 100)
|
||||||
|
.build());
|
||||||
InputStreamInfo stream2 =
|
InputStreamInfo stream2 =
|
||||||
new InputStreamInfo(
|
new InputStreamInfo(
|
||||||
VideoFrameProcessor.INPUT_TYPE_BITMAP,
|
VideoFrameProcessor.INPUT_TYPE_BITMAP,
|
||||||
ImmutableList.of(new Contrast(.5f)),
|
ImmutableList.of(new Contrast(.5f)),
|
||||||
new FrameInfo.Builder(/* width= */ 200, /* height= */ 200).build());
|
new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, /* width= */ 200, /* height= */ 200)
|
||||||
|
.build());
|
||||||
InputStreamInfo stream3 =
|
InputStreamInfo stream3 =
|
||||||
new InputStreamInfo(
|
new InputStreamInfo(
|
||||||
VideoFrameProcessor.INPUT_TYPE_BITMAP,
|
VideoFrameProcessor.INPUT_TYPE_BITMAP,
|
||||||
ImmutableList.of(),
|
ImmutableList.of(),
|
||||||
new FrameInfo.Builder(/* width= */ 300, /* height= */ 300).build());
|
new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, /* width= */ 300, /* height= */ 300)
|
||||||
|
.build());
|
||||||
|
|
||||||
registerInputStream(defaultVideoFrameProcessor, stream1);
|
registerInputStream(defaultVideoFrameProcessor, stream1);
|
||||||
registerInputStream(defaultVideoFrameProcessor, stream2);
|
registerInputStream(defaultVideoFrameProcessor, stream2);
|
||||||
@ -179,7 +183,6 @@ public class DefaultVideoFrameProcessorTest {
|
|||||||
.create(
|
.create(
|
||||||
getApplicationContext(),
|
getApplicationContext(),
|
||||||
DebugViewProvider.NONE,
|
DebugViewProvider.NONE,
|
||||||
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
|
||||||
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
||||||
/* renderFramesAutomatically= */ true,
|
/* renderFramesAutomatically= */ true,
|
||||||
/* listenerExecutor= */ MoreExecutors.directExecutor(),
|
/* listenerExecutor= */ MoreExecutors.directExecutor(),
|
||||||
|
@ -292,7 +292,6 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
|
|||||||
.create(
|
.create(
|
||||||
getApplicationContext(),
|
getApplicationContext(),
|
||||||
DebugViewProvider.NONE,
|
DebugViewProvider.NONE,
|
||||||
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
|
||||||
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
||||||
renderFramesAutomatically,
|
renderFramesAutomatically,
|
||||||
MoreExecutors.directExecutor(),
|
MoreExecutors.directExecutor(),
|
||||||
@ -349,7 +348,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
|
|||||||
.registerInputStream(
|
.registerInputStream(
|
||||||
INPUT_TYPE_SURFACE,
|
INPUT_TYPE_SURFACE,
|
||||||
/* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer),
|
/* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer),
|
||||||
new FrameInfo.Builder(WIDTH, HEIGHT).build());
|
new FrameInfo.Builder(ColorInfo.SDR_BT709_LIMITED, WIDTH, HEIGHT).build());
|
||||||
videoFrameProcessorReadyCountDownLatch.await();
|
videoFrameProcessorReadyCountDownLatch.await();
|
||||||
blankFrameProducer.produceBlankFrames(inputPresentationTimesUs);
|
blankFrameProducer.produceBlankFrames(inputPresentationTimesUs);
|
||||||
defaultVideoFrameProcessor.signalEndOfInput();
|
defaultVideoFrameProcessor.signalEndOfInput();
|
||||||
|
@ -172,7 +172,6 @@ public class FrameDropTest {
|
|||||||
.create(
|
.create(
|
||||||
getApplicationContext(),
|
getApplicationContext(),
|
||||||
DebugViewProvider.NONE,
|
DebugViewProvider.NONE,
|
||||||
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
|
||||||
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
||||||
/* renderFramesAutomatically= */ true,
|
/* renderFramesAutomatically= */ true,
|
||||||
MoreExecutors.directExecutor(),
|
MoreExecutors.directExecutor(),
|
||||||
@ -241,7 +240,9 @@ public class FrameDropTest {
|
|||||||
}
|
}
|
||||||
})),
|
})),
|
||||||
frameDropEffect),
|
frameDropEffect),
|
||||||
new FrameInfo.Builder(BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT).build());
|
new FrameInfo.Builder(
|
||||||
|
ColorInfo.SDR_BT709_LIMITED, BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT)
|
||||||
|
.build());
|
||||||
videoFrameProcessorReadyCountDownLatch.await();
|
videoFrameProcessorReadyCountDownLatch.await();
|
||||||
checkNoVideoFrameProcessingExceptionIsThrown(videoFrameProcessingExceptionReference);
|
checkNoVideoFrameProcessingExceptionIsThrown(videoFrameProcessingExceptionReference);
|
||||||
blankFrameProducer.produceBlankFrames(inputPresentationTimesUs);
|
blankFrameProducer.produceBlankFrames(inputPresentationTimesUs);
|
||||||
|
@ -246,7 +246,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
public DefaultVideoFrameProcessor create(
|
public DefaultVideoFrameProcessor create(
|
||||||
Context context,
|
Context context,
|
||||||
DebugViewProvider debugViewProvider,
|
DebugViewProvider debugViewProvider,
|
||||||
ColorInfo inputColorInfo,
|
|
||||||
ColorInfo outputColorInfo,
|
ColorInfo outputColorInfo,
|
||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
Executor listenerExecutor,
|
Executor listenerExecutor,
|
||||||
@ -268,7 +267,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
createOpenGlObjectsAndFrameProcessor(
|
createOpenGlObjectsAndFrameProcessor(
|
||||||
context,
|
context,
|
||||||
debugViewProvider,
|
debugViewProvider,
|
||||||
inputColorInfo,
|
|
||||||
outputColorInfo,
|
outputColorInfo,
|
||||||
enableColorTransfers,
|
enableColorTransfers,
|
||||||
renderFramesAutomatically,
|
renderFramesAutomatically,
|
||||||
@ -321,9 +319,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
private final List<Effect> activeEffects;
|
private final List<Effect> activeEffects;
|
||||||
private final Object lock;
|
private final Object lock;
|
||||||
private final boolean enableColorTransfers;
|
private final boolean enableColorTransfers;
|
||||||
private final ColorInfo firstInputColorInfo;
|
|
||||||
private final ColorInfo outputColorInfo;
|
private final ColorInfo outputColorInfo;
|
||||||
|
|
||||||
|
private @MonotonicNonNull ColorInfo firstInputColorInfo;
|
||||||
|
|
||||||
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
|
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
|
||||||
private volatile boolean inputStreamEnded;
|
private volatile boolean inputStreamEnded;
|
||||||
|
|
||||||
@ -339,7 +338,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
boolean enableColorTransfers,
|
boolean enableColorTransfers,
|
||||||
ColorInfo firstInputColorInfo,
|
|
||||||
ColorInfo outputColorInfo) {
|
ColorInfo outputColorInfo) {
|
||||||
this.context = context;
|
this.context = context;
|
||||||
this.glObjectsProvider = glObjectsProvider;
|
this.glObjectsProvider = glObjectsProvider;
|
||||||
@ -353,7 +351,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
this.activeEffects = new ArrayList<>();
|
this.activeEffects = new ArrayList<>();
|
||||||
this.lock = new Object();
|
this.lock = new Object();
|
||||||
this.enableColorTransfers = enableColorTransfers;
|
this.enableColorTransfers = enableColorTransfers;
|
||||||
this.firstInputColorInfo = firstInputColorInfo;
|
|
||||||
this.outputColorInfo = outputColorInfo;
|
this.outputColorInfo = outputColorInfo;
|
||||||
this.finalShaderProgramWrapper = finalShaderProgramWrapper;
|
this.finalShaderProgramWrapper = finalShaderProgramWrapper;
|
||||||
this.intermediateGlShaderPrograms = new ArrayList<>();
|
this.intermediateGlShaderPrograms = new ArrayList<>();
|
||||||
@ -438,6 +435,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
return inputSwitcher.getInputSurface();
|
return inputSwitcher.getInputSurface();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* <p>The {@link FrameInfo}'s {@link ColorInfo} must not change between different calls to this
|
||||||
|
* method.
|
||||||
|
*/
|
||||||
|
// TODO: b/307952514: Remove this javadoc after FrameInfo.colorInfo may change between calls.
|
||||||
@Override
|
@Override
|
||||||
public void registerInputStream(
|
public void registerInputStream(
|
||||||
@InputType int inputType, List<Effect> effects, FrameInfo frameInfo) {
|
@InputType int inputType, List<Effect> effects, FrameInfo frameInfo) {
|
||||||
@ -463,10 +467,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
|
|
||||||
synchronized (lock) {
|
synchronized (lock) {
|
||||||
// An input stream is pending until its effects are configured.
|
// An input stream is pending until its effects are configured.
|
||||||
// TODO: b/307952514 - Move inputColorInfo's API from Factory.create into registerInputStream,
|
InputStreamInfo pendingInputStreamInfo = new InputStreamInfo(inputType, effects, frameInfo);
|
||||||
// and from StreamInfo into FrameInfo.
|
|
||||||
InputStreamInfo pendingInputStreamInfo =
|
|
||||||
new InputStreamInfo(inputType, effects, this.firstInputColorInfo, frameInfo);
|
|
||||||
if (!registeredFirstInputStream) {
|
if (!registeredFirstInputStream) {
|
||||||
registeredFirstInputStream = true;
|
registeredFirstInputStream = true;
|
||||||
inputStreamRegisteredCondition.close();
|
inputStreamRegisteredCondition.close();
|
||||||
@ -611,7 +612,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor(
|
private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor(
|
||||||
Context context,
|
Context context,
|
||||||
DebugViewProvider debugViewProvider,
|
DebugViewProvider debugViewProvider,
|
||||||
ColorInfo inputColorInfo,
|
|
||||||
ColorInfo outputColorInfo,
|
ColorInfo outputColorInfo,
|
||||||
boolean enableColorTransfers,
|
boolean enableColorTransfers,
|
||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
@ -685,7 +685,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
finalShaderProgramWrapper,
|
finalShaderProgramWrapper,
|
||||||
renderFramesAutomatically,
|
renderFramesAutomatically,
|
||||||
enableColorTransfers,
|
enableColorTransfers,
|
||||||
/* firstInputColorInfo= */ inputColorInfo,
|
|
||||||
outputColorInfo);
|
outputColorInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -802,7 +801,18 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
*/
|
*/
|
||||||
private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure)
|
private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure)
|
||||||
throws VideoFrameProcessingException {
|
throws VideoFrameProcessingException {
|
||||||
checkColors(firstInputColorInfo, outputColorInfo, enableColorTransfers);
|
// TODO: b/307952514 - Remove this color check, and reinitialize the InputSwitcher's
|
||||||
|
// samplingGlShaderProgram instead.
|
||||||
|
checkState(
|
||||||
|
firstInputColorInfo == null
|
||||||
|
|| firstInputColorInfo.equals(inputStreamInfo.frameInfo.colorInfo));
|
||||||
|
if (inputStreamInfo.frameInfo.colorInfo != null) {
|
||||||
|
firstInputColorInfo = inputStreamInfo.frameInfo.colorInfo;
|
||||||
|
}
|
||||||
|
checkColors(
|
||||||
|
/* inputColorInfo= */ inputStreamInfo.frameInfo.colorInfo,
|
||||||
|
outputColorInfo,
|
||||||
|
enableColorTransfers);
|
||||||
if (forceReconfigure || !activeEffects.equals(inputStreamInfo.effects)) {
|
if (forceReconfigure || !activeEffects.equals(inputStreamInfo.effects)) {
|
||||||
if (!intermediateGlShaderPrograms.isEmpty()) {
|
if (!intermediateGlShaderPrograms.isEmpty()) {
|
||||||
for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) {
|
for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) {
|
||||||
@ -830,8 +840,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
activeEffects.addAll(inputStreamInfo.effects);
|
activeEffects.addAll(inputStreamInfo.effects);
|
||||||
}
|
}
|
||||||
|
|
||||||
inputSwitcher.switchToInput(
|
inputSwitcher.switchToInput(inputStreamInfo.inputType, inputStreamInfo.frameInfo);
|
||||||
inputStreamInfo.inputType, inputStreamInfo.frameInfo, inputStreamInfo.colorInfo);
|
|
||||||
inputStreamRegisteredCondition.open();
|
inputStreamRegisteredCondition.open();
|
||||||
listenerExecutor.execute(
|
listenerExecutor.execute(
|
||||||
() ->
|
() ->
|
||||||
@ -940,14 +949,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
private static final class InputStreamInfo {
|
private static final class InputStreamInfo {
|
||||||
public final @InputType int inputType;
|
public final @InputType int inputType;
|
||||||
public final List<Effect> effects;
|
public final List<Effect> effects;
|
||||||
public final ColorInfo colorInfo;
|
|
||||||
public final FrameInfo frameInfo;
|
public final FrameInfo frameInfo;
|
||||||
|
|
||||||
public InputStreamInfo(
|
public InputStreamInfo(@InputType int inputType, List<Effect> effects, FrameInfo frameInfo) {
|
||||||
@InputType int inputType, List<Effect> effects, ColorInfo colorInfo, FrameInfo frameInfo) {
|
|
||||||
this.inputType = inputType;
|
this.inputType = inputType;
|
||||||
this.effects = effects;
|
this.effects = effects;
|
||||||
this.colorInfo = colorInfo;
|
|
||||||
this.frameInfo = frameInfo;
|
this.frameInfo = frameInfo;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -152,12 +152,9 @@ import org.checkerframework.checker.nullness.qual.Nullable;
|
|||||||
*
|
*
|
||||||
* @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to.
|
* @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to.
|
||||||
* @param inputFrameInfo The {@link FrameInfo} associated with the new input.
|
* @param inputFrameInfo The {@link FrameInfo} associated with the new input.
|
||||||
* @param inputColorInfo The {@link ColorInfo} associated with the new input.
|
|
||||||
*/
|
*/
|
||||||
public void switchToInput(
|
public void switchToInput(
|
||||||
@VideoFrameProcessor.InputType int newInputType,
|
@VideoFrameProcessor.InputType int newInputType, FrameInfo inputFrameInfo)
|
||||||
FrameInfo inputFrameInfo,
|
|
||||||
ColorInfo inputColorInfo)
|
|
||||||
throws VideoFrameProcessingException {
|
throws VideoFrameProcessingException {
|
||||||
checkStateNotNull(downstreamShaderProgram);
|
checkStateNotNull(downstreamShaderProgram);
|
||||||
checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType);
|
checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType);
|
||||||
@ -170,7 +167,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
|
|||||||
// TODO: b/307952514 - When switchToInput is called, and the inputColorInfo doesn't match
|
// TODO: b/307952514 - When switchToInput is called, and the inputColorInfo doesn't match
|
||||||
// the prior inputColorInfo, recreate and reinitialize the input.samplingGlShaderProgram.
|
// the prior inputColorInfo, recreate and reinitialize the input.samplingGlShaderProgram.
|
||||||
input.setSamplingGlShaderProgram(
|
input.setSamplingGlShaderProgram(
|
||||||
createSamplingShaderProgram(inputColorInfo, newInputType));
|
createSamplingShaderProgram(inputFrameInfo.colorInfo, newInputType));
|
||||||
}
|
}
|
||||||
input.setChainingListener(
|
input.setChainingListener(
|
||||||
new GatedChainingListenerWrapper(
|
new GatedChainingListenerWrapper(
|
||||||
|
@ -147,9 +147,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
|
|||||||
videoFrameProcessorFactory.create(
|
videoFrameProcessorFactory.create(
|
||||||
context,
|
context,
|
||||||
debugViewProvider,
|
debugViewProvider,
|
||||||
// Pre-processing VideoFrameProcessors have converted the inputColor to outputColor
|
|
||||||
// already.
|
|
||||||
/* inputColorInfo= */ outputColorInfo,
|
|
||||||
outputColorInfo,
|
outputColorInfo,
|
||||||
/* renderFramesAutomatically= */ true,
|
/* renderFramesAutomatically= */ true,
|
||||||
/* listenerExecutor= */ MoreExecutors.directExecutor(),
|
/* listenerExecutor= */ MoreExecutors.directExecutor(),
|
||||||
@ -233,7 +230,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
|
|||||||
.create(
|
.create(
|
||||||
context,
|
context,
|
||||||
DebugViewProvider.NONE,
|
DebugViewProvider.NONE,
|
||||||
inputColorInfo,
|
|
||||||
outputColorInfo,
|
outputColorInfo,
|
||||||
// Pre-processors render frames as soon as available, to VideoCompositor.
|
// Pre-processors render frames as soon as available, to VideoCompositor.
|
||||||
/* renderFramesAutomatically= */ true,
|
/* renderFramesAutomatically= */ true,
|
||||||
@ -361,7 +357,11 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
|
|||||||
.registerInputStream(
|
.registerInputStream(
|
||||||
INPUT_TYPE_TEXTURE_ID,
|
INPUT_TYPE_TEXTURE_ID,
|
||||||
compositionEffects,
|
compositionEffects,
|
||||||
new FrameInfo.Builder(outputTexture.width, outputTexture.height).build());
|
// Pre-processing VideoFrameProcessors have converted the inputColor to outputColor
|
||||||
|
// already, so use outputColorInfo for the input color to the
|
||||||
|
// compositionVideoFrameProcessor.
|
||||||
|
new FrameInfo.Builder(outputColorInfo, outputTexture.width, outputTexture.height)
|
||||||
|
.build());
|
||||||
compositionVideoFrameProcessorInputStreamRegistered = true;
|
compositionVideoFrameProcessorInputStreamRegistered = true;
|
||||||
// Return as the VideoFrameProcessor rejects input textures until the input is registered.
|
// Return as the VideoFrameProcessor rejects input textures until the input is registered.
|
||||||
return;
|
return;
|
||||||
|
@ -63,6 +63,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph {
|
|||||||
*
|
*
|
||||||
* <p>{@code videoCompositorSettings} must be {@link VideoCompositorSettings#DEFAULT}.
|
* <p>{@code videoCompositorSettings} must be {@link VideoCompositorSettings#DEFAULT}.
|
||||||
*/
|
*/
|
||||||
|
// TODO: b/307952514 - Remove inputColorInfo reference in VideoGraph constructor.
|
||||||
public SingleInputVideoGraph(
|
public SingleInputVideoGraph(
|
||||||
Context context,
|
Context context,
|
||||||
VideoFrameProcessor.Factory videoFrameProcessorFactory,
|
VideoFrameProcessor.Factory videoFrameProcessorFactory,
|
||||||
@ -109,7 +110,6 @@ public abstract class SingleInputVideoGraph implements VideoGraph {
|
|||||||
videoFrameProcessorFactory.create(
|
videoFrameProcessorFactory.create(
|
||||||
context,
|
context,
|
||||||
debugViewProvider,
|
debugViewProvider,
|
||||||
inputColorInfo,
|
|
||||||
outputColorInfo,
|
outputColorInfo,
|
||||||
renderFramesAutomatically,
|
renderFramesAutomatically,
|
||||||
/* listenerExecutor= */ MoreExecutors.directExecutor(),
|
/* listenerExecutor= */ MoreExecutors.directExecutor(),
|
||||||
|
@ -259,10 +259,7 @@ public final class CompositingVideoSinkProvider
|
|||||||
// Lazily initialize the handler here so it's initialized on the playback looper.
|
// Lazily initialize the handler here so it's initialized on the playback looper.
|
||||||
handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null);
|
handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null);
|
||||||
|
|
||||||
ColorInfo inputColorInfo =
|
ColorInfo inputColorInfo = getAdjustedInputColorInfo(sourceFormat.colorInfo);
|
||||||
sourceFormat.colorInfo != null && ColorInfo.isTransferHdr(sourceFormat.colorInfo)
|
|
||||||
? sourceFormat.colorInfo
|
|
||||||
: ColorInfo.SDR_BT709_LIMITED;
|
|
||||||
ColorInfo outputColorInfo = inputColorInfo;
|
ColorInfo outputColorInfo = inputColorInfo;
|
||||||
if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) {
|
if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) {
|
||||||
// SurfaceView only supports BT2020 PQ input. Therefore, convert HLG to PQ.
|
// SurfaceView only supports BT2020 PQ input. Therefore, convert HLG to PQ.
|
||||||
@ -552,6 +549,12 @@ public final class CompositingVideoSinkProvider
|
|||||||
videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs);
|
videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) {
|
||||||
|
return inputColorInfo != null && ColorInfo.isTransferHdr(inputColorInfo)
|
||||||
|
? inputColorInfo
|
||||||
|
: ColorInfo.SDR_BT709_LIMITED;
|
||||||
|
}
|
||||||
|
|
||||||
/** Receives input from an ExoPlayer renderer and forwards it to the video graph. */
|
/** Receives input from an ExoPlayer renderer and forwards it to the video graph. */
|
||||||
private static final class VideoSinkImpl implements VideoSink {
|
private static final class VideoSinkImpl implements VideoSink {
|
||||||
private final Context context;
|
private final Context context;
|
||||||
@ -562,6 +565,7 @@ public final class CompositingVideoSinkProvider
|
|||||||
@Nullable private Effect rotationEffect;
|
@Nullable private Effect rotationEffect;
|
||||||
|
|
||||||
@Nullable private Format inputFormat;
|
@Nullable private Format inputFormat;
|
||||||
|
private @MonotonicNonNull ColorInfo firstInputColorInfo;
|
||||||
@InputType int inputType;
|
@InputType int inputType;
|
||||||
private long inputStreamOffsetUs;
|
private long inputStreamOffsetUs;
|
||||||
private boolean pendingInputStreamOffsetChange;
|
private boolean pendingInputStreamOffsetChange;
|
||||||
@ -779,10 +783,15 @@ public final class CompositingVideoSinkProvider
|
|||||||
}
|
}
|
||||||
effects.addAll(videoEffects);
|
effects.addAll(videoEffects);
|
||||||
Format inputFormat = checkNotNull(this.inputFormat);
|
Format inputFormat = checkNotNull(this.inputFormat);
|
||||||
|
if (firstInputColorInfo == null) {
|
||||||
|
// TODO: b/307952514 - Get inputColorInfo from inputFormat for each stream, after this value
|
||||||
|
// can change per-stream.
|
||||||
|
firstInputColorInfo = getAdjustedInputColorInfo(inputFormat.colorInfo);
|
||||||
|
}
|
||||||
videoFrameProcessor.registerInputStream(
|
videoFrameProcessor.registerInputStream(
|
||||||
inputType,
|
inputType,
|
||||||
effects,
|
effects,
|
||||||
new FrameInfo.Builder(inputFormat.width, inputFormat.height)
|
new FrameInfo.Builder(firstInputColorInfo, inputFormat.width, inputFormat.height)
|
||||||
.setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio)
|
.setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio)
|
||||||
.build());
|
.build());
|
||||||
}
|
}
|
||||||
@ -908,7 +917,6 @@ public final class CompositingVideoSinkProvider
|
|||||||
public VideoFrameProcessor create(
|
public VideoFrameProcessor create(
|
||||||
Context context,
|
Context context,
|
||||||
DebugViewProvider debugViewProvider,
|
DebugViewProvider debugViewProvider,
|
||||||
ColorInfo inputColorInfo,
|
|
||||||
ColorInfo outputColorInfo,
|
ColorInfo outputColorInfo,
|
||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
Executor listenerExecutor,
|
Executor listenerExecutor,
|
||||||
@ -919,7 +927,6 @@ public final class CompositingVideoSinkProvider
|
|||||||
.create(
|
.create(
|
||||||
context,
|
context,
|
||||||
debugViewProvider,
|
debugViewProvider,
|
||||||
inputColorInfo,
|
|
||||||
outputColorInfo,
|
outputColorInfo,
|
||||||
renderFramesAutomatically,
|
renderFramesAutomatically,
|
||||||
listenerExecutor,
|
listenerExecutor,
|
||||||
|
@ -262,6 +262,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||||||
private final AtomicReference<VideoFrameProcessingException> videoFrameProcessingException;
|
private final AtomicReference<VideoFrameProcessingException> videoFrameProcessingException;
|
||||||
private final VideoFrameProcessor videoFrameProcessor;
|
private final VideoFrameProcessor videoFrameProcessor;
|
||||||
private final ImmutableList<Effect> effects;
|
private final ImmutableList<Effect> effects;
|
||||||
|
private final ColorInfo inputColorInfo;
|
||||||
private final @MonotonicNonNull BitmapReader bitmapReader;
|
private final @MonotonicNonNull BitmapReader bitmapReader;
|
||||||
|
|
||||||
private VideoFrameProcessorTestRunner(
|
private VideoFrameProcessorTestRunner(
|
||||||
@ -290,7 +291,6 @@ public final class VideoFrameProcessorTestRunner {
|
|||||||
videoFrameProcessorFactory.create(
|
videoFrameProcessorFactory.create(
|
||||||
getApplicationContext(),
|
getApplicationContext(),
|
||||||
DebugViewProvider.NONE,
|
DebugViewProvider.NONE,
|
||||||
inputColorInfo,
|
|
||||||
outputColorInfo,
|
outputColorInfo,
|
||||||
/* renderFramesAutomatically= */ true,
|
/* renderFramesAutomatically= */ true,
|
||||||
/* listenerExecutor= */ MoreExecutors.directExecutor(),
|
/* listenerExecutor= */ MoreExecutors.directExecutor(),
|
||||||
@ -335,6 +335,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
this.effects = effects;
|
this.effects = effects;
|
||||||
|
this.inputColorInfo = inputColorInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void processFirstFrameAndEnd() throws Exception {
|
public void processFirstFrameAndEnd() throws Exception {
|
||||||
@ -348,6 +349,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||||||
INPUT_TYPE_SURFACE,
|
INPUT_TYPE_SURFACE,
|
||||||
effects,
|
effects,
|
||||||
new FrameInfo.Builder(
|
new FrameInfo.Builder(
|
||||||
|
inputColorInfo,
|
||||||
mediaFormat.getInteger(MediaFormat.KEY_WIDTH),
|
mediaFormat.getInteger(MediaFormat.KEY_WIDTH),
|
||||||
mediaFormat.getInteger(MediaFormat.KEY_HEIGHT))
|
mediaFormat.getInteger(MediaFormat.KEY_HEIGHT))
|
||||||
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
||||||
@ -377,7 +379,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||||||
videoFrameProcessor.registerInputStream(
|
videoFrameProcessor.registerInputStream(
|
||||||
INPUT_TYPE_BITMAP,
|
INPUT_TYPE_BITMAP,
|
||||||
effects,
|
effects,
|
||||||
new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight())
|
new FrameInfo.Builder(inputColorInfo, inputBitmap.getWidth(), inputBitmap.getHeight())
|
||||||
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
||||||
.setOffsetToAddUs(offsetToAddUs)
|
.setOffsetToAddUs(offsetToAddUs)
|
||||||
.build());
|
.build());
|
||||||
@ -393,7 +395,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||||||
videoFrameProcessor.registerInputStream(
|
videoFrameProcessor.registerInputStream(
|
||||||
INPUT_TYPE_BITMAP,
|
INPUT_TYPE_BITMAP,
|
||||||
effects,
|
effects,
|
||||||
new FrameInfo.Builder(width, height)
|
new FrameInfo.Builder(inputColorInfo, width, height)
|
||||||
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
||||||
.build());
|
.build());
|
||||||
videoFrameProcessorReadyCondition.block();
|
videoFrameProcessorReadyCondition.block();
|
||||||
@ -406,7 +408,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||||||
videoFrameProcessor.registerInputStream(
|
videoFrameProcessor.registerInputStream(
|
||||||
INPUT_TYPE_TEXTURE_ID,
|
INPUT_TYPE_TEXTURE_ID,
|
||||||
effects,
|
effects,
|
||||||
new FrameInfo.Builder(inputTexture.width, inputTexture.height)
|
new FrameInfo.Builder(inputColorInfo, inputTexture.width, inputTexture.height)
|
||||||
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
||||||
.build());
|
.build());
|
||||||
videoFrameProcessor.setOnInputFrameProcessedListener(
|
videoFrameProcessor.setOnInputFrameProcessedListener(
|
||||||
|
@ -54,6 +54,7 @@ import java.util.concurrent.atomic.AtomicLong;
|
|||||||
long initialTimestampOffsetUs) {
|
long initialTimestampOffsetUs) {
|
||||||
this.videoFrameProcessor = videoFrameProcessor;
|
this.videoFrameProcessor = videoFrameProcessor;
|
||||||
this.mediaItemOffsetUs = new AtomicLong();
|
this.mediaItemOffsetUs = new AtomicLong();
|
||||||
|
// TODO: b/307952514 - Remove inputColorInfo reference.
|
||||||
this.inputColorInfo = inputColorInfo;
|
this.inputColorInfo = inputColorInfo;
|
||||||
this.initialTimestampOffsetUs = initialTimestampOffsetUs;
|
this.initialTimestampOffsetUs = initialTimestampOffsetUs;
|
||||||
this.presentation = presentation;
|
this.presentation = presentation;
|
||||||
@ -70,7 +71,7 @@ import java.util.concurrent.atomic.AtomicLong;
|
|||||||
videoFrameProcessor.registerInputStream(
|
videoFrameProcessor.registerInputStream(
|
||||||
getInputType(checkNotNull(trackFormat.sampleMimeType)),
|
getInputType(checkNotNull(trackFormat.sampleMimeType)),
|
||||||
createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation),
|
createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation),
|
||||||
new FrameInfo.Builder(decodedSize.getWidth(), decodedSize.getHeight())
|
new FrameInfo.Builder(inputColorInfo, decodedSize.getWidth(), decodedSize.getHeight())
|
||||||
.setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio)
|
.setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio)
|
||||||
.setOffsetToAddUs(initialTimestampOffsetUs + mediaItemOffsetUs.get())
|
.setOffsetToAddUs(initialTimestampOffsetUs + mediaItemOffsetUs.get())
|
||||||
.build());
|
.build());
|
||||||
|
Loading…
x
Reference in New Issue
Block a user