From 6f109ffa6ab7180aeda80efdf354e2126c71ed2f Mon Sep 17 00:00:00 2001 From: christosts Date: Mon, 11 Mar 2024 09:26:20 -0700 Subject: [PATCH] Composition preview: play multiple images PiperOrigin-RevId: 614690669 --- .../util/ConstantRateTimestampIterator.java | 21 +--- .../ConstantRateTimestampIteratorTest.java | 24 ----- .../video/CompositingVideoSinkProvider.java | 102 ++++++++++++++++-- 3 files changed, 94 insertions(+), 53 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java b/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java index d12e891667..b02b409643 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java @@ -33,7 +33,6 @@ public final class ConstantRateTimestampIterator implements TimestampIterator { private final long durationUs; private final float frameRate; private final double framesDurationUs; - private final long startingTimestampUs; private final int totalNumberOfFramesToAdd; private int framesAdded; @@ -47,26 +46,10 @@ public final class ConstantRateTimestampIterator implements TimestampIterator { public ConstantRateTimestampIterator( @IntRange(from = 1) long durationUs, @FloatRange(from = 0, fromInclusive = false) float frameRate) { - this(durationUs, frameRate, /* startingTimestampUs= */ 0); - } - - /** - * Creates an instance that outputs timestamps from {@code startingTimestampUs}. - * - * @param durationUs The duration the timestamps should span over, in microseconds. - * @param frameRate The frame rate in frames per second. - * @param startingTimestampUs The first timestamp output from the iterator. - */ - public ConstantRateTimestampIterator( - @IntRange(from = 1) long durationUs, - @FloatRange(from = 0, fromInclusive = false) float frameRate, - @IntRange(from = 0) long startingTimestampUs) { checkArgument(durationUs > 0); checkArgument(frameRate > 0); - checkArgument(startingTimestampUs >= 0); this.durationUs = durationUs; this.frameRate = frameRate; - this.startingTimestampUs = startingTimestampUs; this.totalNumberOfFramesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND)); framesDurationUs = C.MICROS_PER_SECOND / frameRate; } @@ -84,7 +67,7 @@ public final class ConstantRateTimestampIterator implements TimestampIterator { @Override public ConstantRateTimestampIterator copyOf() { - return new ConstantRateTimestampIterator(durationUs, frameRate, startingTimestampUs); + return new ConstantRateTimestampIterator(durationUs, frameRate); } @Override @@ -97,7 +80,7 @@ public final class ConstantRateTimestampIterator implements TimestampIterator { /** Returns the timestamp after {@code numberOfFrames}, in microseconds. */ private long getTimestampUsAfter(int numberOfFrames) { - long timestampUs = round(startingTimestampUs + framesDurationUs * numberOfFrames); + long timestampUs = round(framesDurationUs * numberOfFrames); // Check for possible overflow. checkState(timestampUs >= 0); return timestampUs; diff --git a/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java b/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java index 313604b3f5..122a61a195 100644 --- a/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java @@ -77,30 +77,6 @@ public class ConstantRateTimestampIteratorTest { assertThat(constantRateTimestampIterator.getLastTimestampUs()).isEqualTo(C.TIME_UNSET); } - @Test - public void timestampIterator_withNonZeroStartingTime_firstOutputsStartingTimestamp() { - ConstantRateTimestampIterator constantRateTimestampIterator = - new ConstantRateTimestampIterator( - /* durationUs= */ C.MICROS_PER_SECOND, - /* frameRate= */ 2, - /* startingTimestampUs= */ 1234); - - assertThat(constantRateTimestampIterator.next()).isEqualTo(1234); - assertThat(constantRateTimestampIterator.getLastTimestampUs()) - .isEqualTo(1234 + C.MICROS_PER_SECOND / 2); - } - - @Test - public void copyOf_withNonZeroStartingTime_firstOutputsStartingTimestamp() { - ConstantRateTimestampIterator constantRateTimestampIterator = - new ConstantRateTimestampIterator( - /* durationUs= */ C.MICROS_PER_SECOND, - /* frameRate= */ 2, - /* startingTimestampUs= */ 1234); - - assertThat(constantRateTimestampIterator.copyOf().next()).isEqualTo(1234); - } - private static List generateList(TimestampIterator iterator) { ArrayList list = new ArrayList<>(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 4acf1f710f..fa5203917f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -705,12 +705,7 @@ public final class CompositingVideoSinkProvider // the duration of the first video. Thus this correction is need to correct for the different // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. long bufferPresentationTimeUs = framePresentationTimeUs + inputStreamOffsetUs; - if (pendingInputStreamOffsetChange) { - compositingVideoSinkProvider.onStreamOffsetChange( - /* bufferPresentationTimeUs= */ bufferPresentationTimeUs, - /* streamOffsetUs= */ inputStreamOffsetUs); - pendingInputStreamOffsetChange = false; - } + maybeSetStreamOffsetChange(bufferPresentationTimeUs); lastBufferPresentationTimeUs = bufferPresentationTimeUs; if (isLastFrame) { finalBufferPresentationTimeUs = bufferPresentationTimeUs; @@ -720,11 +715,29 @@ public final class CompositingVideoSinkProvider @Override public boolean queueBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator) { - if (checkStateNotNull(videoFrameProcessor).queueInputBitmap(inputBitmap, timestampIterator)) { - lastBufferPresentationTimeUs = timestampIterator.getLastTimestampUs(); - return true; + if (!maybeRegisterPendingInputStream()) { + return false; } - return false; + + // The sink takes bitmaps with monotonically increasing, non-offset frame timestamps. Ensure + // the produced timestamps include the stream offset. + OffsetTimestampIterator offsetTimestampIterator = + new OffsetTimestampIterator(timestampIterator, inputStreamOffsetUs); + if (!checkStateNotNull(videoFrameProcessor) + .queueInputBitmap(inputBitmap, offsetTimestampIterator)) { + return false; + } + + // Create a copy of iterator because we need to take the next timestamp but we must not alter + // the state of the iterator. + TimestampIterator copyTimestampIterator = offsetTimestampIterator.copyOf(); + long bufferPresentationTimeUs = copyTimestampIterator.next(); + long lastBufferPresentationTimeUs = copyTimestampIterator.getLastTimestampUs(); + checkState(lastBufferPresentationTimeUs != C.TIME_UNSET); + maybeSetStreamOffsetChange(bufferPresentationTimeUs); + this.lastBufferPresentationTimeUs = lastBufferPresentationTimeUs; + finalBufferPresentationTimeUs = lastBufferPresentationTimeUs; + return true; } @Override @@ -765,6 +778,35 @@ public final class CompositingVideoSinkProvider inputStreamOffsetUs = streamOffsetUs; } + private void maybeSetStreamOffsetChange(long bufferPresentationTimeUs) { + if (pendingInputStreamOffsetChange) { + compositingVideoSinkProvider.onStreamOffsetChange( + /* bufferPresentationTimeUs= */ bufferPresentationTimeUs, + /* streamOffsetUs= */ inputStreamOffsetUs); + pendingInputStreamOffsetChange = false; + } + } + + /** + * Attempt to register any pending input stream to the video graph input and returns {@code + * true} if a pending stream was registered and/or there is no pending input stream waiting for + * registration, hence it's safe to queue images or frames to the video graph input. + */ + private boolean maybeRegisterPendingInputStream() { + if (pendingInputStreamBufferPresentationTimeUs == C.TIME_UNSET) { + return true; + } + // An input stream is fully decoded, wait until all of its frames are released before queueing + // input frame from the next input stream. + if (compositingVideoSinkProvider.hasReleasedFrame( + pendingInputStreamBufferPresentationTimeUs)) { + maybeRegisterInputStream(); + pendingInputStreamBufferPresentationTimeUs = C.TIME_UNSET; + return true; + } + return false; + } + private void maybeRegisterInputStream() { if (inputFormat == null) { return; @@ -785,6 +827,7 @@ public final class CompositingVideoSinkProvider inputFormat.height) .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) .build()); + finalBufferPresentationTimeUs = C.TIME_UNSET; } // CompositingVideoSinkProvider.Listener implementation @@ -957,4 +1000,43 @@ public final class CompositingVideoSinkProvider listener); } } + + /** + * A {@link TimestampIterator} that wraps another {@link TimestampIterator} and adds an offset to + * the returnd timestamps. + */ + private static class OffsetTimestampIterator implements TimestampIterator { + + private final TimestampIterator timestampIterator; + private final long offset; + + public OffsetTimestampIterator(TimestampIterator timestampIterator, long offset) { + this.timestampIterator = timestampIterator; + this.offset = offset; + } + + @Override + public boolean hasNext() { + return timestampIterator.hasNext(); + } + + @Override + public long next() { + return offset + timestampIterator.next(); + } + + @Override + public long getLastTimestampUs() { + long last = timestampIterator.getLastTimestampUs(); + if (last != C.TIME_UNSET) { + last += offset; + } + return last; + } + + @Override + public TimestampIterator copyOf() { + return new OffsetTimestampIterator(timestampIterator.copyOf(), offset); + } + } }