From 0f08c97221eba37e81f8e917c544573e5fbed5f2 Mon Sep 17 00:00:00 2001 From: kimvde Date: Mon, 28 Apr 2025 04:57:35 -0700 Subject: [PATCH] Handle rendering in VideoGraph time Before this CL, the buffer adjustment (which allows to convert from ExoPlayer time to VideoGraph time) was added to the frame timestamps before feeding them to the VideoGraph, and then subtracted at the VideoGraph output. The playback position and stream start position used for rendering were in ExoPlayer time. This doesn't work for multi-sequence though because the adjustment might be different depending on the player (after a seek for example). To solve this problem, this CL handles rendering in VideoGraph time instead of ExoPlayer time. More concretely, the VideoGraph output timestamps are unchanged, and the playback position and stream start position are converted to VideoGraph time. PiperOrigin-RevId: 752260744 --- .../exoplayer/video/DefaultVideoSink.java | 25 ++- .../video/MediaCodecVideoRenderer.java | 8 +- .../video/PlaybackVideoGraphWrapper.java | 151 ++++++++++-------- .../video/VideoFrameRenderControl.java | 6 +- .../media3/exoplayer/video/VideoSink.java | 10 +- .../transformer/BufferingVideoSink.java | 8 +- .../transformer/SequenceRenderersFactory.java | 12 +- 7 files changed, 129 insertions(+), 91 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java index a653cce23c..16270d58e4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java @@ -45,6 +45,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * * *

The {@linkplain #getInputSurface() input} and {@linkplain #setOutputSurfaceInfo(Surface, Size) @@ -59,7 +61,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Nullable private Surface outputSurface; private Format inputFormat; private long streamStartPositionUs; - private long bufferTimestampAdjustmentUs; private Listener listener; private Executor listenerExecutor; private VideoFrameMetadataListener videoFrameMetadataListener; @@ -104,6 +105,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return true; } + /** + * {@inheritDoc} + * + *

This method will always throw an {@link UnsupportedOperationException}. + */ @Override public void redraw() { throw new UnsupportedOperationException(); @@ -163,9 +169,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new UnsupportedOperationException(); } + /** + * {@inheritDoc} + * + *

This method will always throw an {@link UnsupportedOperationException}. + */ @Override public void setBufferTimestampAdjustmentUs(long bufferTimestampAdjustmentUs) { - this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs; + throw new UnsupportedOperationException(); } @Override @@ -220,8 +231,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public boolean handleInputFrame( long framePresentationTimeUs, VideoFrameHandler videoFrameHandler) { videoFrameHandlers.add(videoFrameHandler); - long bufferPresentationTimeUs = framePresentationTimeUs - bufferTimestampAdjustmentUs; - videoFrameRenderControl.onFrameAvailableForRendering(bufferPresentationTimeUs); + videoFrameRenderControl.onFrameAvailableForRendering(framePresentationTimeUs); listenerExecutor.execute(() -> listener.onFrameAvailableForRendering()); return true; } @@ -232,7 +242,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; *

This method will always throw an {@link UnsupportedOperationException}. */ @Override - public boolean handleInputBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator) { + public boolean handleInputBitmap(Bitmap inputBitmap, TimestampIterator bufferTimestampIterator) { throw new UnsupportedOperationException(); } @@ -269,8 +279,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void renderFrame( - long renderTimeNs, long bufferPresentationTimeUs, boolean isFirstFrame) { + public void renderFrame(long renderTimeNs, long framePresentationTimeUs, boolean isFirstFrame) { if (isFirstFrame && outputSurface != null) { listenerExecutor.execute(() -> listener.onFirstFrameRendered()); } @@ -278,7 +287,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // onVideoSizeChanged is announced after the first frame is available for rendering. Format format = outputFormat == null ? new Format.Builder().build() : outputFormat; videoFrameMetadataListener.onVideoFrameAboutToBeRendered( - /* presentationTimeUs= */ bufferPresentationTimeUs, + /* presentationTimeUs= */ framePresentationTimeUs, /* releaseTimeNs= */ renderTimeNs, format, /* mediaFormat= */ null); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 9ebadf2a8e..316fe82eec 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -626,6 +626,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer boolean isLastFrame, boolean treatDroppedBuffersAsSkipped) throws ExoPlaybackException { + if (videoSink != null && ownsVideoSink) { + // When using PlaybackVideoGraphWrapper, positionUs is shifted by the buffer timestamp + // adjustment. Shift it back to the player position. + positionUs -= getBufferTimestampAdjustmentUs(); + } if (minEarlyUsToDropDecoderInput != C.TIME_UNSET) { // TODO: b/161996553 - Remove the isAwayFromLastResetPosition check when audio pre-rolling // is implemented correctly. Audio codecs such as Opus require pre-roll samples to be decoded @@ -1734,9 +1739,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer skipOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } - long framePresentationTimeUs = bufferPresentationTimeUs + getBufferTimestampAdjustmentUs(); return videoSink.handleInputFrame( - framePresentationTimeUs, + bufferPresentationTimeUs, new VideoSink.VideoFrameHandler() { @Override public void render(long renderTimestampNs) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index c9f24411da..02498ff163 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -326,23 +326,16 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { private @State int state; /** - * The buffer presentation time of the frame most recently output by the video graph, in + * The frame presentation time of the frame most recently output by the video graph, in * microseconds. */ - private long lastOutputBufferPresentationTimeUs; + private long lastOutputFramePresentationTimeUs; - /** The buffer presentation time, in microseconds, of the final frame in the stream. */ - private long finalBufferPresentationTimeUs; + /** The frame presentation time, in microseconds, of the final frame in the stream. */ + private long finalFramePresentationTimeUs; private boolean hasSignaledEndOfVideoGraphOutputStream; - /** - * Converts the buffer timestamp (the player position, with renderer offset) to the composition - * timestamp, in microseconds. The composition time starts from zero, add this adjustment to - * buffer timestamp to get the composition time. - */ - private long bufferTimestampAdjustmentUs; - private int totalVideoInputCount; private int registeredVideoInputCount; @@ -372,8 +365,8 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { requestOpenGlToneMapping = builder.requestOpenGlToneMapping; videoGraphOutputFormat = new Format.Builder().build(); outputStreamStartPositionUs = C.TIME_UNSET; - lastOutputBufferPresentationTimeUs = C.TIME_UNSET; - finalBufferPresentationTimeUs = C.TIME_UNSET; + lastOutputFramePresentationTimeUs = C.TIME_UNSET; + finalFramePresentationTimeUs = C.TIME_UNSET; totalVideoInputCount = C.LENGTH_UNSET; state = STATE_CREATED; } @@ -492,12 +485,11 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { listener.onFrameAvailableForRendering(); } - long bufferPresentationTimeUs = framePresentationTimeUs - bufferTimestampAdjustmentUs; if (isRedrawnFrame) { // Redrawn frames are rendered directly in the processing pipeline. if (videoFrameMetadataListener != null) { videoFrameMetadataListener.onVideoFrameAboutToBeRendered( - /* presentationTimeUs= */ bufferPresentationTimeUs, + /* presentationTimeUs= */ framePresentationTimeUs, /* releaseTimeNs= */ C.TIME_UNSET, videoGraphOutputFormat, /* mediaFormat= */ null); @@ -507,8 +499,8 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { // The frame presentation time is relative to the start of the Composition and without the // renderer offset - lastOutputBufferPresentationTimeUs = bufferPresentationTimeUs; - StreamChangeInfo streamChangeInfo = pendingStreamChanges.pollFloor(bufferPresentationTimeUs); + lastOutputFramePresentationTimeUs = framePresentationTimeUs; + StreamChangeInfo streamChangeInfo = pendingStreamChanges.pollFloor(framePresentationTimeUs); if (streamChangeInfo != null) { outputStreamStartPositionUs = streamChangeInfo.startPositionUs; outputStreamFirstFrameReleaseInstruction = streamChangeInfo.firstFrameReleaseInstruction; @@ -516,8 +508,8 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { } defaultVideoSink.handleInputFrame(framePresentationTimeUs, videoFrameHandler); boolean isLastFrame = - finalBufferPresentationTimeUs != C.TIME_UNSET - && bufferPresentationTimeUs >= finalBufferPresentationTimeUs; + finalFramePresentationTimeUs != C.TIME_UNSET + && framePresentationTimeUs >= finalFramePresentationTimeUs; if (isLastFrame) { signalEndOfVideoGraphOutputStream(); } @@ -667,8 +659,8 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { outputStreamFirstFrameReleaseInstruction = streamChangeInfo.firstFrameReleaseInstruction; onOutputStreamChanged(); } - lastOutputBufferPresentationTimeUs = C.TIME_UNSET; - finalBufferPresentationTimeUs = C.TIME_UNSET; + lastOutputFramePresentationTimeUs = C.TIME_UNSET; + finalFramePresentationTimeUs = C.TIME_UNSET; hasSignaledEndOfVideoGraphOutputStream = false; // Handle pending video graph callbacks to ensure video size changes reach the video render // control. @@ -693,11 +685,6 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { defaultVideoSink.setPlaybackSpeed(speed); } - private void setBufferTimestampAdjustment(long bufferTimestampAdjustmentUs) { - this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs; - defaultVideoSink.setBufferTimestampAdjustmentUs(bufferTimestampAdjustmentUs); - } - private void setChangeFrameRateStrategy( @C.VideoChangeFrameRateStrategy int changeFrameRateStrategy) { defaultVideoSink.setChangeFrameRateStrategy(changeFrameRateStrategy); @@ -736,10 +723,8 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { private @InputType int inputType; private long inputBufferTimestampAdjustmentUs; - /** - * The buffer presentation timestamp, in microseconds, of the most recently registered frame. - */ - private long lastBufferPresentationTimeUs; + /** The frame presentation timestamp, in microseconds, of the most recently registered frame. */ + private long lastFramePresentationTimeUs; private VideoSink.Listener listener; private Executor listenerExecutor; @@ -755,7 +740,7 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { videoFrameProcessorMaxPendingFrameCount = getMaxPendingFramesCountForMediaCodecDecoders(context); videoEffects = ImmutableList.of(); - lastBufferPresentationTimeUs = C.TIME_UNSET; + lastFramePresentationTimeUs = C.TIME_UNSET; listener = VideoSink.Listener.NO_OP; listenerExecutor = NO_OP_EXECUTOR; } @@ -799,10 +784,10 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { } // Resignal EOS only for the last item. boolean needsResignalEndOfCurrentInputStream = signaledEndOfStream; - long replayedPresentationTimeUs = lastOutputBufferPresentationTimeUs; + long replayedPresentationTimeUs = lastOutputFramePresentationTimeUs; PlaybackVideoGraphWrapper.this.flush(/* resetPosition= */ false); checkNotNull(videoGraph).redraw(); - lastOutputBufferPresentationTimeUs = replayedPresentationTimeUs; + lastOutputFramePresentationTimeUs = replayedPresentationTimeUs; if (needsResignalEndOfCurrentInputStream) { signalEndOfCurrentInputStream(); } @@ -813,7 +798,7 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { if (isInitialized()) { checkNotNull(videoGraph).flush(); } - lastBufferPresentationTimeUs = C.TIME_UNSET; + lastFramePresentationTimeUs = C.TIME_UNSET; PlaybackVideoGraphWrapper.this.flush(resetPosition); signaledEndOfStream = false; // Don't change input stream start position or reset the pending input stream timestamp info @@ -830,8 +815,8 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { @Override public void signalEndOfCurrentInputStream() { - finalBufferPresentationTimeUs = lastBufferPresentationTimeUs; - if (lastOutputBufferPresentationTimeUs >= finalBufferPresentationTimeUs) { + finalFramePresentationTimeUs = lastFramePresentationTimeUs; + if (lastOutputFramePresentationTimeUs >= finalFramePresentationTimeUs) { PlaybackVideoGraphWrapper.this.signalEndOfVideoGraphOutputStream(); } } @@ -863,17 +848,23 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { this.videoEffects = ImmutableList.copyOf(videoEffects); this.inputType = inputType; this.inputFormat = format; - finalBufferPresentationTimeUs = C.TIME_UNSET; + finalFramePresentationTimeUs = C.TIME_UNSET; hasSignaledEndOfVideoGraphOutputStream = false; registerInputStream(format); - // Input timestamps should always be positive because they are offset by ExoPlayer. Adding a - // stream change info to the queue with timestamp 0 should therefore always apply it as long - // as it is the only one in the queue. - long fromTimestampUs = - lastBufferPresentationTimeUs == C.TIME_UNSET ? 0 : lastBufferPresentationTimeUs + 1; + long fromTimestampUs; + if (lastFramePresentationTimeUs == C.TIME_UNSET) { + // Add a stream change info to the queue with a large negative timestamp to always apply it + // as long as it is the only one in the queue. + fromTimestampUs = Long.MIN_VALUE / 2; + } else { + fromTimestampUs = lastFramePresentationTimeUs + 1; + } pendingStreamChanges.add( fromTimestampUs, - new StreamChangeInfo(startPositionUs, firstFrameReleaseInstruction, fromTimestampUs)); + new StreamChangeInfo( + /* startPositionUs= */ startPositionUs + inputBufferTimestampAdjustmentUs, + firstFrameReleaseInstruction, + fromTimestampUs)); } @Override @@ -954,11 +945,6 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { @Override public void setBufferTimestampAdjustmentUs(long bufferTimestampAdjustmentUs) { inputBufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs; - // The buffer timestamp adjustment is only allowed to change after a flush to make sure that - // the buffer timestamps are increasing. We can update the buffer timestamp adjustment - // directly at the output of the VideoGraph because no frame has been input yet following the - // flush. - PlaybackVideoGraphWrapper.this.setBufferTimestampAdjustment(inputBufferTimestampAdjustmentUs); } @Override @@ -981,7 +967,7 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { @Override public boolean handleInputFrame( - long framePresentationTimeUs, VideoFrameHandler videoFrameHandler) { + long bufferPresentationTimeUs, VideoFrameHandler videoFrameHandler) { checkState(isInitialized()); if (!shouldRenderToInputVideoSink()) { @@ -1002,10 +988,11 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { // duration of the first video. Thus this correction is needed to account for the different // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. // - // inputBufferTimestampAdjustmentUs adjusts the frame presentation time (which is relative to - // the start of a composition) to the buffer timestamp (that corresponds to the player - // position). - lastBufferPresentationTimeUs = framePresentationTimeUs - inputBufferTimestampAdjustmentUs; + // inputBufferTimestampAdjustmentUs adjusts the buffer timestamp (that corresponds to the + // player position) to the frame presentation time (which is relative to the start of a + // composition). + long framePresentationTimeUs = bufferPresentationTimeUs + inputBufferTimestampAdjustmentUs; + lastFramePresentationTimeUs = framePresentationTimeUs; // Use the frame presentation time as render time so that the SurfaceTexture is accompanied // by this timestamp. Setting a realtime based release time is only relevant when rendering to // a SurfaceView, but we render to a surface in this case. @@ -1014,25 +1001,29 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { } @Override - public boolean handleInputBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator) { + public boolean handleInputBitmap( + Bitmap inputBitmap, TimestampIterator bufferTimestampIterator) { checkState(isInitialized()); - if (!shouldRenderToInputVideoSink() - || !checkNotNull(videoGraph) - .queueInputBitmap(inputIndex, inputBitmap, timestampIterator)) { + if (!shouldRenderToInputVideoSink()) { + return false; + } + TimestampIterator frameTimestampIterator = + new ShiftingTimestampIterator(bufferTimestampIterator, inputBufferTimestampAdjustmentUs); + if (!checkNotNull(videoGraph) + .queueInputBitmap(inputIndex, inputBitmap, frameTimestampIterator)) { return false; } - // TimestampIterator generates frame time. - long lastBufferPresentationTimeUs = - timestampIterator.getLastTimestampUs() - inputBufferTimestampAdjustmentUs; - checkState(lastBufferPresentationTimeUs != C.TIME_UNSET); - this.lastBufferPresentationTimeUs = lastBufferPresentationTimeUs; + long lastFramePresentationTimeUs = frameTimestampIterator.getLastTimestampUs(); + checkState(lastFramePresentationTimeUs != C.TIME_UNSET); + this.lastFramePresentationTimeUs = lastFramePresentationTimeUs; return true; } @Override public void render(long positionUs, long elapsedRealtimeUs) throws VideoSinkException { - PlaybackVideoGraphWrapper.this.render(positionUs, elapsedRealtimeUs); + PlaybackVideoGraphWrapper.this.render( + /* positionUs= */ positionUs + inputBufferTimestampAdjustmentUs, elapsedRealtimeUs); } @Override @@ -1152,6 +1143,40 @@ public final class PlaybackVideoGraphWrapper implements VideoGraph.Listener { } } + private static final class ShiftingTimestampIterator implements TimestampIterator { + + private final TimestampIterator timestampIterator; + private final long shift; + + public ShiftingTimestampIterator(TimestampIterator timestampIterator, long shift) { + this.timestampIterator = timestampIterator; + this.shift = shift; + } + + @Override + public boolean hasNext() { + return timestampIterator.hasNext(); + } + + @Override + public long next() { + return timestampIterator.next() + shift; + } + + @Override + public TimestampIterator copyOf() { + return new ShiftingTimestampIterator(timestampIterator.copyOf(), shift); + } + + @Override + public long getLastTimestampUs() { + long unshiftedLastTimestampUs = timestampIterator.getLastTimestampUs(); + return unshiftedLastTimestampUs == C.TIME_UNSET + ? C.TIME_UNSET + : unshiftedLastTimestampUs + shift; + } + } + /** Delays reflection for loading a {@link VideoGraph.Factory SingleInputVideoGraph} instance. */ private static final class ReflectiveSingleInputVideoGraphFactory implements VideoGraph.Factory { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java index 4c8b9c1907..dd0fa34c2e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java @@ -185,8 +185,12 @@ import androidx.media3.exoplayer.ExoPlaybackException; videoFrameReleaseControl.onStreamChanged(firstFrameReleaseInstruction); outputStreamStartPositionUs = streamStartPositionUs; } else { + // Add a start position to the queue with a large negative timestamp to always apply it as + // long as it is the only one in the queue. streamStartPositionsUs.add( - latestInputPresentationTimeUs == C.TIME_UNSET ? 0 : latestInputPresentationTimeUs + 1, + latestInputPresentationTimeUs == C.TIME_UNSET + ? Long.MIN_VALUE / 2 + : latestInputPresentationTimeUs + 1, streamStartPositionUs); } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java index 2acb03f55d..6ec49ee8e2 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java @@ -277,12 +277,12 @@ public interface VideoSink { *

Must be called after the corresponding stream is {@linkplain #onInputStreamChanged(int, * Format, long, int, List) signaled}. * - * @param framePresentationTimeUs The frame's presentation time, in microseconds. + * @param bufferPresentationTimeUs The buffer presentation time, in microseconds. * @param videoFrameHandler The {@link VideoFrameHandler} used to handle the input frame. * @return Whether the frame was handled successfully. If {@code false}, the caller can try again * later. */ - boolean handleInputFrame(long framePresentationTimeUs, VideoFrameHandler videoFrameHandler); + boolean handleInputFrame(long bufferPresentationTimeUs, VideoFrameHandler videoFrameHandler); /** * Handles an input {@link Bitmap}. @@ -291,12 +291,12 @@ public interface VideoSink { * Format, long, int, List) signaled}. * * @param inputBitmap The {@link Bitmap} to queue to the video sink. - * @param timestampIterator The times within the current stream that the bitmap should be shown - * at. The timestamps should be monotonically increasing. + * @param bufferTimestampIterator The buffer presentation times within the current stream that the + * bitmap should be shown at. The timestamps should be monotonically increasing. * @return Whether the bitmap was queued successfully. If {@code false}, the caller can try again * later. */ - boolean handleInputBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator); + boolean handleInputBitmap(Bitmap inputBitmap, TimestampIterator bufferTimestampIterator); /** * Incrementally renders processed video frames to the output surface. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/BufferingVideoSink.java b/libraries/transformer/src/main/java/androidx/media3/transformer/BufferingVideoSink.java index 7d7b3910e7..0dd63ac284 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/BufferingVideoSink.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/BufferingVideoSink.java @@ -233,9 +233,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; */ @Override public boolean handleInputFrame( - long framePresentationTimeUs, VideoFrameHandler videoFrameHandler) { + long bufferPresentationTimeUs, VideoFrameHandler videoFrameHandler) { return videoSink != null - && videoSink.handleInputFrame(framePresentationTimeUs, videoFrameHandler); + && videoSink.handleInputFrame(bufferPresentationTimeUs, videoFrameHandler); } /** @@ -245,8 +245,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * sink} is {@code null}. */ @Override - public boolean handleInputBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator) { - return videoSink != null && videoSink.handleInputBitmap(inputBitmap, timestampIterator); + public boolean handleInputBitmap(Bitmap inputBitmap, TimestampIterator bufferTimestampIterator) { + return videoSink != null && videoSink.handleInputBitmap(inputBitmap, bufferTimestampIterator); } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java index 67ff6efc6b..143215eb09 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java @@ -500,7 +500,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private long streamStartPositionUs; private boolean mayRenderStartOfStream; private @VideoSink.FirstFrameReleaseInstruction int nextFirstFrameReleaseInstruction; - private long offsetToCompositionTimeUs; private @MonotonicNonNull WakeupListener wakeupListener; public SequenceImageRenderer( @@ -598,7 +597,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // The media item might have been repeated in the sequence. int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid); currentEditedMediaItem = sequence.editedMediaItems.get(mediaItemIndex); - offsetToCompositionTimeUs = getOffsetToCompositionTimeUs(sequence, mediaItemIndex, offsetUs); + long offsetToCompositionTimeUs = + getOffsetToCompositionTimeUs(sequence, mediaItemIndex, offsetUs); videoSink.setBufferTimestampAdjustmentUs(offsetToCompositionTimeUs); timestampIterator = createTimestampIterator(/* positionUs= */ startPositionUs); videoEffects = currentEditedMediaItem.effects.videoEffects; @@ -663,14 +663,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private ConstantRateTimestampIterator createTimestampIterator(long positionUs) { - long streamOffsetUs = getStreamOffsetUs(); - long imageBaseTimestampUs = streamOffsetUs + offsetToCompositionTimeUs; - long positionWithinImage = positionUs - streamOffsetUs; - long firstBitmapTimeUs = imageBaseTimestampUs + positionWithinImage; long lastBitmapTimeUs = - imageBaseTimestampUs + checkNotNull(currentEditedMediaItem).getPresentationDurationUs(); + getStreamOffsetUs() + checkNotNull(currentEditedMediaItem).getPresentationDurationUs(); return new ConstantRateTimestampIterator( - /* startPositionUs= */ firstBitmapTimeUs, + /* startPositionUs= */ positionUs, /* endPositionUs= */ lastBitmapTimeUs, DEFAULT_FRAME_RATE); }