diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java index 6696d82d27..b71c3e06f3 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java @@ -113,12 +113,12 @@ import java.util.concurrent.Executor; @Override public void signalEndOfCurrentInputStream() { - throw new UnsupportedOperationException(); + videoFrameRenderControl.signalEndOfInput(); } @Override public boolean isEnded() { - throw new UnsupportedOperationException(); + return videoFrameRenderControl.isEnded(); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index 5eeb688ff4..ab1a6736bf 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -252,6 +252,11 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private @State int state; @Nullable private Renderer.WakeupListener wakeupListener; + /** The buffer presentation time, in microseconds, of the final frame in the stream. */ + private long finalBufferPresentationTimeUs; + + private boolean hasSignaledEndOfCurrentInputStream; + /** * Converts the buffer timestamp (the player position, with renderer offset) to the composition * timestamp, in microseconds. The composition time starts from zero, add this adjustment to @@ -275,6 +280,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video state = STATE_CREATED; videoGraphOutputFormat = new Format.Builder().build(); addListener(inputVideoSink); + finalBufferPresentationTimeUs = C.TIME_UNSET; } /** @@ -378,6 +384,12 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video outputStreamStartPositionUs = newOutputStreamStartPositionUs; } videoFrameRenderControl.onFrameAvailableForRendering(bufferPresentationTimeUs); + if (finalBufferPresentationTimeUs != C.TIME_UNSET + && bufferPresentationTimeUs >= finalBufferPresentationTimeUs) { + // TODO b/257464707 - Support extensively modified media. + defaultVideoSink.signalEndOfCurrentInputStream(); + hasSignaledEndOfCurrentInputStream = true; + } } @Override @@ -454,8 +466,10 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video /* rendererOtherwiseReady= */ rendererOtherwiseReady && pendingFlushCount == 0); } - private boolean hasReleasedFrame(long presentationTimeUs) { - return pendingFlushCount == 0 && videoFrameRenderControl.hasReleasedFrame(presentationTimeUs); + private boolean isEnded() { + return pendingFlushCount == 0 + && hasSignaledEndOfCurrentInputStream + && defaultVideoSink.isEnded(); } /** @@ -484,6 +498,8 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video defaultVideoSink.setStreamTimestampInfo( lastStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET); } + finalBufferPresentationTimeUs = C.TIME_UNSET; + hasSignaledEndOfCurrentInputStream = false; // Handle pending video graph callbacks to ensure video size changes reach the video render // control. checkStateNotNull(handler).post(() -> pendingFlushCount--); @@ -522,9 +538,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private long inputBufferTimestampAdjustmentUs; private long lastResetPositionUs; - /** The buffer presentation time, in microseconds, of the final frame in the stream. */ - private long finalBufferPresentationTimeUs; - /** * The buffer presentation timestamp, in microseconds, of the most recently registered frame. */ @@ -541,7 +554,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); videoEffects = ImmutableList.of(); - finalBufferPresentationTimeUs = C.TIME_UNSET; lastBufferPresentationTimeUs = C.TIME_UNSET; listener = VideoSink.Listener.NO_OP; listenerExecutor = NO_OP_EXECUTOR; @@ -590,7 +602,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video if (isInitialized()) { videoFrameProcessor.flush(); } - finalBufferPresentationTimeUs = C.TIME_UNSET; lastBufferPresentationTimeUs = C.TIME_UNSET; PlaybackVideoGraphWrapper.this.flush(resetPosition); // Don't change input stream start position or reset the pending input stream timestamp info @@ -612,9 +623,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video @Override public boolean isEnded() { - return isInitialized() - && finalBufferPresentationTimeUs != C.TIME_UNSET - && PlaybackVideoGraphWrapper.this.hasReleasedFrame(finalBufferPresentationTimeUs); + return isInitialized() && PlaybackVideoGraphWrapper.this.isEnded(); } @Override @@ -630,6 +639,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video this.inputType = inputType; this.inputFormat = format; finalBufferPresentationTimeUs = C.TIME_UNSET; + hasSignaledEndOfCurrentInputStream = false; registerInputStream(format); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java index 16db1ebeb7..2d65efecec 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java @@ -77,8 +77,12 @@ import androidx.media3.exoplayer.ExoPlaybackException; /** A queue of unprocessed input frame timestamps. */ private final LongArrayQueue presentationTimestampsUs; - private long lastInputPresentationTimeUs; - private long lastOutputPresentationTimeUs; + private long latestInputPresentationTimeUs; + private long latestOutputPresentationTimeUs; + + /** The presentation time of the final frame to render. */ + private long lastPresentationTimeUs; + private VideoSize outputVideoSize; private long outputStreamStartPositionUs; @@ -91,16 +95,18 @@ import androidx.media3.exoplayer.ExoPlaybackException; videoSizes = new TimedValueQueue<>(); streamStartPositionsUs = new TimedValueQueue<>(); presentationTimestampsUs = new LongArrayQueue(); - lastInputPresentationTimeUs = C.TIME_UNSET; + latestInputPresentationTimeUs = C.TIME_UNSET; outputVideoSize = VideoSize.UNKNOWN; - lastOutputPresentationTimeUs = C.TIME_UNSET; + latestOutputPresentationTimeUs = C.TIME_UNSET; + lastPresentationTimeUs = C.TIME_UNSET; } /** Flushes the renderer. */ public void flush() { presentationTimestampsUs.clear(); - lastInputPresentationTimeUs = C.TIME_UNSET; - lastOutputPresentationTimeUs = C.TIME_UNSET; + latestInputPresentationTimeUs = C.TIME_UNSET; + latestOutputPresentationTimeUs = C.TIME_UNSET; + lastPresentationTimeUs = C.TIME_UNSET; if (streamStartPositionsUs.size() > 0) { // There is a pending streaming start position change. If seeking within the same stream, keep // the pending start position with min timestamp to ensure the start position is applied on @@ -120,18 +126,6 @@ import androidx.media3.exoplayer.ExoPlaybackException; } } - /** - * Returns whether the renderer has released a frame after a specific presentation timestamp. - * - * @param presentationTimeUs The requested timestamp, in microseconds. - * @return Whether the renderer has released a frame with a timestamp greater than or equal to - * {@code presentationTimeUs}. - */ - public boolean hasReleasedFrame(long presentationTimeUs) { - return lastOutputPresentationTimeUs != C.TIME_UNSET - && lastOutputPresentationTimeUs >= presentationTimeUs; - } - /** * Incrementally renders available video frames. * @@ -160,17 +154,17 @@ import androidx.media3.exoplayer.ExoPlaybackException; return; case VideoFrameReleaseControl.FRAME_RELEASE_SKIP: case VideoFrameReleaseControl.FRAME_RELEASE_DROP: - lastOutputPresentationTimeUs = presentationTimeUs; + latestOutputPresentationTimeUs = presentationTimeUs; dropFrame(); break; case VideoFrameReleaseControl.FRAME_RELEASE_IGNORE: // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush // VideoGraph input frames in this case. - lastOutputPresentationTimeUs = presentationTimeUs; + latestOutputPresentationTimeUs = presentationTimeUs; break; case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED: - lastOutputPresentationTimeUs = presentationTimeUs; + latestOutputPresentationTimeUs = presentationTimeUs; renderFrame( /* shouldRenderImmediately= */ frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -184,13 +178,13 @@ import androidx.media3.exoplayer.ExoPlaybackException; /** Called when the size of the available frames has changed. */ public void onVideoSizeChanged(int width, int height) { videoSizes.add( - lastInputPresentationTimeUs == C.TIME_UNSET ? 0 : lastInputPresentationTimeUs + 1, + latestInputPresentationTimeUs == C.TIME_UNSET ? 0 : latestInputPresentationTimeUs + 1, new VideoSize(width, height)); } public void onStreamStartPositionChanged(long streamStartPositionUs) { streamStartPositionsUs.add( - lastInputPresentationTimeUs == C.TIME_UNSET ? 0 : lastInputPresentationTimeUs + 1, + latestInputPresentationTimeUs == C.TIME_UNSET ? 0 : latestInputPresentationTimeUs + 1, streamStartPositionUs); } @@ -201,8 +195,31 @@ import androidx.media3.exoplayer.ExoPlaybackException; */ public void onFrameAvailableForRendering(long presentationTimeUs) { presentationTimestampsUs.add(presentationTimeUs); - lastInputPresentationTimeUs = presentationTimeUs; - // TODO b/257464707 - Support extensively modified media. + latestInputPresentationTimeUs = presentationTimeUs; + lastPresentationTimeUs = C.TIME_UNSET; + } + + /** + * Signals the end of input. + * + *

If a frame becomes {@linkplain #onFrameAvailableForRendering(long) available} after calling + * this method, the end of input signal is ignored. + */ + public void signalEndOfInput() { + lastPresentationTimeUs = latestInputPresentationTimeUs; + } + + /** + * Returns whether all the frames have been rendered to the output surface. + * + *

This method returns {@code true} if the last frame that became {@linkplain + * #onFrameAvailableForRendering(long) available} before {@linkplain #signalEndOfInput() + * signalling the end of input} has been rendered, and if no frame has become available in the + * mean time. + */ + public boolean isEnded() { + return lastPresentationTimeUs != C.TIME_UNSET + && latestOutputPresentationTimeUs == lastPresentationTimeUs; } private void dropFrame() { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java index 7ab727c642..dc92b8c20d 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java @@ -230,17 +230,17 @@ public class VideoFrameRenderControlTest { } @Test - public void hasReleasedFrame_noFrameReleased_returnsFalse() { + public void isEnded_endOfInputNotSignaled_returnsFalse() { VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); VideoFrameRenderControl videoFrameRenderControl = new VideoFrameRenderControl( mock(VideoFrameRenderControl.FrameRenderer.class), videoFrameReleaseControl); - assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isFalse(); + assertThat(videoFrameRenderControl.isEnded()).isFalse(); } @Test - public void hasReleasedFrame_frameIsReleased_returnsTrue() throws Exception { + public void isEnded_endOfInputSignaled_returnsTrue() throws Exception { VideoFrameRenderControl.FrameRenderer frameRenderer = mock(VideoFrameRenderControl.FrameRenderer.class); VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); @@ -252,22 +252,13 @@ public class VideoFrameRenderControlTest { /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0); videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + videoFrameRenderControl.signalEndOfInput(); - InOrder inOrder = Mockito.inOrder(frameRenderer); - inOrder - .verify(frameRenderer) - .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); - inOrder - .verify(frameRenderer) - .renderFrame( - /* renderTimeNs= */ anyLong(), - /* presentationTimeUs= */ eq(0L), - /* isFirstFrame= */ eq(true)); - assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isTrue(); + assertThat(videoFrameRenderControl.isEnded()).isTrue(); } @Test - public void hasReleasedFrame_frameIsReleasedAndFlushed_returnsFalse() throws Exception { + public void isEnded_afterFlush_returnsFalse() throws Exception { VideoFrameRenderControl.FrameRenderer frameRenderer = mock(VideoFrameRenderControl.FrameRenderer.class); VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); @@ -279,21 +270,9 @@ public class VideoFrameRenderControlTest { /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0); videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); - - InOrder inOrder = Mockito.inOrder(frameRenderer); - inOrder - .verify(frameRenderer) - .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); - inOrder - .verify(frameRenderer) - .renderFrame( - /* renderTimeNs= */ anyLong(), - /* presentationTimeUs= */ eq(0L), - /* isFirstFrame= */ eq(true)); - videoFrameRenderControl.flush(); - assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isFalse(); + assertThat(videoFrameRenderControl.isEnded()).isFalse(); } private static VideoFrameReleaseControl createVideoFrameReleaseControl() {