diff --git a/RELEASENOTES.md b/RELEASENOTES.md index d0c1cff582..c1a3a290c9 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -76,6 +76,8 @@ * Fix issue where HDR color info handling causes codec mishavior and prevents adaptive format switches for SDR video tracks ([#1158](https://github.com/androidx/media/issues/1158)). + * Fix issue where `Listener.onRenderedFirstFrame()` arrives too early when + switching surfaces mid-playback. * Text: * WebVTT: Prevent directly consecutive cues from creating spurious additional `CuesWithTiming` instances from `WebvttParser.parse` diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 63e44611cc..27edc90f99 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -660,7 +660,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } videoFrameReleaseControl.reset(); if (joining) { - videoFrameReleaseControl.join(); + // Don't render next frame immediately to let the codec catch up with the playback position + // first. This prevents a stuttering effect caused by showing the first frame and then + // dropping many of the subsequent frames during the catch up phase. + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false); } maybeSetupTunnelingForFirstFrame(); consecutiveDroppedFrameCount = 0; @@ -835,7 +838,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer // If we know the video size, report it again immediately. maybeRenotifyVideoSizeChanged(); if (state == STATE_STARTED) { - videoFrameReleaseControl.join(); + // We want to "join" playback to prevent an intermediate buffering state in the player + // before we rendered the new first frame. Since there is no reason to believe the next + // frame is delayed and the renderer needs to catch up, we still request to render the + // next frame as soon as possible. + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true); } // When effects previewing is enabled, set display surface and an unknown size. if (videoSinkProvider.isInitialized()) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java index dc9dcd933c..1b29ff95b6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java @@ -175,6 +175,7 @@ public final class VideoFrameReleaseControl { private long lastReleaseRealtimeUs; private long lastPresentationTimeUs; private long joiningDeadlineMs; + private boolean joiningRenderNextFrameImmediately; private float playbackSpeed; private Clock clock; @@ -298,8 +299,17 @@ public final class VideoFrameReleaseControl { } } - /** Joins the release control to a new stream. */ - public void join() { + /** + * Joins the release control to a new stream. + * + *

The release control will pretend to be {@linkplain #isReady ready} for short time even if + * the first frame hasn't been rendered yet to avoid interrupting an ongoing playback. + * + * @param renderNextFrameImmediately Whether the next frame should be released as soon as possible + * or only at its preferred scheduled release time. + */ + public void join(boolean renderNextFrameImmediately) { + joiningRenderNextFrameImmediately = renderNextFrameImmediately; joiningDeadlineMs = allowedJoiningTimeMs > 0 ? (clock.elapsedRealtime() + allowedJoiningTimeMs) : C.TIME_UNSET; } @@ -353,8 +363,9 @@ public final class VideoFrameReleaseControl { frameReleaseInfo.releaseTimeNs = frameReleaseHelper.adjustReleaseTime(systemTimeNs + (frameReleaseInfo.earlyUs * 1_000)); frameReleaseInfo.earlyUs = (frameReleaseInfo.releaseTimeNs - systemTimeNs) / 1_000; - // While joining, late frames are skipped. - boolean treatDropAsSkip = joiningDeadlineMs != C.TIME_UNSET; + // While joining, late frames are skipped while we catch up with the playback position. + boolean treatDropAsSkip = + joiningDeadlineMs != C.TIME_UNSET && !joiningRenderNextFrameImmediately; if (frameTimingEvaluator.shouldIgnoreFrame( frameReleaseInfo.earlyUs, positionUs, elapsedRealtimeUs, isLastFrame, treatDropAsSkip)) { return FRAME_RELEASE_IGNORE; @@ -425,8 +436,8 @@ public final class VideoFrameReleaseControl { /** Returns whether a frame should be force released. */ private boolean shouldForceRelease( long positionUs, long earlyUs, long outputStreamStartPositionUs) { - if (joiningDeadlineMs != C.TIME_UNSET) { - // No force releasing during joining. + if (joiningDeadlineMs != C.TIME_UNSET && !joiningRenderNextFrameImmediately) { + // No force releasing of the initial or late frames during joining unless requested. return false; } switch (firstFrameState) { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java index 581d80ac51..109efcf04a 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java @@ -44,25 +44,52 @@ public class VideoFrameReleaseControlTest { } @Test - public void isReady_withinJoiningDeadline_returnsTrue() { + public void isReady_withinJoiningDeadlineWhenRenderingNextFrameImmediately_returnsTrue() { FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100); videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.join(); + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true); assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue(); } @Test - public void isReady_joiningDeadlineExceeded_returnsFalse() { + public void isReady_withinJoiningDeadlineWhenNotRenderingNextFrameImmediately_returnsTrue() { FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100); videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.join(); + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false); + + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue(); + } + + @Test + public void isReady_joiningDeadlineExceededWhenRenderingNextFrameImmediately_returnsFalse() { + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100); + videoFrameReleaseControl.setClock(clock); + + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true); + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue(); + + clock.advanceTime(/* timeDiffMs= */ 101); + + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse(); + } + + @Test + public void isReady_joiningDeadlineExceededWhenNotRenderingNextFrameImmediately_returnsFalse() { + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100); + videoFrameReleaseControl.setClock(clock); + + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false); assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue(); clock.advanceTime(/* timeDiffMs= */ 101); @@ -323,7 +350,9 @@ public class VideoFrameReleaseControlTest { } @Test - public void getFrameReleaseAction_dropWhileJoining_returnsSkip() throws ExoPlaybackException { + public void + getFrameReleaseAction_lateFrameWhileJoiningWhenNotRenderingFirstFrameImmediately_returnsSkip() + throws ExoPlaybackException { VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); @@ -337,26 +366,12 @@ public class VideoFrameReleaseControlTest { /* allowedJoiningTimeMs= */ 1234); videoFrameReleaseControl.setClock(clock); videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - videoFrameReleaseControl.onStarted(); - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - clock.advanceTime(/* timeDiffMs= */ 40); - // Start joining. - videoFrameReleaseControl.join(); + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false); - // Second frame. + // First output is TRY_AGAIN_LATER because the time hasn't moved yet assertThat( videoFrameReleaseControl.getFrameReleaseAction( /* presentationTimeUs= */ 5_000, @@ -365,9 +380,64 @@ public class VideoFrameReleaseControlTest { /* outputStreamStartPositionUs= */ 0, /* isLastFrame= */ false, frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); + // Late frame should be marked as skipped + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 5_000, + /* positionUs= */ 11_000, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP); } + @Test + public void + getFrameReleaseAction_lateFrameWhileJoiningWhenRenderingFirstFrameImmediately_returnsDropAfterInitialImmediateRelease() + throws ExoPlaybackException { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), + new TestFrameTimingEvaluator( + /* shouldForceRelease= */ false, + /* shouldDropFrame= */ true, + /* shouldIgnoreFrame= */ false), + /* allowedJoiningTimeMs= */ 1234); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameReleaseControl.onStarted(); + + // Start joining. + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true); + + // First output is to force render the next frame. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 5_000, + /* positionUs= */ 10_000, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + // Further late frames should be marked as dropped. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 6_000, + /* positionUs= */ 11_000, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP); + } + @Test public void getFrameReleaseAction_shouldIgnore() throws ExoPlaybackException { VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =