loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refine "join" mode in video renderer for surface changes.

Browse Source 
The join mode is used for two cases: surface switching and mid-playback
enabling of video.

In both cases, we want to pretend to be ready despite not having rendered
a new "first frame". So far, we also avoided force-rendering the first
frame immediately because it causes a stuttering effect for the
mid-playback enable case. The surface switch case doesn't have this
stuttering issue as the same codec is used without interruption. Not
force-rendering the frame immediately causes the first-frame rendered
callback to arrive too early though, which may lead to cases where
apps hide shutter views too quickly.

This problem can be solved by only avoiding the force-render for the
mid-playback enabling case, but not for the surface switching case.

PiperOrigin-RevId: 622105916
This commit is contained in:
tonihei 2024-04-05 01:39:51 -07:00 committed by Copybara-Service
parent 8867642681
commit e0fa697edf
4 changed files with 119 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -76,6 +76,8 @@
* Fix issue where HDR color info handling causes codec mishavior and
prevents adaptive format switches for SDR video tracks
([#1158](https://github.com/androidx/media/issues/1158)).
* Fix issue where `Listener.onRenderedFirstFrame()` arrives too early when
switching surfaces mid-playback.
* Text:
* WebVTT: Prevent directly consecutive cues from creating spurious
additional `CuesWithTiming` instances from `WebvttParser.parse`

11
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -660,7 +660,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
}
videoFrameReleaseControl.reset();
if (joining) {
videoFrameReleaseControl.join();
// Don't render next frame immediately to let the codec catch up with the playback position
// first. This prevents a stuttering effect caused by showing the first frame and then
// dropping many of the subsequent frames during the catch up phase.
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
}
maybeSetupTunnelingForFirstFrame();
consecutiveDroppedFrameCount = 0;
@ -835,7 +838,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
// If we know the video size, report it again immediately.
maybeRenotifyVideoSizeChanged();
if (state == STATE_STARTED) {
videoFrameReleaseControl.join();
// We want to "join" playback to prevent an intermediate buffering state in the player
// before we rendered the new first frame. Since there is no reason to believe the next
// frame is delayed and the renderer needs to catch up, we still request to render the
// next frame as soon as possible.
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
}
// When effects previewing is enabled, set display surface and an unknown size.
if (videoSinkProvider.isInitialized()) {

23
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java
View File

@ -175,6 +175,7 @@ public final class VideoFrameReleaseControl {
private long lastReleaseRealtimeUs;
private long lastPresentationTimeUs;
private long joiningDeadlineMs;
private boolean joiningRenderNextFrameImmediately;
private float playbackSpeed;
private Clock clock;
@ -298,8 +299,17 @@ public final class VideoFrameReleaseControl {
}
}
/** Joins the release control to a new stream. */
public void join() {
/**
* Joins the release control to a new stream.
*
* <p>The release control will pretend to be {@linkplain #isReady ready} for short time even if
* the first frame hasn't been rendered yet to avoid interrupting an ongoing playback.
*
* @param renderNextFrameImmediately Whether the next frame should be released as soon as possible
* or only at its preferred scheduled release time.
*/
public void join(boolean renderNextFrameImmediately) {
joiningRenderNextFrameImmediately = renderNextFrameImmediately;
joiningDeadlineMs =
allowedJoiningTimeMs > 0 ? (clock.elapsedRealtime() + allowedJoiningTimeMs) : C.TIME_UNSET;
}
@ -353,8 +363,9 @@ public final class VideoFrameReleaseControl {
frameReleaseInfo.releaseTimeNs =
frameReleaseHelper.adjustReleaseTime(systemTimeNs + (frameReleaseInfo.earlyUs * 1_000));
frameReleaseInfo.earlyUs = (frameReleaseInfo.releaseTimeNs - systemTimeNs) / 1_000;
// While joining, late frames are skipped.
boolean treatDropAsSkip = joiningDeadlineMs != C.TIME_UNSET;
// While joining, late frames are skipped while we catch up with the playback position.
boolean treatDropAsSkip =
joiningDeadlineMs != C.TIME_UNSET && !joiningRenderNextFrameImmediately;
if (frameTimingEvaluator.shouldIgnoreFrame(
frameReleaseInfo.earlyUs, positionUs, elapsedRealtimeUs, isLastFrame, treatDropAsSkip)) {
return FRAME_RELEASE_IGNORE;
@ -425,8 +436,8 @@ public final class VideoFrameReleaseControl {
/** Returns whether a frame should be force released. */
private boolean shouldForceRelease(
long positionUs, long earlyUs, long outputStreamStartPositionUs) {
if (joiningDeadlineMs != C.TIME_UNSET) {
// No force releasing during joining.
if (joiningDeadlineMs != C.TIME_UNSET && !joiningRenderNextFrameImmediately) {
// No force releasing of the initial or late frames during joining unless requested.
return false;
}
switch (firstFrameState) {

112
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java
View File

@ -44,25 +44,52 @@ public class VideoFrameReleaseControlTest {
}
@Test
public void isReady_withinJoiningDeadline_returnsTrue() {
public void isReady_withinJoiningDeadlineWhenRenderingNextFrameImmediately_returnsTrue() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.join();
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
}
@Test
public void isReady_joiningDeadlineExceeded_returnsFalse() {
public void isReady_withinJoiningDeadlineWhenNotRenderingNextFrameImmediately_returnsTrue() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.join();
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
}
@Test
public void isReady_joiningDeadlineExceededWhenRenderingNextFrameImmediately_returnsFalse() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
clock.advanceTime(/* timeDiffMs= */ 101);
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse();
}
@Test
public void isReady_joiningDeadlineExceededWhenNotRenderingNextFrameImmediately_returnsFalse() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
clock.advanceTime(/* timeDiffMs= */ 101);
@ -323,7 +350,9 @@ public class VideoFrameReleaseControlTest {
}
@Test
public void getFrameReleaseAction_dropWhileJoining_returnsSkip() throws ExoPlaybackException {
public void
getFrameReleaseAction_lateFrameWhileJoiningWhenNotRenderingFirstFrameImmediately_returnsSkip()
throws ExoPlaybackException {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
@ -337,26 +366,12 @@ public class VideoFrameReleaseControlTest {
/* allowedJoiningTimeMs= */ 1234);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
clock.advanceTime(/* timeDiffMs= */ 40);
// Start joining.
videoFrameReleaseControl.join();
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
// Second frame.
// First output is TRY_AGAIN_LATER because the time hasn't moved yet
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 5_000,
@ -365,9 +380,64 @@ public class VideoFrameReleaseControlTest {
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER);
// Late frame should be marked as skipped
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 5_000,
/* positionUs= */ 11_000,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP);
}
@Test
public void
getFrameReleaseAction_lateFrameWhileJoiningWhenRenderingFirstFrameImmediately_returnsDropAfterInitialImmediateRelease()
throws ExoPlaybackException {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
new VideoFrameReleaseControl(
ApplicationProvider.getApplicationContext(),
new TestFrameTimingEvaluator(
/* shouldForceRelease= */ false,
/* shouldDropFrame= */ true,
/* shouldIgnoreFrame= */ false),
/* allowedJoiningTimeMs= */ 1234);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// Start joining.
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
// First output is to force render the next frame.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 5_000,
/* positionUs= */ 10_000,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
// Further late frames should be marked as dropped.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 6_000,
/* positionUs= */ 11_000,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP);
}
@Test
public void getFrameReleaseAction_shouldIgnore() throws ExoPlaybackException {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =