Refine "join" mode in video renderer for surface changes.

The join mode is used for two cases: surface switching and mid-playback
enabling of video.

In both cases, we want to pretend to be ready despite not having rendered
a new "first frame". So far, we also avoided force-rendering the first
frame immediately because it causes a stuttering effect for the
mid-playback enable case. The surface switch case doesn't have this
stuttering issue as the same codec is used without interruption. Not
force-rendering the frame immediately causes the first-frame rendered
callback to arrive too early though, which may lead to cases where
apps hide shutter views too quickly.

This problem can be solved by only avoiding the force-render for the
mid-playback enabling case, but not for the surface switching case.

PiperOrigin-RevId: 622105916

RELEASENOTES.md

@@ -76,6 +76,8 @@
     *   Fix issue where HDR color info handling causes codec mishavior and
         prevents adaptive format switches for SDR video tracks
         ([#1158](https://github.com/androidx/media/issues/1158)).
+    *   Fix issue where `Listener.onRenderedFirstFrame()` arrives too early when
+        switching surfaces mid-playback.
 *   Text:
     *   WebVTT: Prevent directly consecutive cues from creating spurious
         additional `CuesWithTiming` instances from `WebvttParser.parse`

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java

@@ -660,7 +660,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
     }
     videoFrameReleaseControl.reset();
     if (joining) {
-      videoFrameReleaseControl.join();
+      // Don't render next frame immediately to let the codec catch up with the playback position
+      // first. This prevents a stuttering effect caused by showing the first frame and then
+      // dropping many of the subsequent frames during the catch up phase.
+      videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
     }
     maybeSetupTunnelingForFirstFrame();
     consecutiveDroppedFrameCount = 0;
@@ -835,7 +838,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
         // If we know the video size, report it again immediately.
         maybeRenotifyVideoSizeChanged();
         if (state == STATE_STARTED) {
-          videoFrameReleaseControl.join();
+          // We want to "join" playback to prevent an intermediate buffering state in the player
+          // before we rendered the new first frame. Since there is no reason to believe the next
+          // frame is delayed and the renderer needs to catch up, we still request to render the
+          // next frame as soon as possible.
+          videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
         }
         // When effects previewing is enabled, set display surface and an unknown size.
         if (videoSinkProvider.isInitialized()) {

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java

@@ -175,6 +175,7 @@ public final class VideoFrameReleaseControl {
   private long lastReleaseRealtimeUs;
   private long lastPresentationTimeUs;
   private long joiningDeadlineMs;
+  private boolean joiningRenderNextFrameImmediately;
   private float playbackSpeed;
   private Clock clock;
 
@@ -298,8 +299,17 @@ public final class VideoFrameReleaseControl {
     }
   }
 
-  /** Joins the release control to a new stream. */
+  /**
-  public void join() {
+   * Joins the release control to a new stream.
+   *
+   * <p>The release control will pretend to be {@linkplain #isReady ready} for short time even if
+   * the first frame hasn't been rendered yet to avoid interrupting an ongoing playback.
+   *
+   * @param renderNextFrameImmediately Whether the next frame should be released as soon as possible
+   *     or only at its preferred scheduled release time.
+   */
+  public void join(boolean renderNextFrameImmediately) {
+    joiningRenderNextFrameImmediately = renderNextFrameImmediately;
     joiningDeadlineMs =
         allowedJoiningTimeMs > 0 ? (clock.elapsedRealtime() + allowedJoiningTimeMs) : C.TIME_UNSET;
   }
@@ -353,8 +363,9 @@ public final class VideoFrameReleaseControl {
     frameReleaseInfo.releaseTimeNs =
         frameReleaseHelper.adjustReleaseTime(systemTimeNs + (frameReleaseInfo.earlyUs * 1_000));
     frameReleaseInfo.earlyUs = (frameReleaseInfo.releaseTimeNs - systemTimeNs) / 1_000;
-    // While joining, late frames are skipped.
+    // While joining, late frames are skipped while we catch up with the playback position.
-    boolean treatDropAsSkip = joiningDeadlineMs != C.TIME_UNSET;
+    boolean treatDropAsSkip =
+        joiningDeadlineMs != C.TIME_UNSET && !joiningRenderNextFrameImmediately;
     if (frameTimingEvaluator.shouldIgnoreFrame(
         frameReleaseInfo.earlyUs, positionUs, elapsedRealtimeUs, isLastFrame, treatDropAsSkip)) {
       return FRAME_RELEASE_IGNORE;
@@ -425,8 +436,8 @@ public final class VideoFrameReleaseControl {
   /** Returns whether a frame should be force released. */
   private boolean shouldForceRelease(
       long positionUs, long earlyUs, long outputStreamStartPositionUs) {
-    if (joiningDeadlineMs != C.TIME_UNSET) {
+    if (joiningDeadlineMs != C.TIME_UNSET && !joiningRenderNextFrameImmediately) {
-      // No force releasing during joining.
+      // No force releasing of the initial or late frames during joining unless requested.
       return false;
     }
     switch (firstFrameState) {

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java

@@ -44,25 +44,52 @@ public class VideoFrameReleaseControlTest {
   }
 
   @Test
-  public void isReady_withinJoiningDeadline_returnsTrue() {
+  public void isReady_withinJoiningDeadlineWhenRenderingNextFrameImmediately_returnsTrue() {
     FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
     VideoFrameReleaseControl videoFrameReleaseControl =
         createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
     videoFrameReleaseControl.setClock(clock);
 
-    videoFrameReleaseControl.join();
+    videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
 
     assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
   }
 
   @Test
-  public void isReady_joiningDeadlineExceeded_returnsFalse() {
+  public void isReady_withinJoiningDeadlineWhenNotRenderingNextFrameImmediately_returnsTrue() {
     FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
     VideoFrameReleaseControl videoFrameReleaseControl =
         createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
     videoFrameReleaseControl.setClock(clock);
 
-    videoFrameReleaseControl.join();
+    videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
+
+    assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
+  }
+
+  @Test
+  public void isReady_joiningDeadlineExceededWhenRenderingNextFrameImmediately_returnsFalse() {
+    FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
+    VideoFrameReleaseControl videoFrameReleaseControl =
+        createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
+    videoFrameReleaseControl.setClock(clock);
+
+    videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
+    assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
+
+    clock.advanceTime(/* timeDiffMs= */ 101);
+
+    assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse();
+  }
+
+  @Test
+  public void isReady_joiningDeadlineExceededWhenNotRenderingNextFrameImmediately_returnsFalse() {
+    FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
+    VideoFrameReleaseControl videoFrameReleaseControl =
+        createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
+    videoFrameReleaseControl.setClock(clock);
+
+    videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
     assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
 
     clock.advanceTime(/* timeDiffMs= */ 101);
@@ -323,7 +350,9 @@ public class VideoFrameReleaseControlTest {
   }
 
   @Test
-  public void getFrameReleaseAction_dropWhileJoining_returnsSkip() throws ExoPlaybackException {
+  public void
+      getFrameReleaseAction_lateFrameWhileJoiningWhenNotRenderingFirstFrameImmediately_returnsSkip()
+          throws ExoPlaybackException {
     VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
         new VideoFrameReleaseControl.FrameReleaseInfo();
     FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
@@ -337,26 +366,12 @@ public class VideoFrameReleaseControlTest {
             /* allowedJoiningTimeMs= */ 1234);
     videoFrameReleaseControl.setClock(clock);
     videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
-
     videoFrameReleaseControl.onStarted();
 
-    // First frame released.
-    assertThat(
-            videoFrameReleaseControl.getFrameReleaseAction(
-                /* presentationTimeUs= */ 0,
-                /* positionUs= */ 0,
-                /* elapsedRealtimeUs= */ 0,
-                /* outputStreamStartPositionUs= */ 0,
-                /* isLastFrame= */ false,
-                frameReleaseInfo))
-        .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
-    videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
-    clock.advanceTime(/* timeDiffMs= */ 40);
-
     // Start joining.
-    videoFrameReleaseControl.join();
+    videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
 
-    // Second frame.
+    // First output is TRY_AGAIN_LATER because the time hasn't moved yet
     assertThat(
             videoFrameReleaseControl.getFrameReleaseAction(
                 /* presentationTimeUs= */ 5_000,
@@ -365,9 +380,64 @@ public class VideoFrameReleaseControlTest {
                 /* outputStreamStartPositionUs= */ 0,
                 /* isLastFrame= */ false,
                 frameReleaseInfo))
+        .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER);
+    // Late frame should be marked as skipped
+    assertThat(
+            videoFrameReleaseControl.getFrameReleaseAction(
+                /* presentationTimeUs= */ 5_000,
+                /* positionUs= */ 11_000,
+                /* elapsedRealtimeUs= */ 0,
+                /* outputStreamStartPositionUs= */ 0,
+                /* isLastFrame= */ false,
+                frameReleaseInfo))
         .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP);
   }
 
+  @Test
+  public void
+      getFrameReleaseAction_lateFrameWhileJoiningWhenRenderingFirstFrameImmediately_returnsDropAfterInitialImmediateRelease()
+          throws ExoPlaybackException {
+    VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
+        new VideoFrameReleaseControl.FrameReleaseInfo();
+    FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
+    VideoFrameReleaseControl videoFrameReleaseControl =
+        new VideoFrameReleaseControl(
+            ApplicationProvider.getApplicationContext(),
+            new TestFrameTimingEvaluator(
+                /* shouldForceRelease= */ false,
+                /* shouldDropFrame= */ true,
+                /* shouldIgnoreFrame= */ false),
+            /* allowedJoiningTimeMs= */ 1234);
+    videoFrameReleaseControl.setClock(clock);
+    videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
+    videoFrameReleaseControl.onStarted();
+
+    // Start joining.
+    videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
+
+    // First output is to force render the next frame.
+    assertThat(
+            videoFrameReleaseControl.getFrameReleaseAction(
+                /* presentationTimeUs= */ 5_000,
+                /* positionUs= */ 10_000,
+                /* elapsedRealtimeUs= */ 0,
+                /* outputStreamStartPositionUs= */ 0,
+                /* isLastFrame= */ false,
+                frameReleaseInfo))
+        .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
+    videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
+    // Further late frames should be marked as dropped.
+    assertThat(
+            videoFrameReleaseControl.getFrameReleaseAction(
+                /* presentationTimeUs= */ 6_000,
+                /* positionUs= */ 11_000,
+                /* elapsedRealtimeUs= */ 0,
+                /* outputStreamStartPositionUs= */ 0,
+                /* isLastFrame= */ false,
+                frameReleaseInfo))
+        .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP);
+  }
+
   @Test
   public void getFrameReleaseAction_shouldIgnore() throws ExoPlaybackException {
     VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =