Implement transfer of resources API for renderer pre-warming

At the point of playing period transition pre-warming has completed and the renderers should receive necessary resources for playback. This CL adds the `Renderer.MessageType` `MSG_TRANSFER_RESOURCES` to direct a renderer to transfer relevant resources to another renderer. PiperOrigin-RevId: 713372754
2025-04-30 06:46:50 +08:00 · 2025-01-08 12:10:29 -08:00 · 2025-01-08 12:10:29 -08:00 · 314413365b
commit 314413365b
parent 7d4ddfbb91
10 changed files with 233 additions and 83 deletions
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java
@ -1655,7 +1655,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
    rendererPositionUs = MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US;
    try {
      disableRenderers();
-    } catch (RuntimeException e) {
+    } catch (RuntimeException | ExoPlaybackException e) {
      // There's nothing we can do.
      Log.e(TAG, "Disable failed.", e);
    }
@ -1919,14 +1919,14 @@ import java.util.concurrent.atomic.AtomicBoolean;
    nextPendingMessageIndexHint = nextPendingMessageIndex;
  }

-  private void disableRenderers() {
+  private void disableRenderers() throws ExoPlaybackException {
    for (int i = 0; i < renderers.length; i++) {
      disableRenderer(/* rendererIndex= */ i);
    }
    prewarmingMediaPeriodDiscontinuity = C.TIME_UNSET;
  }

-  private void disableRenderer(int rendererIndex) {
+  private void disableRenderer(int rendererIndex) throws ExoPlaybackException {
    int enabledRendererCountBeforeDisabling = renderers[rendererIndex].getEnabledRendererCount();
    renderers[rendererIndex].disable(mediaClock);
    maybeTriggerOnRendererReadyChanged(rendererIndex, /* allowsPlayback= */ false);
@ -2639,7 +2639,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
    }
  }

-  private void maybeHandlePrewarmingTransition() {
+  private void maybeHandlePrewarmingTransition() throws ExoPlaybackException {
    for (RendererHolder renderer : renderers) {
      renderer.maybeHandlePrewarmingTransition();
    }
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java
@ -209,7 +209,8 @@ public interface Renderer extends PlayerMessage.Target {
        MSG_SET_VIDEO_EFFECTS,
        MSG_SET_VIDEO_OUTPUT_RESOLUTION,
        MSG_SET_IMAGE_OUTPUT,
-        MSG_SET_PRIORITY
+        MSG_SET_PRIORITY,
+        MSG_TRANSFER_RESOURCES
      })
  public @interface MessageType {}

@ -347,6 +348,13 @@ public interface Renderer extends PlayerMessage.Target {
   */
  int MSG_SET_PRIORITY = 16;

+  /**
+   * The type of message that can be passed to a renderer to direct it to transfer relevant
+   * resources to another renderer. The message payload should be a instance of the same {@link
+   * Renderer} type as the renderer being passed the message.
+   */
+  int MSG_TRANSFER_RESOURCES = 17;
+
  /**
   * Applications or extensions may define custom {@code MSG_*} constants that can be passed to
   * renderers. These custom constants must be greater than or equal to this value.
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/RendererHolder.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/RendererHolder.java
@ -18,6 +18,7 @@ package androidx.media3.exoplayer;
 import static androidx.media3.common.C.TRACK_TYPE_VIDEO;
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.exoplayer.Renderer.MSG_TRANSFER_RESOURCES;
 import static androidx.media3.exoplayer.Renderer.STATE_DISABLED;
 import static androidx.media3.exoplayer.Renderer.STATE_ENABLED;
 import static androidx.media3.exoplayer.Renderer.STATE_STARTED;
@ -533,22 +534,42 @@ import java.util.Objects;
   * @param mediaClock To call {@link DefaultMediaClock#onRendererDisabled} if disabling a {@link
   *     Renderer}.
   */
-  public void disable(DefaultMediaClock mediaClock) {
+  public void disable(DefaultMediaClock mediaClock) throws ExoPlaybackException {
    disableRenderer(primaryRenderer, mediaClock);
    if (secondaryRenderer != null) {
+      boolean shouldTransferResources =
+          isRendererEnabled(secondaryRenderer)
+              && prewarmingState != RENDERER_PREWARMING_STATE_TRANSITIONING_TO_SECONDARY;
      disableRenderer(secondaryRenderer, mediaClock);
-      // Release resources for other renderer
      maybeResetRenderer(/* resetPrimary= */ false);
+      if (shouldTransferResources) {
+        transferResources(/* transferToPrimary= */ true);
+      }
    }
    prewarmingState = RENDERER_PREWARMING_STATE_NOT_PREWARMING_USING_PRIMARY;
  }

-  public void maybeHandlePrewarmingTransition() {
-    if (prewarmingState == RENDERER_PREWARMING_STATE_TRANSITIONING_TO_PRIMARY
-        || prewarmingState == RENDERER_PREWARMING_STATE_PREWARMING_PRIMARY) {
+  /** Handles transition of pre-warming state and resources. */
+  public void maybeHandlePrewarmingTransition() throws ExoPlaybackException {
+    if (prewarmingState == RENDERER_PREWARMING_STATE_TRANSITIONING_TO_SECONDARY
+        || prewarmingState == RENDERER_PREWARMING_STATE_TRANSITIONING_TO_PRIMARY) {
+      transferResources(
+          /* transferToPrimary= */ prewarmingState
+              == RENDERER_PREWARMING_STATE_TRANSITIONING_TO_PRIMARY);
+      prewarmingState =
+          prewarmingState == RENDERER_PREWARMING_STATE_TRANSITIONING_TO_PRIMARY
+              ? RENDERER_PREWARMING_STATE_NOT_PREWARMING_USING_PRIMARY
+              : RENDERER_PREWARMING_STATE_NOT_PREWARMING_USING_SECONDARY;
+    } else if (prewarmingState == RENDERER_PREWARMING_STATE_PREWARMING_PRIMARY) {
      prewarmingState = RENDERER_PREWARMING_STATE_NOT_PREWARMING_USING_PRIMARY;
-    } else if (prewarmingState == RENDERER_PREWARMING_STATE_TRANSITIONING_TO_SECONDARY) {
-      prewarmingState = RENDERER_PREWARMING_STATE_NOT_PREWARMING_USING_SECONDARY;
+    }
+  }
+
+  private void transferResources(boolean transferToPrimary) throws ExoPlaybackException {
+    if (transferToPrimary) {
+      checkNotNull(secondaryRenderer).handleMessage(MSG_TRANSFER_RESOURCES, primaryRenderer);
+    } else {
+      primaryRenderer.handleMessage(MSG_TRANSFER_RESOURCES, checkNotNull(secondaryRenderer));
    }
  }

--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
@ -973,6 +973,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
        rendererPriority = (int) checkNotNull(message);
        updateCodecImportance();
        break;
+      case MSG_TRANSFER_RESOURCES:
+        {
+          Surface surface = this.displaySurface;
+          setOutput(null);
+          ((MediaCodecVideoRenderer) checkNotNull(message))
+              .handleMessage(MSG_SET_VIDEO_OUTPUT, surface);
+        }
+        break;
      default:
        super.handleMessage(messageType, message);
    }
@ -1509,7 +1517,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
    // We are not rendering on a surface, the renderer will wait until a surface is set.
    if (displaySurface == null) {
      // Skip frames in sync with playback, so we'll be at the right frame if the mode changes.
-      if (videoFrameReleaseInfo.getEarlyUs() < 0
+      if ((videoFrameReleaseInfo.getEarlyUs() < 0
+              && shouldSkipLateBuffersWhileUsingPlaceholderSurface())
          || (videoFrameReleaseInfo.getEarlyUs() < 30_000
              && frameReleaseAction != VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER)) {
        skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
@ -1662,6 +1671,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
    return true;
  }

+  /** Returns whether to skip late buffers while using a placeholder surface. */
+  protected boolean shouldSkipLateBuffersWhileUsingPlaceholderSurface() {
+    return true;
+  }
+
  /**
   * Returns whether to force rendering an output buffer.
   *
--- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/PrewarmingRendererPlaybackTest.java
+++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/PrewarmingRendererPlaybackTest.java
@ -15,6 +15,8 @@
 */
 package androidx.media3.exoplayer.e2etest;

+import static androidx.media3.test.utils.robolectric.TestPlayerRunHelper.run;
+
 import android.content.Context;
 import android.graphics.SurfaceTexture;
 import android.os.Handler;
@ -129,6 +131,61 @@ public class PrewarmingRendererPlaybackTest {
            + ".dump");
  }

+  @Test
+  public void playback_withStopDuringPlaybackWithSecondaryVideoRenderer_dumpsCorrectOutput()
+      throws Exception {
+    Context applicationContext = ApplicationProvider.getApplicationContext();
+    CapturingRenderersWithSecondaryVideoRendererFactory capturingRenderersFactory =
+        new CapturingRenderersWithSecondaryVideoRendererFactory(applicationContext);
+    ExoPlayer player =
+        new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
+            .setClock(new FakeClock(/* isAutoAdvancing= */ true))
+            .build();
+    Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1));
+    player.setVideoSurface(surface);
+    PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory);
+    // Create media item containing a single sample.
+    MediaItem mediaItem =
+        new MediaItem.Builder()
+            .setUri(TEST_MP4_URI)
+            .setClippingConfiguration(
+                new MediaItem.ClippingConfiguration.Builder()
+                    .setStartPositionMs(0)
+                    .setEndPositionMs(25)
+                    .build())
+            .build();
+    player.addMediaItems(ImmutableList.of(mediaItem, mediaItem));
+    // Disable audio renderer for simpler dump file.
+    player.setTrackSelectionParameters(
+        player
+            .getTrackSelectionParameters()
+            .buildUpon()
+            .setTrackTypeDisabled(C.TRACK_TYPE_AUDIO, true)
+            .build());
+    player.prepare();
+    player.play();
+
+    run(player).untilStartOfMediaItem(1);
+
+    // Stop and reset player to simulate stop, reset, and transition back to using primary.
+    player.stop();
+    player.seekTo(/* mediaItemIndex= */ 1, /* positionMs= */ 0);
+    player.prepare();
+    player.play();
+
+    TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
+
+    player.release();
+    surface.release();
+
+    DumpFileAsserts.assertOutput(
+        applicationContext,
+        playbackOutput,
+        "playbackdumps/prewarmingRenderer/"
+            + "twoItemPlaylist-clippedWithStopDuringPlaybackWithSecondaryVideoRenderer"
+            + ".dump");
+  }
+
  @Test
  public void playback_withMultipleMediaItemsWithClippingConfigurations_dumpsCorrectOutput()
      throws Exception {
--- a/libraries/test_data/src/test/assets/playbackdumps/prewarmingRenderer/threeItemPlaylist-withClippingConfigurations.dump
+++ b/libraries/test_data/src/test/assets/playbackdumps/prewarmingRenderer/threeItemPlaylist-withClippingConfigurations.dump
@ -482,11 +482,11 @@ MediaCodecAdapter (exotest.video.avc):
    output buffer #7:
      timeUs = 1000000333666
      size = 6061
-      rendered = false
+      rendered = true
    output buffer #8:
      timeUs = 1000000266933
      size = 992
-      rendered = false
+      rendered = true
    output buffer #9:
      timeUs = 1000000233566
      size = 623
@ -494,35 +494,35 @@ MediaCodecAdapter (exotest.video.avc):
    output buffer #10:
      timeUs = 1000000300300
      size = 421
-      rendered = false
+      rendered = true
    output buffer #11:
      timeUs = 1000000433766
      size = 4899
-      rendered = false
+      rendered = true
    output buffer #12:
      timeUs = 1000000400400
      size = 568
-      rendered = false
+      rendered = true
    output buffer #13:
      timeUs = 1000000367033
      size = 620
-      rendered = false
+      rendered = true
    output buffer #14:
      timeUs = 1000000567233
      size = 5450
-      rendered = false
+      rendered = true
    output buffer #15:
      timeUs = 1000000500500
      size = 1051
-      rendered = false
+      rendered = true
    output buffer #16:
      timeUs = 1000000467133
      size = 874
-      rendered = false
+      rendered = true
    output buffer #17:
      timeUs = 1000000533866
      size = 781
-      rendered = false
+      rendered = true
 MediaCodecAdapter (exotest.video.avc):
  inputBuffers:
    count = 31
--- a/libraries/test_data/src/test/assets/playbackdumps/prewarmingRenderer/threeItemPlaylist-withSecondaryVideoRenderer.dump
+++ b/libraries/test_data/src/test/assets/playbackdumps/prewarmingRenderer/threeItemPlaylist-withSecondaryVideoRenderer.dump
@ -319,123 +319,123 @@ MediaCodecAdapter (exotest.video.avc):
    output buffer #0:
      timeUs = 1000001024000
      size = 36692
-      rendered = false
+      rendered = true
    output buffer #1:
      timeUs = 1000001090733
      size = 5312
-      rendered = false
+      rendered = true
    output buffer #2:
      timeUs = 1000001057366
      size = 599
-      rendered = false
+      rendered = true
    output buffer #3:
      timeUs = 1000001224200
      size = 7735
-      rendered = false
+      rendered = true
    output buffer #4:
      timeUs = 1000001157466
      size = 987
-      rendered = false
+      rendered = true
    output buffer #5:
      timeUs = 1000001124100
      size = 673
-      rendered = false
+      rendered = true
    output buffer #6:
      timeUs = 1000001190833
      size = 523
-      rendered = false
+      rendered = true
    output buffer #7:
      timeUs = 1000001357666
      size = 6061
-      rendered = false
+      rendered = true
    output buffer #8:
      timeUs = 1000001290933
      size = 992
-      rendered = false
+      rendered = true
    output buffer #9:
      timeUs = 1000001257566
      size = 623
-      rendered = false
+      rendered = true
    output buffer #10:
      timeUs = 1000001324300
      size = 421
-      rendered = false
+      rendered = true
    output buffer #11:
      timeUs = 1000001457766
      size = 4899
-      rendered = false
+      rendered = true
    output buffer #12:
      timeUs = 1000001424400
      size = 568
-      rendered = false
+      rendered = true
    output buffer #13:
      timeUs = 1000001391033
      size = 620
-      rendered = false
+      rendered = true
    output buffer #14:
      timeUs = 1000001591233
      size = 5450
-      rendered = false
+      rendered = true
    output buffer #15:
      timeUs = 1000001524500
      size = 1051
-      rendered = false
+      rendered = true
    output buffer #16:
      timeUs = 1000001491133
      size = 874
-      rendered = false
+      rendered = true
    output buffer #17:
      timeUs = 1000001557866
      size = 781
-      rendered = false
+      rendered = true
    output buffer #18:
      timeUs = 1000001724700
      size = 4725
-      rendered = false
+      rendered = true
    output buffer #19:
      timeUs = 1000001657966
      size = 1022
-      rendered = false
+      rendered = true
    output buffer #20:
      timeUs = 1000001624600
      size = 790
-      rendered = false
+      rendered = true
    output buffer #21:
      timeUs = 1000001691333
      size = 610
-      rendered = false
+      rendered = true
    output buffer #22:
      timeUs = 1000001858166
      size = 2751
-      rendered = false
+      rendered = true
    output buffer #23:
      timeUs = 1000001791433
      size = 745
-      rendered = false
+      rendered = true
    output buffer #24:
      timeUs = 1000001758066
      size = 621
-      rendered = false
+      rendered = true
    output buffer #25:
      timeUs = 1000001824800
      size = 505
-      rendered = false
+      rendered = true
    output buffer #26:
      timeUs = 1000001991633
      size = 1268
-      rendered = false
+      rendered = true
    output buffer #27:
      timeUs = 1000001924900
      size = 880
-      rendered = false
+      rendered = true
    output buffer #28:
      timeUs = 1000001891533
      size = 530
-      rendered = false
+      rendered = true
    output buffer #29:
      timeUs = 1000001958266
      size = 568
-      rendered = false
+      rendered = true
 MediaCodecAdapter (exotest.video.avc):
  inputBuffers:
    count = 31
--- a/libraries/test_data/src/test/assets/playbackdumps/prewarmingRenderer/twoItemPlaylist-clippedWithStopDuringPlaybackWithSecondaryVideoRenderer.dump
+++ b/libraries/test_data/src/test/assets/playbackdumps/prewarmingRenderer/twoItemPlaylist-clippedWithStopDuringPlaybackWithSecondaryVideoRenderer.dump
@ -0,0 +1,44 @@
+MediaCodecAdapter (exotest.video.avc):
+  inputBuffers:
+    count = 2
+    input buffer #0:
+      timeUs = 1000000000000
+      contents = length 36692, hash D216076E
+    input buffer #1:
+      timeUs = 0
+      flags = 4
+      contents = length 0, hash 1
+  outputBuffers:
+    count = 1
+    output buffer #0:
+      timeUs = 1000000000000
+      size = 36692
+      rendered = true
+MediaCodecAdapter (exotest.video.avc):
+  inputBuffers:
+    count = 1
+    input buffer #0:
+      timeUs = 1000000025000
+      contents = length 36692, hash D216076E
+  outputBuffers:
+    count = 1
+    output buffer #0:
+      timeUs = 1000000025000
+      size = 36692
+      rendered = true
+MediaCodecAdapter (exotest.video.avc):
+  inputBuffers:
+    count = 2
+    input buffer #0:
+      timeUs = 1000000000000
+      contents = length 36692, hash D216076E
+    input buffer #1:
+      timeUs = 0
+      flags = 4
+      contents = length 0, hash 1
+  outputBuffers:
+    count = 1
+    output buffer #0:
+      timeUs = 1000000000000
+      size = 36692
+      rendered = true
--- a/libraries/test_data/src/test/assets/playbackdumps/prewarmingRenderer/twoItemsPlaylist-withSecondaryVideoRenderer.dump
+++ b/libraries/test_data/src/test/assets/playbackdumps/prewarmingRenderer/twoItemsPlaylist-withSecondaryVideoRenderer.dump
@ -962,123 +962,123 @@ MediaCodecAdapter (exotest.video.avc):
    output buffer #0:
      timeUs = 1000001024000
      size = 36692
-      rendered = false
+      rendered = true
    output buffer #1:
      timeUs = 1000001090733
      size = 5312
-      rendered = false
+      rendered = true
    output buffer #2:
      timeUs = 1000001057366
      size = 599
-      rendered = false
+      rendered = true
    output buffer #3:
      timeUs = 1000001224200
      size = 7735
-      rendered = false
+      rendered = true
    output buffer #4:
      timeUs = 1000001157466
      size = 987
-      rendered = false
+      rendered = true
    output buffer #5:
      timeUs = 1000001124100
      size = 673
-      rendered = false
+      rendered = true
    output buffer #6:
      timeUs = 1000001190833
      size = 523
-      rendered = false
+      rendered = true
    output buffer #7:
      timeUs = 1000001357666
      size = 6061
-      rendered = false
+      rendered = true
    output buffer #8:
      timeUs = 1000001290933
      size = 992
-      rendered = false
+      rendered = true
    output buffer #9:
      timeUs = 1000001257566
      size = 623
-      rendered = false
+      rendered = true
    output buffer #10:
      timeUs = 1000001324300
      size = 421
-      rendered = false
+      rendered = true
    output buffer #11:
      timeUs = 1000001457766
      size = 4899
-      rendered = false
+      rendered = true
    output buffer #12:
      timeUs = 1000001424400
      size = 568
-      rendered = false
+      rendered = true
    output buffer #13:
      timeUs = 1000001391033
      size = 620
-      rendered = false
+      rendered = true
    output buffer #14:
      timeUs = 1000001591233
      size = 5450
-      rendered = false
+      rendered = true
    output buffer #15:
      timeUs = 1000001524500
      size = 1051
-      rendered = false
+      rendered = true
    output buffer #16:
      timeUs = 1000001491133
      size = 874
-      rendered = false
+      rendered = true
    output buffer #17:
      timeUs = 1000001557866
      size = 781
-      rendered = false
+      rendered = true
    output buffer #18:
      timeUs = 1000001724700
      size = 4725
-      rendered = false
+      rendered = true
    output buffer #19:
      timeUs = 1000001657966
      size = 1022
-      rendered = false
+      rendered = true
    output buffer #20:
      timeUs = 1000001624600
      size = 790
-      rendered = false
+      rendered = true
    output buffer #21:
      timeUs = 1000001691333
      size = 610
-      rendered = false
+      rendered = true
    output buffer #22:
      timeUs = 1000001858166
      size = 2751
-      rendered = false
+      rendered = true
    output buffer #23:
      timeUs = 1000001791433
      size = 745
-      rendered = false
+      rendered = true
    output buffer #24:
      timeUs = 1000001758066
      size = 621
-      rendered = false
+      rendered = true
    output buffer #25:
      timeUs = 1000001824800
      size = 505
-      rendered = false
+      rendered = true
    output buffer #26:
      timeUs = 1000001991633
      size = 1268
-      rendered = false
+      rendered = true
    output buffer #27:
      timeUs = 1000001924900
      size = 880
-      rendered = false
+      rendered = true
    output buffer #28:
      timeUs = 1000001891533
      size = 530
-      rendered = false
+      rendered = true
    output buffer #29:
      timeUs = 1000001958266
      size = 568
-      rendered = false
+      rendered = true
 AudioSink:
  buffer count = 90
  config:
--- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
+++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
@ -234,6 +234,12 @@ public class CapturingRenderersFactory implements RenderersFactory, Dumper.Dumpa
      // Do not skip buffers with identical vsync times as we can't control this from tests.
      return false;
    }
+
+    @Override
+    protected boolean shouldSkipLateBuffersWhileUsingPlaceholderSurface() {
+      // Do not skip buffers while using placeholder surface due to slow processing.
+      return false;
+    }
  }

  /**