Allow replays with CompositionPlayer

PiperOrigin-RevId: 745536073

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java

@@ -366,7 +366,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
 
   @Override
   public VideoSink getSink(int inputIndex) {
-    checkState(!contains(inputVideoSinks, inputIndex));
+    if (contains(inputVideoSinks, inputIndex)) {
+      return inputVideoSinks.get(inputIndex);
+    }
     InputVideoSink inputVideoSink = new InputVideoSink(context, inputIndex);
     if (inputIndex == PRIMARY_SEQUENCE_INDEX) {
       addListener(inputVideoSink);
@@ -730,7 +732,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
 
     @Override
     public void redraw() {
-      checkState(isInitialized());
+      if (!isInitialized()) {
+        return;
+      }
       // Resignal EOS only for the last item.
       boolean needsResignalEndOfCurrentInputStream = signaledEndOfStream;
       long replayedPresentationTimeUs = lastOutputBufferPresentationTimeUs;

libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java

@@ -481,6 +481,7 @@ public final class AndroidTestUtil {
                   .setFrameRate(30.00f)
                   .setCodecs("avc1.42C033")
                   .build())
+          .setVideoDurationUs(1_000_000L)
           .build();
 
   public static final AssetInfo MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS =

libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java

@@ -32,9 +32,11 @@ import androidx.annotation.Nullable;
 import androidx.media3.common.Format;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.PlaybackException;
+import androidx.media3.common.Player;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.util.Util;
 import androidx.media3.effect.Contrast;
+import androidx.media3.effect.GlEffect;
 import androidx.media3.exoplayer.DefaultRenderersFactory;
 import androidx.media3.exoplayer.ExoPlayer;
 import androidx.media3.exoplayer.Renderer;
@@ -43,6 +45,12 @@ import androidx.media3.exoplayer.util.EventLogger;
 import androidx.media3.exoplayer.video.VideoFrameMetadataListener;
 import androidx.media3.exoplayer.video.VideoRendererEventListener;
 import androidx.media3.transformer.AndroidTestUtil.ReplayVideoRenderer;
+import androidx.media3.transformer.Composition;
+import androidx.media3.transformer.CompositionPlayer;
+import androidx.media3.transformer.EditedMediaItem;
+import androidx.media3.transformer.EditedMediaItemSequence;
+import androidx.media3.transformer.Effects;
+import androidx.media3.transformer.InputTimestampRecordingShaderProgram;
 import androidx.media3.transformer.PlayerTestListener;
 import androidx.media3.transformer.SurfaceTestActivity;
 import androidx.test.ext.junit.rules.ActivityScenarioRule;
@@ -64,8 +72,11 @@ public class ReplayCacheTest {
   private static final long TEST_TIMEOUT_MS = isRunningOnEmulator() ? 20_000 : 10_000;
 
   private static final MediaItem VIDEO_MEDIA_ITEM_1 = MediaItem.fromUri(MP4_ASSET.uri);
+  private static final long VIDEO_MEDIA_ITEM_1_DURATION_US = MP4_ASSET.videoDurationUs;
   private static final MediaItem VIDEO_MEDIA_ITEM_2 =
       MediaItem.fromUri(MP4_ASSET_WITH_INCREASING_TIMESTAMPS.uri);
+  private static final long VIDEO_MEDIA_ITEM_2_DURATION_US =
+      MP4_ASSET_WITH_INCREASING_TIMESTAMPS.videoDurationUs;
 
   @Rule
   public ActivityScenarioRule<SurfaceTestActivity> rule =
@@ -74,6 +85,7 @@ public class ReplayCacheTest {
   private final Context context = getInstrumentation().getContext().getApplicationContext();
   private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
 
+  private CompositionPlayer compositionPlayer;
   private ExoPlayer exoPlayer;
   private SurfaceView surfaceView;
 
@@ -88,6 +100,9 @@ public class ReplayCacheTest {
     getInstrumentation()
         .runOnMainSync(
             () -> {
+              if (compositionPlayer != null) {
+                compositionPlayer.release();
+              }
               if (exoPlayer != null) {
                 exoPlayer.release();
               }
@@ -159,4 +174,108 @@ public class ReplayCacheTest {
     // we don't currently replay (the frame at media item transition).
     assertThat(playedFrameTimestampsUs).hasSize(119);
   }
+
+  @Test
+  public void enableReplay_withCompositionPlayerSingleSequence_playsSequence() throws Exception {
+    assumeTrue(
+        "The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite"
+            + " using MediaFormat.KEY_ALLOW_FRAME_DROP.",
+        !Util.isRunningOnEmulator());
+    PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS * 1000);
+    InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram =
+        new InputTimestampRecordingShaderProgram();
+
+    instrumentation.runOnMainSync(
+        () -> {
+          compositionPlayer =
+              new CompositionPlayer.Builder(context)
+                  .experimentalSetEnableReplayableCache(true)
+                  .build();
+          compositionPlayer.setVideoSurfaceView(surfaceView);
+          compositionPlayer.addListener(playerTestListener);
+          compositionPlayer.addListener(
+              new Player.Listener() {
+                @Override
+                public void onRenderedFirstFrame() {
+                  compositionPlayer.experimentalRedrawLastFrame();
+                  compositionPlayer.play();
+                }
+              });
+          compositionPlayer.setComposition(
+              new Composition.Builder(
+                      new EditedMediaItemSequence.Builder(
+                              new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_1)
+                                  .setDurationUs(VIDEO_MEDIA_ITEM_1_DURATION_US)
+                                  .build(),
+                              new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_2)
+                                  .setDurationUs(VIDEO_MEDIA_ITEM_2_DURATION_US)
+                                  .build())
+                          .build())
+                  .setEffects(
+                      new Effects(
+                          /* audioProcessors= */ ImmutableList.of(),
+                          /* videoEffects= */ ImmutableList.of(
+                              new Contrast(0.5f),
+                              (GlEffect)
+                                  (context, useHdr) -> inputTimestampRecordingShaderProgram)))
+                  .build());
+          compositionPlayer.prepare();
+        });
+
+    playerTestListener.waitUntilPlayerEnded();
+
+    int countOfFirstFrameRendered = 0;
+    for (long timestampUs : inputTimestampRecordingShaderProgram.getInputTimestampsUs()) {
+      if (timestampUs == 0) {
+        countOfFirstFrameRendered++;
+      }
+    }
+    assertThat(countOfFirstFrameRendered).isEqualTo(2);
+  }
+
+  @Test
+  public void rapidReplay_withCompositionPlayerSingleSequence_playsSequence() throws Exception {
+    assumeTrue(
+        "The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite"
+            + " using MediaFormat.KEY_ALLOW_FRAME_DROP.",
+        !Util.isRunningOnEmulator());
+    PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS);
+    Handler mainHandler = new Handler(instrumentation.getTargetContext().getMainLooper());
+
+    instrumentation.runOnMainSync(
+        () -> {
+          compositionPlayer =
+              new CompositionPlayer.Builder(context)
+                  .experimentalSetEnableReplayableCache(true)
+                  .build();
+          compositionPlayer.setVideoSurfaceView(surfaceView);
+          compositionPlayer.addListener(playerTestListener);
+          compositionPlayer.setComposition(
+              new Composition.Builder(
+                      new EditedMediaItemSequence.Builder(
+                              new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_1)
+                                  .setDurationUs(VIDEO_MEDIA_ITEM_1_DURATION_US)
+                                  .build(),
+                              new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_2)
+                                  .setDurationUs(VIDEO_MEDIA_ITEM_2_DURATION_US)
+                                  .build())
+                          .build())
+                  .setEffects(
+                      new Effects(
+                          /* audioProcessors= */ ImmutableList.of(),
+                          /* videoEffects= */ ImmutableList.of(new Contrast(0.5f))))
+                  .build());
+          compositionPlayer.prepare();
+          compositionPlayer.play();
+        });
+
+    playerTestListener.waitUntilPlayerReady();
+    for (int i = 0; i < 180; i++) {
+      // Replaying every 10 ms.
+      mainHandler.postDelayed(
+          compositionPlayer::experimentalRedrawLastFrame, /* delayMillis= */ 10 * i);
+    }
+
+    playerTestListener.waitUntilPlayerEnded();
+  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java

@@ -53,6 +53,7 @@ import androidx.media3.common.util.Log;
 import androidx.media3.common.util.Size;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
+import androidx.media3.effect.DefaultVideoFrameProcessor;
 import androidx.media3.effect.SingleInputVideoGraph;
 import androidx.media3.effect.TimestampAdjustment;
 import androidx.media3.exoplayer.ExoPlaybackException;
@@ -125,6 +126,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
     private boolean videoPrewarmingEnabled;
     private Clock clock;
     private VideoGraph.@MonotonicNonNull Factory videoGraphFactory;
+    private boolean enableReplayableCache;
     private boolean built;
 
     /**
@@ -246,6 +248,21 @@ public final class CompositionPlayer extends SimpleBasePlayer
       return this;
     }
 
+    /**
+     * Sets whether to enable replayable cache.
+     *
+     * <p>By default, the replayable cache is not enabled. Enable it to achieve accurate effect
+     * update, at the cost of using more power and computing resources.
+     *
+     * @param enableReplayableCache Whether replayable cache is enabled.
+     * @return This builder, for convenience.
+     */
+    @CanIgnoreReturnValue
+    public Builder experimentalSetEnableReplayableCache(boolean enableReplayableCache) {
+      this.enableReplayableCache = enableReplayableCache;
+      return this;
+    }
+
     /**
      * Builds the {@link CompositionPlayer} instance. Must be called at most once.
      *
@@ -262,7 +279,11 @@ public final class CompositionPlayer extends SimpleBasePlayer
         audioSink = new DefaultAudioSink.Builder(context).build();
       }
       if (videoGraphFactory == null) {
-        videoGraphFactory = new SingleInputVideoGraph.Factory();
+        videoGraphFactory =
+            new SingleInputVideoGraph.Factory(
+                new DefaultVideoFrameProcessor.Factory.Builder()
+                    .setEnableReplayableCache(enableReplayableCache)
+                    .build());
       }
       CompositionPlayer compositionPlayer = new CompositionPlayer(this);
       built = true;
@@ -311,11 +332,13 @@ public final class CompositionPlayer extends SimpleBasePlayer
   private final VideoGraph.Factory videoGraphFactory;
   private final boolean videoPrewarmingEnabled;
   private final HandlerWrapper compositionInternalListenerHandler;
+  private final boolean enableReplayableCache;
 
   /** Maps from input index to whether the video track is selected in that sequence. */
   private final SparseBooleanArray videoTracksSelected;
 
   private @MonotonicNonNull HandlerThread playbackThread;
+  private @MonotonicNonNull HandlerWrapper playbackThreadHandler;
   private @MonotonicNonNull CompositionPlayerInternal compositionPlayerInternal;
   private @MonotonicNonNull ImmutableList<MediaItemData> playlist;
   private @MonotonicNonNull Composition composition;
@@ -352,6 +375,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
     videoGraphFactory = checkNotNull(builder.videoGraphFactory);
     videoPrewarmingEnabled = builder.videoPrewarmingEnabled;
     compositionInternalListenerHandler = clock.createHandler(builder.looper, /* callback= */ null);
+    this.enableReplayableCache = builder.enableReplayableCache;
     videoTracksSelected = new SparseBooleanArray();
     players = new ArrayList<>();
     compositionDurationUs = C.TIME_UNSET;
@@ -398,6 +422,21 @@ public final class CompositionPlayer extends SimpleBasePlayer
     this.composition = composition;
   }
 
+  /**
+   * Forces the effect pipeline to redraw the effects immediately.
+   *
+   * <p>The player must be {@linkplain Builder#experimentalSetEnableReplayableCache built with
+   * replayable cache support}.
+   */
+  public void experimentalRedrawLastFrame() {
+    checkState(enableReplayableCache);
+    if (playbackThreadHandler == null || playbackVideoGraphWrapper == null) {
+      // Ignore replays before setting a composition.
+      return;
+    }
+    playbackThreadHandler.post(() -> checkNotNull(playbackVideoGraphWrapper).getSink(0).redraw());
+  }
+
   /** Sets the {@link Surface} and {@link Size} to render to. */
   @VisibleForTesting
   public void setVideoSurface(Surface surface, Size videoOutputSize) {
@@ -714,6 +753,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
     compositionDurationUs = getCompositionDurationUs(composition);
     playbackThread = new HandlerThread("CompositionPlaybackThread", Process.THREAD_PRIORITY_AUDIO);
     playbackThread.start();
+    playbackThreadHandler = clock.createHandler(playbackThread.getLooper(), /* callback= */ null);
     // Create the audio and video composition components now in order to setup the audio and video
     // pipelines. Once this method returns, further access to the audio and video graph wrappers
     // must done on the playback thread only, to ensure related components are accessed from one
@@ -734,6 +774,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
             .setClock(clock)
             .setRequestOpenGlToneMapping(
                 composition.hdrMode == Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
+            .setEnableReplayableCache(enableReplayableCache)
             .build();
     playbackVideoGraphWrapper.addListener(this);