diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index 9b7e735644..a6edceb483 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -366,7 +366,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video @Override public VideoSink getSink(int inputIndex) { - checkState(!contains(inputVideoSinks, inputIndex)); + if (contains(inputVideoSinks, inputIndex)) { + return inputVideoSinks.get(inputIndex); + } InputVideoSink inputVideoSink = new InputVideoSink(context, inputIndex); if (inputIndex == PRIMARY_SEQUENCE_INDEX) { addListener(inputVideoSink); @@ -730,7 +732,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video @Override public void redraw() { - checkState(isInitialized()); + if (!isInitialized()) { + return; + } // Resignal EOS only for the last item. boolean needsResignalEndOfCurrentInputStream = signaledEndOfStream; long replayedPresentationTimeUs = lastOutputBufferPresentationTimeUs; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index c10a14b8c2..c626c75179 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -481,6 +481,7 @@ public final class AndroidTestUtil { .setFrameRate(30.00f) .setCodecs("avc1.42C033") .build()) + .setVideoDurationUs(1_000_000L) .build(); public static final AssetInfo MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS = diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java index 6fd569f1b8..f0cb68c9a2 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java @@ -32,9 +32,11 @@ import androidx.annotation.Nullable; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.PlaybackException; +import androidx.media3.common.Player; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.Util; import androidx.media3.effect.Contrast; +import androidx.media3.effect.GlEffect; import androidx.media3.exoplayer.DefaultRenderersFactory; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.Renderer; @@ -43,6 +45,12 @@ import androidx.media3.exoplayer.util.EventLogger; import androidx.media3.exoplayer.video.VideoFrameMetadataListener; import androidx.media3.exoplayer.video.VideoRendererEventListener; import androidx.media3.transformer.AndroidTestUtil.ReplayVideoRenderer; +import androidx.media3.transformer.Composition; +import androidx.media3.transformer.CompositionPlayer; +import androidx.media3.transformer.EditedMediaItem; +import androidx.media3.transformer.EditedMediaItemSequence; +import androidx.media3.transformer.Effects; +import androidx.media3.transformer.InputTimestampRecordingShaderProgram; import androidx.media3.transformer.PlayerTestListener; import androidx.media3.transformer.SurfaceTestActivity; import androidx.test.ext.junit.rules.ActivityScenarioRule; @@ -64,8 +72,11 @@ public class ReplayCacheTest { private static final long TEST_TIMEOUT_MS = isRunningOnEmulator() ? 20_000 : 10_000; private static final MediaItem VIDEO_MEDIA_ITEM_1 = MediaItem.fromUri(MP4_ASSET.uri); + private static final long VIDEO_MEDIA_ITEM_1_DURATION_US = MP4_ASSET.videoDurationUs; private static final MediaItem VIDEO_MEDIA_ITEM_2 = MediaItem.fromUri(MP4_ASSET_WITH_INCREASING_TIMESTAMPS.uri); + private static final long VIDEO_MEDIA_ITEM_2_DURATION_US = + MP4_ASSET_WITH_INCREASING_TIMESTAMPS.videoDurationUs; @Rule public ActivityScenarioRule rule = @@ -74,6 +85,7 @@ public class ReplayCacheTest { private final Context context = getInstrumentation().getContext().getApplicationContext(); private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + private CompositionPlayer compositionPlayer; private ExoPlayer exoPlayer; private SurfaceView surfaceView; @@ -88,6 +100,9 @@ public class ReplayCacheTest { getInstrumentation() .runOnMainSync( () -> { + if (compositionPlayer != null) { + compositionPlayer.release(); + } if (exoPlayer != null) { exoPlayer.release(); } @@ -159,4 +174,108 @@ public class ReplayCacheTest { // we don't currently replay (the frame at media item transition). assertThat(playedFrameTimestampsUs).hasSize(119); } + + @Test + public void enableReplay_withCompositionPlayerSingleSequence_playsSequence() throws Exception { + assumeTrue( + "The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite" + + " using MediaFormat.KEY_ALLOW_FRAME_DROP.", + !Util.isRunningOnEmulator()); + PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS * 1000); + InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram = + new InputTimestampRecordingShaderProgram(); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = + new CompositionPlayer.Builder(context) + .experimentalSetEnableReplayableCache(true) + .build(); + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(playerTestListener); + compositionPlayer.addListener( + new Player.Listener() { + @Override + public void onRenderedFirstFrame() { + compositionPlayer.experimentalRedrawLastFrame(); + compositionPlayer.play(); + } + }); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence.Builder( + new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_1) + .setDurationUs(VIDEO_MEDIA_ITEM_1_DURATION_US) + .build(), + new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_2) + .setDurationUs(VIDEO_MEDIA_ITEM_2_DURATION_US) + .build()) + .build()) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(), + /* videoEffects= */ ImmutableList.of( + new Contrast(0.5f), + (GlEffect) + (context, useHdr) -> inputTimestampRecordingShaderProgram))) + .build()); + compositionPlayer.prepare(); + }); + + playerTestListener.waitUntilPlayerEnded(); + + int countOfFirstFrameRendered = 0; + for (long timestampUs : inputTimestampRecordingShaderProgram.getInputTimestampsUs()) { + if (timestampUs == 0) { + countOfFirstFrameRendered++; + } + } + assertThat(countOfFirstFrameRendered).isEqualTo(2); + } + + @Test + public void rapidReplay_withCompositionPlayerSingleSequence_playsSequence() throws Exception { + assumeTrue( + "The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite" + + " using MediaFormat.KEY_ALLOW_FRAME_DROP.", + !Util.isRunningOnEmulator()); + PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS); + Handler mainHandler = new Handler(instrumentation.getTargetContext().getMainLooper()); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = + new CompositionPlayer.Builder(context) + .experimentalSetEnableReplayableCache(true) + .build(); + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(playerTestListener); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence.Builder( + new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_1) + .setDurationUs(VIDEO_MEDIA_ITEM_1_DURATION_US) + .build(), + new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_2) + .setDurationUs(VIDEO_MEDIA_ITEM_2_DURATION_US) + .build()) + .build()) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(), + /* videoEffects= */ ImmutableList.of(new Contrast(0.5f)))) + .build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + playerTestListener.waitUntilPlayerReady(); + for (int i = 0; i < 180; i++) { + // Replaying every 10 ms. + mainHandler.postDelayed( + compositionPlayer::experimentalRedrawLastFrame, /* delayMillis= */ 10 * i); + } + + playerTestListener.waitUntilPlayerEnded(); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java index 0c2a192858..95bc5918df 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java @@ -53,6 +53,7 @@ import androidx.media3.common.util.Log; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; +import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.SingleInputVideoGraph; import androidx.media3.effect.TimestampAdjustment; import androidx.media3.exoplayer.ExoPlaybackException; @@ -125,6 +126,7 @@ public final class CompositionPlayer extends SimpleBasePlayer private boolean videoPrewarmingEnabled; private Clock clock; private VideoGraph.@MonotonicNonNull Factory videoGraphFactory; + private boolean enableReplayableCache; private boolean built; /** @@ -246,6 +248,21 @@ public final class CompositionPlayer extends SimpleBasePlayer return this; } + /** + * Sets whether to enable replayable cache. + * + *

By default, the replayable cache is not enabled. Enable it to achieve accurate effect + * update, at the cost of using more power and computing resources. + * + * @param enableReplayableCache Whether replayable cache is enabled. + * @return This builder, for convenience. + */ + @CanIgnoreReturnValue + public Builder experimentalSetEnableReplayableCache(boolean enableReplayableCache) { + this.enableReplayableCache = enableReplayableCache; + return this; + } + /** * Builds the {@link CompositionPlayer} instance. Must be called at most once. * @@ -262,7 +279,11 @@ public final class CompositionPlayer extends SimpleBasePlayer audioSink = new DefaultAudioSink.Builder(context).build(); } if (videoGraphFactory == null) { - videoGraphFactory = new SingleInputVideoGraph.Factory(); + videoGraphFactory = + new SingleInputVideoGraph.Factory( + new DefaultVideoFrameProcessor.Factory.Builder() + .setEnableReplayableCache(enableReplayableCache) + .build()); } CompositionPlayer compositionPlayer = new CompositionPlayer(this); built = true; @@ -311,11 +332,13 @@ public final class CompositionPlayer extends SimpleBasePlayer private final VideoGraph.Factory videoGraphFactory; private final boolean videoPrewarmingEnabled; private final HandlerWrapper compositionInternalListenerHandler; + private final boolean enableReplayableCache; /** Maps from input index to whether the video track is selected in that sequence. */ private final SparseBooleanArray videoTracksSelected; private @MonotonicNonNull HandlerThread playbackThread; + private @MonotonicNonNull HandlerWrapper playbackThreadHandler; private @MonotonicNonNull CompositionPlayerInternal compositionPlayerInternal; private @MonotonicNonNull ImmutableList playlist; private @MonotonicNonNull Composition composition; @@ -352,6 +375,7 @@ public final class CompositionPlayer extends SimpleBasePlayer videoGraphFactory = checkNotNull(builder.videoGraphFactory); videoPrewarmingEnabled = builder.videoPrewarmingEnabled; compositionInternalListenerHandler = clock.createHandler(builder.looper, /* callback= */ null); + this.enableReplayableCache = builder.enableReplayableCache; videoTracksSelected = new SparseBooleanArray(); players = new ArrayList<>(); compositionDurationUs = C.TIME_UNSET; @@ -398,6 +422,21 @@ public final class CompositionPlayer extends SimpleBasePlayer this.composition = composition; } + /** + * Forces the effect pipeline to redraw the effects immediately. + * + *

The player must be {@linkplain Builder#experimentalSetEnableReplayableCache built with + * replayable cache support}. + */ + public void experimentalRedrawLastFrame() { + checkState(enableReplayableCache); + if (playbackThreadHandler == null || playbackVideoGraphWrapper == null) { + // Ignore replays before setting a composition. + return; + } + playbackThreadHandler.post(() -> checkNotNull(playbackVideoGraphWrapper).getSink(0).redraw()); + } + /** Sets the {@link Surface} and {@link Size} to render to. */ @VisibleForTesting public void setVideoSurface(Surface surface, Size videoOutputSize) { @@ -714,6 +753,7 @@ public final class CompositionPlayer extends SimpleBasePlayer compositionDurationUs = getCompositionDurationUs(composition); playbackThread = new HandlerThread("CompositionPlaybackThread", Process.THREAD_PRIORITY_AUDIO); playbackThread.start(); + playbackThreadHandler = clock.createHandler(playbackThread.getLooper(), /* callback= */ null); // Create the audio and video composition components now in order to setup the audio and video // pipelines. Once this method returns, further access to the audio and video graph wrappers // must done on the playback thread only, to ensure related components are accessed from one @@ -734,6 +774,7 @@ public final class CompositionPlayer extends SimpleBasePlayer .setClock(clock) .setRequestOpenGlToneMapping( composition.hdrMode == Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL) + .setEnableReplayableCache(enableReplayableCache) .build(); playbackVideoGraphWrapper.addListener(this);