loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-29 22:36:54 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Allow replays with CompositionPlayer

Browse Source 
PiperOrigin-RevId: 745536073
This commit is contained in:
claincly 2025-04-09 04:53:18 -07:00 committed by Copybara-Service
parent 71fb3ad5a5
commit 4bfa154acd
4 changed files with 168 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java
View File

@ -366,7 +366,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
@Override
public VideoSink getSink(int inputIndex) {
checkState(!contains(inputVideoSinks, inputIndex));
if (contains(inputVideoSinks, inputIndex)) {
return inputVideoSinks.get(inputIndex);
}
InputVideoSink inputVideoSink = new InputVideoSink(context, inputIndex);
if (inputIndex == PRIMARY_SEQUENCE_INDEX) {
addListener(inputVideoSink);
@ -730,7 +732,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
@Override
public void redraw() {
checkState(isInitialized());
if (!isInitialized()) {
return;
}
// Resignal EOS only for the last item.
boolean needsResignalEndOfCurrentInputStream = signaledEndOfStream;
long replayedPresentationTimeUs = lastOutputBufferPresentationTimeUs;

1
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -481,6 +481,7 @@ public final class AndroidTestUtil {
.setFrameRate(30.00f)
.setCodecs("avc1.42C033")
.build())
.setVideoDurationUs(1_000_000L)
.build();
public static final AssetInfo MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS =

119
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java
View File

@ -32,9 +32,11 @@ import androidx.annotation.Nullable;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.Player;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.Util;
import androidx.media3.effect.Contrast;
import androidx.media3.effect.GlEffect;
import androidx.media3.exoplayer.DefaultRenderersFactory;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.Renderer;
@ -43,6 +45,12 @@ import androidx.media3.exoplayer.util.EventLogger;
import androidx.media3.exoplayer.video.VideoFrameMetadataListener;
import androidx.media3.exoplayer.video.VideoRendererEventListener;
import androidx.media3.transformer.AndroidTestUtil.ReplayVideoRenderer;
import androidx.media3.transformer.Composition;
import androidx.media3.transformer.CompositionPlayer;
import androidx.media3.transformer.EditedMediaItem;
import androidx.media3.transformer.EditedMediaItemSequence;
import androidx.media3.transformer.Effects;
import androidx.media3.transformer.InputTimestampRecordingShaderProgram;
import androidx.media3.transformer.PlayerTestListener;
import androidx.media3.transformer.SurfaceTestActivity;
import androidx.test.ext.junit.rules.ActivityScenarioRule;
@ -64,8 +72,11 @@ public class ReplayCacheTest {
private static final long TEST_TIMEOUT_MS = isRunningOnEmulator() ? 20_000 : 10_000;
private static final MediaItem VIDEO_MEDIA_ITEM_1 = MediaItem.fromUri(MP4_ASSET.uri);
private static final long VIDEO_MEDIA_ITEM_1_DURATION_US = MP4_ASSET.videoDurationUs;
private static final MediaItem VIDEO_MEDIA_ITEM_2 =
MediaItem.fromUri(MP4_ASSET_WITH_INCREASING_TIMESTAMPS.uri);
private static final long VIDEO_MEDIA_ITEM_2_DURATION_US =
MP4_ASSET_WITH_INCREASING_TIMESTAMPS.videoDurationUs;
@Rule
public ActivityScenarioRule<SurfaceTestActivity> rule =
@ -74,6 +85,7 @@ public class ReplayCacheTest {
private final Context context = getInstrumentation().getContext().getApplicationContext();
private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
private CompositionPlayer compositionPlayer;
private ExoPlayer exoPlayer;
private SurfaceView surfaceView;
@ -88,6 +100,9 @@ public class ReplayCacheTest {
getInstrumentation()
.runOnMainSync(
() -> {
if (compositionPlayer != null) {
compositionPlayer.release();
}
if (exoPlayer != null) {
exoPlayer.release();
}
@ -159,4 +174,108 @@ public class ReplayCacheTest {
// we don't currently replay (the frame at media item transition).
assertThat(playedFrameTimestampsUs).hasSize(119);
}
@Test
public void enableReplay_withCompositionPlayerSingleSequence_playsSequence() throws Exception {
assumeTrue(
"The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite"
+ " using MediaFormat.KEY_ALLOW_FRAME_DROP.",
!Util.isRunningOnEmulator());
PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS * 1000);
InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram =
new InputTimestampRecordingShaderProgram();
instrumentation.runOnMainSync(
() -> {
compositionPlayer =
new CompositionPlayer.Builder(context)
.experimentalSetEnableReplayableCache(true)
.build();
compositionPlayer.setVideoSurfaceView(surfaceView);
compositionPlayer.addListener(playerTestListener);
compositionPlayer.addListener(
new Player.Listener() {
@Override
public void onRenderedFirstFrame() {
compositionPlayer.experimentalRedrawLastFrame();
compositionPlayer.play();
}
});
compositionPlayer.setComposition(
new Composition.Builder(
new EditedMediaItemSequence.Builder(
new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_1)
.setDurationUs(VIDEO_MEDIA_ITEM_1_DURATION_US)
.build(),
new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_2)
.setDurationUs(VIDEO_MEDIA_ITEM_2_DURATION_US)
.build())
.build())
.setEffects(
new Effects(
/* audioProcessors= */ ImmutableList.of(),
/* videoEffects= */ ImmutableList.of(
new Contrast(0.5f),
(GlEffect)
(context, useHdr) -> inputTimestampRecordingShaderProgram)))
.build());
compositionPlayer.prepare();
});
playerTestListener.waitUntilPlayerEnded();
int countOfFirstFrameRendered = 0;
for (long timestampUs : inputTimestampRecordingShaderProgram.getInputTimestampsUs()) {
if (timestampUs == 0) {
countOfFirstFrameRendered++;
}
}
assertThat(countOfFirstFrameRendered).isEqualTo(2);
}
@Test
public void rapidReplay_withCompositionPlayerSingleSequence_playsSequence() throws Exception {
assumeTrue(
"The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite"
+ " using MediaFormat.KEY_ALLOW_FRAME_DROP.",
!Util.isRunningOnEmulator());
PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS);
Handler mainHandler = new Handler(instrumentation.getTargetContext().getMainLooper());
instrumentation.runOnMainSync(
() -> {
compositionPlayer =
new CompositionPlayer.Builder(context)
.experimentalSetEnableReplayableCache(true)
.build();
compositionPlayer.setVideoSurfaceView(surfaceView);
compositionPlayer.addListener(playerTestListener);
compositionPlayer.setComposition(
new Composition.Builder(
new EditedMediaItemSequence.Builder(
new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_1)
.setDurationUs(VIDEO_MEDIA_ITEM_1_DURATION_US)
.build(),
new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM_2)
.setDurationUs(VIDEO_MEDIA_ITEM_2_DURATION_US)
.build())
.build())
.setEffects(
new Effects(
/* audioProcessors= */ ImmutableList.of(),
/* videoEffects= */ ImmutableList.of(new Contrast(0.5f))))
.build());
compositionPlayer.prepare();
compositionPlayer.play();
});
playerTestListener.waitUntilPlayerReady();
for (int i = 0; i < 180; i++) {
// Replaying every 10 ms.
mainHandler.postDelayed(
compositionPlayer::experimentalRedrawLastFrame, /* delayMillis= */ 10 * i);
}
playerTestListener.waitUntilPlayerEnded();
}
}

43
libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java
View File

@ -53,6 +53,7 @@ import androidx.media3.common.util.Log;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.effect.DefaultVideoFrameProcessor;
import androidx.media3.effect.SingleInputVideoGraph;
import androidx.media3.effect.TimestampAdjustment;
import androidx.media3.exoplayer.ExoPlaybackException;
@ -125,6 +126,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
private boolean videoPrewarmingEnabled;
private Clock clock;
private VideoGraph.@MonotonicNonNull Factory videoGraphFactory;
private boolean enableReplayableCache;
private boolean built;
/**
@ -246,6 +248,21 @@ public final class CompositionPlayer extends SimpleBasePlayer
return this;
}
/**
* Sets whether to enable replayable cache.
*
* <p>By default, the replayable cache is not enabled. Enable it to achieve accurate effect
* update, at the cost of using more power and computing resources.
*
* @param enableReplayableCache Whether replayable cache is enabled.
* @return This builder, for convenience.
*/
@CanIgnoreReturnValue
public Builder experimentalSetEnableReplayableCache(boolean enableReplayableCache) {
this.enableReplayableCache = enableReplayableCache;
return this;
}
/**
* Builds the {@link CompositionPlayer} instance. Must be called at most once.
*
@ -262,7 +279,11 @@ public final class CompositionPlayer extends SimpleBasePlayer
audioSink = new DefaultAudioSink.Builder(context).build();
}
if (videoGraphFactory == null) {
videoGraphFactory = new SingleInputVideoGraph.Factory();
videoGraphFactory =
new SingleInputVideoGraph.Factory(
new DefaultVideoFrameProcessor.Factory.Builder()
.setEnableReplayableCache(enableReplayableCache)
.build());
}
CompositionPlayer compositionPlayer = new CompositionPlayer(this);
built = true;
@ -311,11 +332,13 @@ public final class CompositionPlayer extends SimpleBasePlayer
private final VideoGraph.Factory videoGraphFactory;
private final boolean videoPrewarmingEnabled;
private final HandlerWrapper compositionInternalListenerHandler;
private final boolean enableReplayableCache;
/** Maps from input index to whether the video track is selected in that sequence. */
private final SparseBooleanArray videoTracksSelected;
private @MonotonicNonNull HandlerThread playbackThread;
private @MonotonicNonNull HandlerWrapper playbackThreadHandler;
private @MonotonicNonNull CompositionPlayerInternal compositionPlayerInternal;
private @MonotonicNonNull ImmutableList<MediaItemData> playlist;
private @MonotonicNonNull Composition composition;
@ -352,6 +375,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
videoGraphFactory = checkNotNull(builder.videoGraphFactory);
videoPrewarmingEnabled = builder.videoPrewarmingEnabled;
compositionInternalListenerHandler = clock.createHandler(builder.looper, /* callback= */ null);
this.enableReplayableCache = builder.enableReplayableCache;
videoTracksSelected = new SparseBooleanArray();
players = new ArrayList<>();
compositionDurationUs = C.TIME_UNSET;
@ -398,6 +422,21 @@ public final class CompositionPlayer extends SimpleBasePlayer
this.composition = composition;
}
/**
* Forces the effect pipeline to redraw the effects immediately.
*
* <p>The player must be {@linkplain Builder#experimentalSetEnableReplayableCache built with
* replayable cache support}.
*/
public void experimentalRedrawLastFrame() {
checkState(enableReplayableCache);
if (playbackThreadHandler == null || playbackVideoGraphWrapper == null) {
// Ignore replays before setting a composition.
return;
}
playbackThreadHandler.post(() -> checkNotNull(playbackVideoGraphWrapper).getSink(0).redraw());
}
/** Sets the {@link Surface} and {@link Size} to render to. */
@VisibleForTesting
public void setVideoSurface(Surface surface, Size videoOutputSize) {
@ -714,6 +753,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
compositionDurationUs = getCompositionDurationUs(composition);
playbackThread = new HandlerThread("CompositionPlaybackThread", Process.THREAD_PRIORITY_AUDIO);
playbackThread.start();
playbackThreadHandler = clock.createHandler(playbackThread.getLooper(), /* callback= */ null);
// Create the audio and video composition components now in order to setup the audio and video
// pipelines. Once this method returns, further access to the audio and video graph wrappers
// must done on the playback thread only, to ensure related components are accessed from one
@ -734,6 +774,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
.setClock(clock)
.setRequestOpenGlToneMapping(
composition.hdrMode == Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
.setEnableReplayableCache(enableReplayableCache)
.build();
playbackVideoGraphWrapper.addListener(this);