Add video prewarming to CompositionPlayer

PiperOrigin-RevId: 725153751
This commit is contained in:
kimvde 2025-02-10 04:02:19 -08:00 committed by Copybara-Service
parent aa6183e883
commit cadecf0219
3 changed files with 224 additions and 35 deletions

View File

@ -23,6 +23,7 @@ import static androidx.media3.common.util.Util.isRunningOnEmulator;
import static androidx.media3.common.util.Util.usToMs;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static androidx.test.platform.app.InstrumentationRegistry.getInstrumentation;
import static com.google.common.truth.Truth.assertThat;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@ -42,13 +43,19 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
import org.junit.AssumptionViolatedException;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
/** Playback tests for {@link CompositionPlayer} */
@RunWith(AndroidJUnit4.class)
public class CompositionPlaybackTest {
@Rule public final TestName testName = new TestName();
private static final long TEST_TIMEOUT_MS = isRunningOnEmulator() ? 20_000 : 10_000;
private static final MediaItem VIDEO_MEDIA_ITEM = MediaItem.fromUri(MP4_ASSET.uri);
private static final long VIDEO_DURATION_US = MP4_ASSET.videoDurationUs;
@ -66,8 +73,14 @@ public class CompositionPlaybackTest {
private final Context context = getInstrumentation().getContext().getApplicationContext();
private final PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS);
private String testId;
private @MonotonicNonNull CompositionPlayer player;
@Before
public void setUp() {
testId = testName.getMethodName();
}
@After
public void tearDown() {
getInstrumentation()
@ -209,6 +222,12 @@ public class CompositionPlaybackTest {
@Test
public void playback_sequenceOfImageAndVideo_effectsReceiveCorrectTimestamps() throws Exception {
if (isRunningOnEmulator()) {
// The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite
// using MediaFormat.KEY_ALLOW_FRAME_DROP.
recordTestSkipped(context, testId, /* reason= */ "Skipped due to surface dropping frames");
throw new AssumptionViolatedException("Skipped due to surface dropping frames");
}
InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram =
new InputTimestampRecordingShaderProgram();
Effect videoEffect = (GlEffect) (context, useHdr) -> inputTimestampRecordingShaderProgram;

View File

@ -126,7 +126,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToZero_afterPlayingSingleSequenceOfTwoVideos() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<Long> sequenceTimestampsUs =
new ImmutableList.Builder<Long>()
// Plays the first video
@ -150,7 +153,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToFirstVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
// Skips the first three video frames
long seekTimeMs = 100;
ImmutableList<Long> sequenceTimestampsUs =
@ -174,7 +180,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToStartOfSecondVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
// Seeks to the end of the first video
long seekTimeMs = usToMs(VIDEO_DURATION_US);
ImmutableList<Long> sequenceTimestampsUs =
@ -197,7 +206,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToSecondVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
// Skips the first three image frames of the second image.
long seekTimeMs = usToMs(VIDEO_DURATION_US) + 100;
ImmutableList<Long> sequenceTimestampsUs =
@ -222,7 +234,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToEndOfSecondVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
// Seeks to the end of the second video
long seekTimeMs = usToMs(2 * VIDEO_DURATION_US);
ImmutableList<Long> sequenceTimestampsUs =
@ -244,7 +259,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToAfterEndOfSecondVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
long seekTimeMs = usToMs(3 * VIDEO_DURATION_US);
ImmutableList<Long> sequenceTimestampsUs =
new ImmutableList.Builder<Long>()
@ -398,7 +416,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToZero_afterPlayingSingleSequenceOfVideoAndImage() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<Long> sequenceTimestampsUs =
new ImmutableList.Builder<Long>()
// Plays the video
@ -422,7 +443,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToVideo_afterPlayingSingleSequenceOfVideoAndImage() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
// Skips three video frames
long seekTimeMs = 100;
ImmutableList<Long> sequenceTimestampsUs =
@ -447,7 +471,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToImage_afterPlayingSingleSequenceOfVideoAndImage() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
// Skips video frames and three image frames
long seekTimeMs = usToMs(VIDEO_DURATION_US) + 100;
ImmutableList<Long> sequenceTimestampsUs =
@ -472,7 +499,11 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToZero_afterPlayingSingleSequenceOfImageAndVideo() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator()) {
// The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite
// using MediaFormat.KEY_ALLOW_FRAME_DROP.
skipTest("Skipped due to surface dropping frames");
}
ImmutableList<Long> sequenceTimestampsUs =
new ImmutableList.Builder<Long>()
// Plays the image
@ -496,7 +527,11 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToImage_afterPlayingSingleSequenceOfImageAndVideo() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator()) {
// The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite
// using MediaFormat.KEY_ALLOW_FRAME_DROP.
skipTest("Skipped due to surface dropping frames");
}
// Skips three image frames
long seekTimeMs = 100;
ImmutableList<Long> sequenceTimestampsUs =
@ -520,7 +555,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToVideo_afterPlayingSingleSequenceOfImageAndVideo() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
// Skips to the first video frame.
long seekTimeMs = usToMs(IMAGE_DURATION_US);
ImmutableList<Long> sequenceTimestampsUs =
@ -543,7 +581,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToZero_duringPlayingFirstVideoInSingleSequenceOfTwoVideos() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<MediaItemConfig> mediaItems =
ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM);
int numberOfFramesBeforeSeeking = 15;
@ -569,7 +610,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToSecondVideo_duringPlayingFirstVideoInSingleSequenceOfTwoVideos()
throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<MediaItemConfig> mediaItems =
ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM);
int numberOfFramesBeforeSeeking = 15;
@ -596,7 +640,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToFirstVideo_duringPlayingSecondVideoInSingleSequenceOfTwoVideos()
throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<MediaItemConfig> mediaItems =
ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM);
int numberOfFramesBeforeSeeking = 45;
@ -627,7 +674,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToEndOfFirstVideo_duringPlayingFirstVideoInSingleSequenceOfTwoVideos()
throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<MediaItemConfig> mediaItems =
ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM);
int numberOfFramesBeforeSeeking = 15;
@ -652,7 +702,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToEndOfSecondVideo_duringPlayingFirstVideoInSingleSequenceOfTwoVideos()
throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<MediaItemConfig> mediaItems =
ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM);
int numberOfFramesBeforeSeeking = 15;
@ -675,7 +728,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToFirstImage_duringPlayingFirstImageInSequenceOfTwoImages() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<MediaItemConfig> mediaItems = ImmutableList.of(IMAGE_MEDIA_ITEM);
int numberOfFramesBeforeSeeking = 2;
// Should skip the first 3 frames.
@ -722,7 +778,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToImage_duringPlayingFirstImageInSequenceOfVideoAndImage() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<MediaItemConfig> mediaItems =
ImmutableList.of(VIDEO_MEDIA_ITEM, IMAGE_MEDIA_ITEM);
int numberOfFramesBeforeSeeking = 15;
@ -748,7 +807,10 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToVideo_duringPlayingFirstImageInSequenceOfImageAndVideo() throws Exception {
maybeSkipTest();
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
skipTest("Skipped due to failing decoder");
}
ImmutableList<MediaItemConfig> mediaItems =
ImmutableList.of(IMAGE_MEDIA_ITEM, VIDEO_MEDIA_ITEM);
int numberOfFramesBeforeSeeking = 3;
@ -772,12 +834,9 @@ public class CompositionPlayerSeekTest {
assertThat(actualTimestampsUs).isEqualTo(expectedTimestampsUs);
}
private void maybeSkipTest() throws Exception {
if (isRunningOnEmulator() && Util.SDK_INT == 31) {
// The audio decoder is failing on API 31 emulator.
recordTestSkipped(applicationContext, testId, /* reason= */ "Skipped due to failing decoder");
throw new AssumptionViolatedException("Skipped due to failing decoder");
}
private void skipTest(String reason) throws Exception {
recordTestSkipped(applicationContext, testId, reason);
throw new AssumptionViolatedException(reason);
}
/**

View File

@ -21,6 +21,7 @@ import static androidx.media3.common.PlaybackException.ERROR_CODE_VIDEO_FRAME_PR
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.common.util.Util.SDK_INT;
import static androidx.media3.exoplayer.DefaultRenderersFactory.DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS;
import static androidx.media3.exoplayer.DefaultRenderersFactory.MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY;
@ -28,6 +29,7 @@ import android.content.Context;
import android.graphics.Bitmap;
import android.media.MediaFormat;
import android.os.Handler;
import androidx.annotation.ChecksSdkIntAtLeast;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
@ -36,7 +38,6 @@ import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.Timeline;
import androidx.media3.common.util.ConstantRateTimestampIterator;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.ExoPlaybackException;
import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.RenderersFactory;
@ -47,6 +48,7 @@ import androidx.media3.exoplayer.image.ImageDecoder;
import androidx.media3.exoplayer.image.ImageOutput;
import androidx.media3.exoplayer.image.ImageRenderer;
import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.metadata.MetadataOutput;
import androidx.media3.exoplayer.source.MediaSource;
@ -132,7 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
eventHandler,
videoRendererEventListener,
sequence,
videoSink,
new BufferingVideoSink(context),
requestToneMapping));
renderers.add(
new SequenceImageRenderer(sequence, checkStateNotNull(imageDecoderFactory), videoSink));
@ -141,6 +143,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return renderers.toArray(new Renderer[0]);
}
@Nullable
@Override
public Renderer createSecondaryRenderer(
Renderer renderer,
Handler eventHandler,
VideoRendererEventListener videoRendererEventListener,
AudioRendererEventListener audioRendererEventListener,
TextOutput textRendererOutput,
MetadataOutput metadataRendererOutput) {
if (isVideoPrewarmingEnabled() && renderer instanceof SequenceVideoRenderer) {
return new SequenceVideoRenderer(
context,
eventHandler,
videoRendererEventListener,
sequence,
new BufferingVideoSink(context),
requestToneMapping);
}
return null;
}
private static long getOffsetToCompositionTimeUs(
EditedMediaItemSequence sequence, int mediaItemIndex, long offsetUs) {
// Reverse engineer how timestamps and offsets are computed with a ConcatenatingMediaSource2
@ -182,6 +205,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return sequence.editedMediaItems.get(index);
}
@ChecksSdkIntAtLeast(api = 23)
private static boolean isVideoPrewarmingEnabled() {
return SDK_INT >= 23;
}
private static final class SequenceAudioRenderer extends MediaCodecAudioRenderer {
private final EditedMediaItemSequence sequence;
private final AudioGraphInputAudioSink audioSink;
@ -264,10 +292,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
private static final class SequenceVideoRenderer extends MediaCodecVideoRenderer {
private final class SequenceVideoRenderer extends MediaCodecVideoRenderer {
private final EditedMediaItemSequence sequence;
private final VideoSink videoSink;
private final BufferingVideoSink bufferingVideoSink;
private final boolean requestToneMapping;
private ImmutableList<Effect> pendingEffects;
@ -279,7 +307,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Handler eventHandler,
VideoRendererEventListener videoRendererEventListener,
EditedMediaItemSequence sequence,
VideoSink videoSink,
BufferingVideoSink bufferingVideoSink,
boolean requestToneMapping) {
super(
new Builder(context)
@ -291,14 +319,39 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
.setEventListener(videoRendererEventListener)
.setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)
.setAssumedMinimumCodecOperatingRate(DEFAULT_FRAME_RATE)
.setVideoSink(videoSink));
.setVideoSink(bufferingVideoSink));
this.sequence = sequence;
this.videoSink = videoSink;
this.bufferingVideoSink = bufferingVideoSink;
this.requestToneMapping = requestToneMapping;
this.pendingEffects = ImmutableList.of();
experimentalEnableProcessedStreamChangedAtStart();
}
@Override
protected void onEnabled(boolean joining, boolean mayRenderStartOfStream)
throws ExoPlaybackException {
if (mayRenderStartOfStream) {
// Activate the BufferingVideoSink before calling super.onEnabled(), so that it points to a
// VideoSink when executing the super method.
activateBufferingVideoSink();
}
super.onEnabled(joining, mayRenderStartOfStream);
}
@Override
protected void onStarted() {
// Activate the BufferingVideoSink before calling super.onStarted(), so that it points to a
// VideoSink when executing the super method.
activateBufferingVideoSink();
super.onStarted();
}
@Override
protected void onDisabled() {
super.onDisabled();
deactivateBufferingVideoSink();
}
@Override
protected void onStreamChanged(
Format[] formats,
@ -333,13 +386,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
codecOperatingRate,
deviceNeedsNoPostProcessWorkaround,
tunnelingAudioSessionId);
if (requestToneMapping && Util.SDK_INT >= 31) {
if (requestToneMapping && SDK_INT >= 31) {
mediaFormat.setInteger(
MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
}
return mediaFormat;
}
@Override
public void handleMessage(@MessageType int messageType, @Nullable Object message)
throws ExoPlaybackException {
if (messageType == MSG_TRANSFER_RESOURCES) {
// Ignore MSG_TRANSFER_RESOURCES to avoid updating the VideoGraph's output surface.
return;
}
super.handleMessage(messageType, message);
}
@Override
protected boolean shouldInitCodec(MediaCodecInfo codecInfo) {
if (isVideoPrewarmingEnabled()
&& bufferingVideoSink.getVideoSink() == null
&& codecNeedsSetOutputSurfaceWorkaround(codecInfo.name)) {
// Wait until the BufferingVideoSink points to the effect VideoSink to init the codec, so
// that the codec output surface is set to the effect VideoSink input surface.
return false;
}
return super.shouldInitCodec(codecInfo);
}
@Override
protected long getBufferTimestampAdjustmentUs() {
return offsetToCompositionTimeUs;
@ -349,7 +424,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
protected void renderToEndOfStream() {
super.renderToEndOfStream();
if (isLastInSequence(getTimeline(), sequence, checkNotNull(currentEditedMediaItem))) {
videoSink.signalEndOfInput();
bufferingVideoSink.signalEndOfInput();
}
}
@ -358,6 +433,42 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
VideoSink videoSink, @VideoSink.InputType int inputType, Format format) {
videoSink.onInputStreamChanged(inputType, format, pendingEffects);
}
private void activateBufferingVideoSink() {
if (bufferingVideoSink.getVideoSink() != null) {
return;
}
VideoSink frameProcessingVideoSink = checkNotNull(SequenceRenderersFactory.this.videoSink);
bufferingVideoSink.setVideoSink(frameProcessingVideoSink);
@Nullable MediaCodecAdapter codec = getCodec();
if (isVideoPrewarmingEnabled()
&& frameProcessingVideoSink.isInitialized()
&& codec != null
&& !codecNeedsSetOutputSurfaceWorkaround(checkNotNull(getCodecInfo()).name)) {
setOutputSurfaceV23(codec, frameProcessingVideoSink.getInputSurface());
}
}
private void deactivateBufferingVideoSink() {
if (!isVideoPrewarmingEnabled()) {
return;
}
bufferingVideoSink.setVideoSink(null);
// During a seek, it's possible for the renderer to be disabled without having been started.
// When this happens, the BufferingVideoSink can have pending operations, so they need to be
// cleared.
bufferingVideoSink.clearPendingOperations();
@Nullable MediaCodecAdapter codec = getCodec();
if (codec == null) {
return;
}
if (!codecNeedsSetOutputSurfaceWorkaround(checkNotNull(getCodecInfo()).name)) {
// Sets a placeholder surface
setOutputSurfaceV23(codec, bufferingVideoSink.getInputSurface());
} else {
releaseCodec();
}
}
}
private static final class SequenceImageRenderer extends ImageRenderer {