diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlaybackTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlaybackTest.java index 83be5ff372..8b913df85f 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlaybackTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlaybackTest.java @@ -23,6 +23,7 @@ import static androidx.media3.common.util.Util.isRunningOnEmulator; import static androidx.media3.common.util.Util.usToMs; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET; import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET; +import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import static androidx.test.platform.app.InstrumentationRegistry.getInstrumentation; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -42,13 +43,19 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestName; import org.junit.runner.RunWith; /** Playback tests for {@link CompositionPlayer} */ @RunWith(AndroidJUnit4.class) public class CompositionPlaybackTest { + @Rule public final TestName testName = new TestName(); + private static final long TEST_TIMEOUT_MS = isRunningOnEmulator() ? 20_000 : 10_000; private static final MediaItem VIDEO_MEDIA_ITEM = MediaItem.fromUri(MP4_ASSET.uri); private static final long VIDEO_DURATION_US = MP4_ASSET.videoDurationUs; @@ -66,8 +73,14 @@ public class CompositionPlaybackTest { private final Context context = getInstrumentation().getContext().getApplicationContext(); private final PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS); + private String testId; private @MonotonicNonNull CompositionPlayer player; + @Before + public void setUp() { + testId = testName.getMethodName(); + } + @After public void tearDown() { getInstrumentation() @@ -209,6 +222,12 @@ public class CompositionPlaybackTest { @Test public void playback_sequenceOfImageAndVideo_effectsReceiveCorrectTimestamps() throws Exception { + if (isRunningOnEmulator()) { + // The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite + // using MediaFormat.KEY_ALLOW_FRAME_DROP. + recordTestSkipped(context, testId, /* reason= */ "Skipped due to surface dropping frames"); + throw new AssumptionViolatedException("Skipped due to surface dropping frames"); + } InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram = new InputTimestampRecordingShaderProgram(); Effect videoEffect = (GlEffect) (context, useHdr) -> inputTimestampRecordingShaderProgram; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java index 9cbb8ca2f0..20708cd8c1 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java @@ -126,7 +126,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToZero_afterPlayingSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList sequenceTimestampsUs = new ImmutableList.Builder() // Plays the first video @@ -150,7 +153,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToFirstVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } // Skips the first three video frames long seekTimeMs = 100; ImmutableList sequenceTimestampsUs = @@ -174,7 +180,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToStartOfSecondVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } // Seeks to the end of the first video long seekTimeMs = usToMs(VIDEO_DURATION_US); ImmutableList sequenceTimestampsUs = @@ -197,7 +206,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToSecondVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } // Skips the first three image frames of the second image. long seekTimeMs = usToMs(VIDEO_DURATION_US) + 100; ImmutableList sequenceTimestampsUs = @@ -222,7 +234,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToEndOfSecondVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } // Seeks to the end of the second video long seekTimeMs = usToMs(2 * VIDEO_DURATION_US); ImmutableList sequenceTimestampsUs = @@ -244,7 +259,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToAfterEndOfSecondVideo_afterPlayingSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } long seekTimeMs = usToMs(3 * VIDEO_DURATION_US); ImmutableList sequenceTimestampsUs = new ImmutableList.Builder() @@ -398,7 +416,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToZero_afterPlayingSingleSequenceOfVideoAndImage() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList sequenceTimestampsUs = new ImmutableList.Builder() // Plays the video @@ -422,7 +443,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToVideo_afterPlayingSingleSequenceOfVideoAndImage() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } // Skips three video frames long seekTimeMs = 100; ImmutableList sequenceTimestampsUs = @@ -447,7 +471,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToImage_afterPlayingSingleSequenceOfVideoAndImage() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } // Skips video frames and three image frames long seekTimeMs = usToMs(VIDEO_DURATION_US) + 100; ImmutableList sequenceTimestampsUs = @@ -472,7 +499,11 @@ public class CompositionPlayerSeekTest { @Test public void seekToZero_afterPlayingSingleSequenceOfImageAndVideo() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator()) { + // The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite + // using MediaFormat.KEY_ALLOW_FRAME_DROP. + skipTest("Skipped due to surface dropping frames"); + } ImmutableList sequenceTimestampsUs = new ImmutableList.Builder() // Plays the image @@ -496,7 +527,11 @@ public class CompositionPlayerSeekTest { @Test public void seekToImage_afterPlayingSingleSequenceOfImageAndVideo() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator()) { + // The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite + // using MediaFormat.KEY_ALLOW_FRAME_DROP. + skipTest("Skipped due to surface dropping frames"); + } // Skips three image frames long seekTimeMs = 100; ImmutableList sequenceTimestampsUs = @@ -520,7 +555,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToVideo_afterPlayingSingleSequenceOfImageAndVideo() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } // Skips to the first video frame. long seekTimeMs = usToMs(IMAGE_DURATION_US); ImmutableList sequenceTimestampsUs = @@ -543,7 +581,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToZero_duringPlayingFirstVideoInSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList mediaItems = ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM); int numberOfFramesBeforeSeeking = 15; @@ -569,7 +610,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToSecondVideo_duringPlayingFirstVideoInSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList mediaItems = ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM); int numberOfFramesBeforeSeeking = 15; @@ -596,7 +640,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToFirstVideo_duringPlayingSecondVideoInSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList mediaItems = ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM); int numberOfFramesBeforeSeeking = 45; @@ -627,7 +674,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToEndOfFirstVideo_duringPlayingFirstVideoInSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList mediaItems = ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM); int numberOfFramesBeforeSeeking = 15; @@ -652,7 +702,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToEndOfSecondVideo_duringPlayingFirstVideoInSingleSequenceOfTwoVideos() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList mediaItems = ImmutableList.of(VIDEO_MEDIA_ITEM, VIDEO_MEDIA_ITEM); int numberOfFramesBeforeSeeking = 15; @@ -675,7 +728,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToFirstImage_duringPlayingFirstImageInSequenceOfTwoImages() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList mediaItems = ImmutableList.of(IMAGE_MEDIA_ITEM); int numberOfFramesBeforeSeeking = 2; // Should skip the first 3 frames. @@ -722,7 +778,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToImage_duringPlayingFirstImageInSequenceOfVideoAndImage() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList mediaItems = ImmutableList.of(VIDEO_MEDIA_ITEM, IMAGE_MEDIA_ITEM); int numberOfFramesBeforeSeeking = 15; @@ -748,7 +807,10 @@ public class CompositionPlayerSeekTest { @Test public void seekToVideo_duringPlayingFirstImageInSequenceOfImageAndVideo() throws Exception { - maybeSkipTest(); + if (isRunningOnEmulator() && Util.SDK_INT == 31) { + // The audio decoder is failing on API 31 emulator. + skipTest("Skipped due to failing decoder"); + } ImmutableList mediaItems = ImmutableList.of(IMAGE_MEDIA_ITEM, VIDEO_MEDIA_ITEM); int numberOfFramesBeforeSeeking = 3; @@ -772,12 +834,9 @@ public class CompositionPlayerSeekTest { assertThat(actualTimestampsUs).isEqualTo(expectedTimestampsUs); } - private void maybeSkipTest() throws Exception { - if (isRunningOnEmulator() && Util.SDK_INT == 31) { - // The audio decoder is failing on API 31 emulator. - recordTestSkipped(applicationContext, testId, /* reason= */ "Skipped due to failing decoder"); - throw new AssumptionViolatedException("Skipped due to failing decoder"); - } + private void skipTest(String reason) throws Exception { + recordTestSkipped(applicationContext, testId, reason); + throw new AssumptionViolatedException(reason); } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java index cc273d8b9d..92a253278f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java @@ -21,6 +21,7 @@ import static androidx.media3.common.PlaybackException.ERROR_CODE_VIDEO_FRAME_PR import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.SDK_INT; import static androidx.media3.exoplayer.DefaultRenderersFactory.DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS; import static androidx.media3.exoplayer.DefaultRenderersFactory.MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY; @@ -28,6 +29,7 @@ import android.content.Context; import android.graphics.Bitmap; import android.media.MediaFormat; import android.os.Handler; +import androidx.annotation.ChecksSdkIntAtLeast; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; @@ -36,7 +38,6 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.Timeline; import androidx.media3.common.util.ConstantRateTimestampIterator; -import androidx.media3.common.util.Util; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.Renderer; import androidx.media3.exoplayer.RenderersFactory; @@ -47,6 +48,7 @@ import androidx.media3.exoplayer.image.ImageDecoder; import androidx.media3.exoplayer.image.ImageOutput; import androidx.media3.exoplayer.image.ImageRenderer; import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter; +import androidx.media3.exoplayer.mediacodec.MediaCodecInfo; import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; import androidx.media3.exoplayer.metadata.MetadataOutput; import androidx.media3.exoplayer.source.MediaSource; @@ -132,7 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; eventHandler, videoRendererEventListener, sequence, - videoSink, + new BufferingVideoSink(context), requestToneMapping)); renderers.add( new SequenceImageRenderer(sequence, checkStateNotNull(imageDecoderFactory), videoSink)); @@ -141,6 +143,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return renderers.toArray(new Renderer[0]); } + @Nullable + @Override + public Renderer createSecondaryRenderer( + Renderer renderer, + Handler eventHandler, + VideoRendererEventListener videoRendererEventListener, + AudioRendererEventListener audioRendererEventListener, + TextOutput textRendererOutput, + MetadataOutput metadataRendererOutput) { + if (isVideoPrewarmingEnabled() && renderer instanceof SequenceVideoRenderer) { + return new SequenceVideoRenderer( + context, + eventHandler, + videoRendererEventListener, + sequence, + new BufferingVideoSink(context), + requestToneMapping); + } + return null; + } + private static long getOffsetToCompositionTimeUs( EditedMediaItemSequence sequence, int mediaItemIndex, long offsetUs) { // Reverse engineer how timestamps and offsets are computed with a ConcatenatingMediaSource2 @@ -182,6 +205,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return sequence.editedMediaItems.get(index); } + @ChecksSdkIntAtLeast(api = 23) + private static boolean isVideoPrewarmingEnabled() { + return SDK_INT >= 23; + } + private static final class SequenceAudioRenderer extends MediaCodecAudioRenderer { private final EditedMediaItemSequence sequence; private final AudioGraphInputAudioSink audioSink; @@ -264,10 +292,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - private static final class SequenceVideoRenderer extends MediaCodecVideoRenderer { + private final class SequenceVideoRenderer extends MediaCodecVideoRenderer { private final EditedMediaItemSequence sequence; - private final VideoSink videoSink; + private final BufferingVideoSink bufferingVideoSink; private final boolean requestToneMapping; private ImmutableList pendingEffects; @@ -279,7 +307,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Handler eventHandler, VideoRendererEventListener videoRendererEventListener, EditedMediaItemSequence sequence, - VideoSink videoSink, + BufferingVideoSink bufferingVideoSink, boolean requestToneMapping) { super( new Builder(context) @@ -291,14 +319,39 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; .setEventListener(videoRendererEventListener) .setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY) .setAssumedMinimumCodecOperatingRate(DEFAULT_FRAME_RATE) - .setVideoSink(videoSink)); + .setVideoSink(bufferingVideoSink)); this.sequence = sequence; - this.videoSink = videoSink; + this.bufferingVideoSink = bufferingVideoSink; this.requestToneMapping = requestToneMapping; this.pendingEffects = ImmutableList.of(); experimentalEnableProcessedStreamChangedAtStart(); } + @Override + protected void onEnabled(boolean joining, boolean mayRenderStartOfStream) + throws ExoPlaybackException { + if (mayRenderStartOfStream) { + // Activate the BufferingVideoSink before calling super.onEnabled(), so that it points to a + // VideoSink when executing the super method. + activateBufferingVideoSink(); + } + super.onEnabled(joining, mayRenderStartOfStream); + } + + @Override + protected void onStarted() { + // Activate the BufferingVideoSink before calling super.onStarted(), so that it points to a + // VideoSink when executing the super method. + activateBufferingVideoSink(); + super.onStarted(); + } + + @Override + protected void onDisabled() { + super.onDisabled(); + deactivateBufferingVideoSink(); + } + @Override protected void onStreamChanged( Format[] formats, @@ -333,13 +386,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; codecOperatingRate, deviceNeedsNoPostProcessWorkaround, tunnelingAudioSessionId); - if (requestToneMapping && Util.SDK_INT >= 31) { + if (requestToneMapping && SDK_INT >= 31) { mediaFormat.setInteger( MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); } return mediaFormat; } + @Override + public void handleMessage(@MessageType int messageType, @Nullable Object message) + throws ExoPlaybackException { + if (messageType == MSG_TRANSFER_RESOURCES) { + // Ignore MSG_TRANSFER_RESOURCES to avoid updating the VideoGraph's output surface. + return; + } + super.handleMessage(messageType, message); + } + + @Override + protected boolean shouldInitCodec(MediaCodecInfo codecInfo) { + if (isVideoPrewarmingEnabled() + && bufferingVideoSink.getVideoSink() == null + && codecNeedsSetOutputSurfaceWorkaround(codecInfo.name)) { + // Wait until the BufferingVideoSink points to the effect VideoSink to init the codec, so + // that the codec output surface is set to the effect VideoSink input surface. + return false; + } + return super.shouldInitCodec(codecInfo); + } + @Override protected long getBufferTimestampAdjustmentUs() { return offsetToCompositionTimeUs; @@ -349,7 +424,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; protected void renderToEndOfStream() { super.renderToEndOfStream(); if (isLastInSequence(getTimeline(), sequence, checkNotNull(currentEditedMediaItem))) { - videoSink.signalEndOfInput(); + bufferingVideoSink.signalEndOfInput(); } } @@ -358,6 +433,42 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; VideoSink videoSink, @VideoSink.InputType int inputType, Format format) { videoSink.onInputStreamChanged(inputType, format, pendingEffects); } + + private void activateBufferingVideoSink() { + if (bufferingVideoSink.getVideoSink() != null) { + return; + } + VideoSink frameProcessingVideoSink = checkNotNull(SequenceRenderersFactory.this.videoSink); + bufferingVideoSink.setVideoSink(frameProcessingVideoSink); + @Nullable MediaCodecAdapter codec = getCodec(); + if (isVideoPrewarmingEnabled() + && frameProcessingVideoSink.isInitialized() + && codec != null + && !codecNeedsSetOutputSurfaceWorkaround(checkNotNull(getCodecInfo()).name)) { + setOutputSurfaceV23(codec, frameProcessingVideoSink.getInputSurface()); + } + } + + private void deactivateBufferingVideoSink() { + if (!isVideoPrewarmingEnabled()) { + return; + } + bufferingVideoSink.setVideoSink(null); + // During a seek, it's possible for the renderer to be disabled without having been started. + // When this happens, the BufferingVideoSink can have pending operations, so they need to be + // cleared. + bufferingVideoSink.clearPendingOperations(); + @Nullable MediaCodecAdapter codec = getCodec(); + if (codec == null) { + return; + } + if (!codecNeedsSetOutputSurfaceWorkaround(checkNotNull(getCodecInfo()).name)) { + // Sets a placeholder surface + setOutputSurfaceV23(codec, bufferingVideoSink.getInputSurface()); + } else { + releaseCodec(); + } + } } private static final class SequenceImageRenderer extends ImageRenderer {