diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index fb02c89f30..36aae0527e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -52,6 +52,7 @@ import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.PlaybackException; +import androidx.media3.common.Timeline; import androidx.media3.common.VideoSize; import androidx.media3.common.util.Log; import androidx.media3.common.util.MediaFormatUtil; @@ -134,10 +135,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer private static final int HEVC_MAX_INPUT_SIZE_THRESHOLD = 2 * 1024 * 1024; /** The earliest time threshold, in microseconds, after which a frame is considered late. */ - private static final long MIN_EARLY_US_LATE_THRESHOLD = -30_000; + private static final long MIN_EARLY_US_LATE_THRESHOLD = -30_000L; /** The earliest time threshold, in microseconds, after which a frame is considered very late. */ - private static final long MIN_EARLY_US_VERY_LATE_THRESHOLD = -500_000; + private static final long MIN_EARLY_US_VERY_LATE_THRESHOLD = -500_000L; + + /** + * The offset from the {@link Timeline.Period} end duration in microseconds, after which input + * buffers will be treated as if they are last. + */ + private static final long OFFSET_FROM_PERIOD_END_TO_TREAT_AS_LAST_US = 100_000L; private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround; private static boolean deviceNeedsSetOutputSurfaceWorkaround; @@ -179,6 +186,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer /* package */ @Nullable OnFrameRenderedListenerV23 tunnelingOnFrameRenderedListener; @Nullable private VideoFrameMetadataListener frameMetadataListener; private long startPositionUs; + private long periodDurationUs; private boolean videoSinkNeedsRegisterInputStream; /** @@ -418,6 +426,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer reportedVideoSize = null; rendererPriority = C.PRIORITY_PLAYBACK; startPositionUs = C.TIME_UNSET; + periodDurationUs = C.TIME_UNSET; } // FrameTimingEvaluator methods @@ -732,6 +741,19 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer if (this.startPositionUs == C.TIME_UNSET) { this.startPositionUs = startPositionUs; } + updatePeriodDurationUs(mediaPeriodId); + } + + private void updatePeriodDurationUs(MediaSource.MediaPeriodId mediaPeriodId) { + Timeline timeline = getTimeline(); + if (timeline.isEmpty()) { + periodDurationUs = C.TIME_UNSET; + return; + } + periodDurationUs = + timeline + .getPeriodByUid(checkNotNull(mediaPeriodId).periodUid, new Timeline.Period()) + .getDurationUs(); } @Override @@ -813,6 +835,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @Override protected void onDisabled() { reportedVideoSize = null; + periodDurationUs = C.TIME_UNSET; if (videoSink != null) { videoSink.onRendererDisabled(); } else { @@ -1232,10 +1255,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @Override protected boolean shouldSkipDecoderInputBuffer(DecoderInputBuffer buffer) { - // TODO: b/351164714 - Do not apply this optimization for buffers with timestamp near - // the media duration. - if (hasReadStreamToEnd() || buffer.isLastSample()) { - // Last buffer is always decoded. + if (!buffer.notDependedOn()) { + // Buffer is depended on. Do not skip. + return false; + } + if (isBufferProbablyLastSample(buffer)) { + // Make sure to decode and render the last frame. return false; } if (buffer.isEncrypted()) { @@ -1244,7 +1269,21 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer return false; } // Skip buffers without sample dependencies that won't be rendered. - return isBufferBeforeStartTime(buffer) && buffer.notDependedOn(); + return isBufferBeforeStartTime(buffer); + } + + private boolean isBufferProbablyLastSample(DecoderInputBuffer buffer) { + if (hasReadStreamToEnd() || buffer.isLastSample()) { + return true; + } + // TODO: b/352276461 - improve buffer.isLastSample() logic. + // This is a temporary workaround: do not skip buffers close to the period end. + if (periodDurationUs == C.TIME_UNSET) { + // Duration unknown: probably last sample. + return true; + } + long presentationTimeUs = buffer.timeUs - getOutputStreamOffsetUs(); + return periodDurationUs - presentationTimeUs <= OFFSET_FROM_PERIOD_END_TO_TREAT_AS_LAST_US; } private boolean isBufferBeforeStartTime(DecoderInputBuffer buffer) { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java index ed2ea706c2..c51e45e178 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java @@ -15205,6 +15205,35 @@ public class ExoPlayerTest { .isSameInstanceAs(mediaItem2); } + @Test + public void changeMediaItemMidPlayback_preparesSuccessfully() throws TimeoutException { + MediaItem mediaItem1 = new MediaItem.Builder().setUri(SAMPLE_URI).setMediaId("1").build(); + MediaItem mediaItem2 = new MediaItem.Builder().setUri(SAMPLE_URI).setMediaId("2").build(); + ExoPlayer player = + parameterizeTestExoPlayerBuilder( + new TestExoPlayerBuilder(context) + .setRenderersFactory(new DefaultRenderersFactory(context))) + .build(); + Player.Listener listener = mock(Player.Listener.class); + player.addListener(listener); + player.addMediaItem(mediaItem1); + player.prepare(); + runUntilPlaybackState(player, Player.STATE_READY); + player.setMediaItem(mediaItem2); + player.prepare(); + runUntilPlaybackState(player, Player.STATE_READY); + player.release(); + + verify(listener) + .onMediaItemTransition( + eq(mediaItem1), eq(Player.MEDIA_ITEM_TRANSITION_REASON_PLAYLIST_CHANGED)); + verify(listener) + .onMediaItemTransition( + eq(mediaItem2), eq(Player.MEDIA_ITEM_TRANSITION_REASON_PLAYLIST_CHANGED)); + verify(listener, times(2)).onPlaybackStateChanged(Player.STATE_BUFFERING); + verify(listener, times(2)).onPlaybackStateChanged(Player.STATE_READY); + } + @Test public void silenceSkipped_playerEmitOnPositionDiscontinuity() throws Exception { Timeline timeline = diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index e1ef97b6b1..82cd1c9482 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Util.msToUs; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.format; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static androidx.media3.test.utils.FakeTimeline.TimelineWindowDefinition.DEFAULT_WINDOW_OFFSET_IN_FIRST_PERIOD_US; import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; @@ -76,6 +77,7 @@ import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.DefaultAllocator; import androidx.media3.test.utils.FakeMediaPeriod; import androidx.media3.test.utils.FakeSampleStream; +import androidx.media3.test.utils.FakeTimeline; import androidx.media3.test.utils.robolectric.RobolectricUtil; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; @@ -389,6 +391,8 @@ public class MediaCodecVideoRendererTest { /* maxDroppedFramesToNotify= */ 1); mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + FakeTimeline fakeTimeline = new FakeTimeline(); + mediaCodecVideoRenderer.setTimeline(fakeTimeline); mediaCodecVideoRenderer.enable( RendererConfiguration.DEFAULT, new Format[] {VIDEO_H264}, @@ -398,12 +402,12 @@ public class MediaCodecVideoRendererTest { /* mayRenderStartOfStream= */ true, /* startPositionUs= */ 30_000, /* offsetUs= */ 0, - new MediaSource.MediaPeriodId(new Object())); + new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0))); mediaCodecVideoRenderer.start(); mediaCodecVideoRenderer.setCurrentStreamFinal(); mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); - // Call to render has reads all samples including the END_OF_STREAM_ITEM because the + // Call to render has read all samples including the END_OF_STREAM_ITEM because the // previous sample is skipped before decoding. assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue(); int posUs = 30_000; @@ -423,6 +427,76 @@ public class MediaCodecVideoRendererTest { assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1); } + @Test + public void render_withoutSampleDependenciesAndShortDuration_skipsNoDecoderInputBuffers() + throws Exception { + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + FakeTimeline fakeTimeline = + new FakeTimeline( + new FakeTimeline.TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 30_000)); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample( + /* timeUs= */ DEFAULT_WINDOW_OFFSET_IN_FIRST_PERIOD_US, + C.BUFFER_FLAG_KEY_FRAME), + oneByteSample( + /* timeUs= */ DEFAULT_WINDOW_OFFSET_IN_FIRST_PERIOD_US + 10_000, + C.BUFFER_FLAG_NOT_DEPENDED_ON), + oneByteSample( + /* timeUs= */ DEFAULT_WINDOW_OFFSET_IN_FIRST_PERIOD_US + 20_000, + C.BUFFER_FLAG_NOT_DEPENDED_ON), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + // Seek to time after samples. + fakeSampleStream.seekToUs(30_000, /* allowTimeBeyondBuffer= */ true); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + ApplicationProvider.getApplicationContext(), + new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 5), + mediaCodecSelector, + /* allowedJoiningTimeMs= */ 0, + /* enableDecoderFallback= */ false, + /* eventHandler= */ new Handler(testMainLooper), + /* eventListener= */ eventListener, + /* maxDroppedFramesToNotify= */ 1); + mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.setTimeline(fakeTimeline); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ DEFAULT_WINDOW_OFFSET_IN_FIRST_PERIOD_US + 30_000, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0))); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + // Call to render has read all samples including the END_OF_STREAM_ITEM. + assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue(); + int posUs = 30_000; + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + assertThat(argumentDecoderCounters.getValue().skippedInputBufferCount).isEqualTo(0); + } + @Test public void render_withClippingMediaPeriodAndBufferContainingLastAndClippingSamples_rendersLastFrame() @@ -1820,7 +1894,7 @@ public class MediaCodecVideoRendererTest { @Override public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) { int outputIndex = super.dequeueOutputBufferIndex(bufferInfo); - if (outputIndex > 0) { + if (outputIndex >= 0) { bufferCounter++; } return outputIndex; diff --git a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java index 2de304b61c..f34d6aa896 100644 --- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java +++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java @@ -16,6 +16,7 @@ package androidx.media3.exoplayer.dash.e2etest; import static androidx.media3.common.util.Assertions.checkNotNull; +import static com.google.common.truth.Truth.assertThat; import android.content.Context; import android.graphics.SurfaceTexture; @@ -24,6 +25,7 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.Player; import androidx.media3.datasource.DataSource; import androidx.media3.datasource.DefaultDataSource; +import androidx.media3.exoplayer.DecoderCounters; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.Renderer; import androidx.media3.exoplayer.RenderersFactory; @@ -402,4 +404,40 @@ public final class DashPlaybackTest { DumpFileAsserts.assertOutput( applicationContext, playbackOutput, "playbackdumps/dash/optimized_seek.dump"); } + + @Test + public void playVideo_usingWithinGopSampleDependencies_withSeekAfterEoS() throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + CapturingRenderersFactory capturingRenderersFactory = + new CapturingRenderersFactory(applicationContext); + BundledChunkExtractor.Factory chunkExtractorFactory = + new BundledChunkExtractor.Factory().experimentalParseWithinGopSampleDependencies(true); + DataSource.Factory defaultDataSourceFactory = new DefaultDataSource.Factory(applicationContext); + DashMediaSource.Factory dashMediaSourceFactory = + new DashMediaSource.Factory( + /* chunkSourceFactory= */ new DefaultDashChunkSource.Factory( + chunkExtractorFactory, defaultDataSourceFactory, /* maxSegmentsPerLoad= */ 1), + /* manifestDataSourceFactory= */ defaultDataSourceFactory); + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, capturingRenderersFactory) + .setMediaSourceFactory(dashMediaSourceFactory) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .build(); + Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1)); + player.setVideoSurface(surface); + + player.setMediaItem(MediaItem.fromUri("asset:///media/dash/standalone-webvtt/sample.mpd")); + player.seekTo(50_000L); + player.prepare(); + player.play(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + player.release(); + surface.release(); + + DecoderCounters decoderCounters = checkNotNull(player.getVideoDecoderCounters()); + assertThat(decoderCounters.skippedInputBufferCount).isEqualTo(13); + assertThat(decoderCounters.queuedInputBufferCount).isEqualTo(17); + // TODO: b/352276461 - The last frame might not be rendered. When the bug is fixed, + // assert on the full playback dump. + } }