From 10dcdd1df5f04311f6ae06d53830437cfa4d24d5 Mon Sep 17 00:00:00 2001 From: tonihei Date: Mon, 1 Nov 2021 10:23:58 +0000 Subject: [PATCH] Add large renderer position offset. This helps to prevent issues where decoders can't handle negative timestamps. In particular it avoids issues when the media accidentally or intentionally starts with small negative timestamps. But it also helps to prevent other renderer resets at a later point, for example if a live stream with a large start offset is enqueued in the playlist. #minor-release PiperOrigin-RevId: 406786977 --- .../exoplayer/ExoPlayerImplInternal.java | 9 +-- .../media3/exoplayer/MediaPeriodQueue.java | 24 +++++++- .../exoplayer/MediaPeriodQueueTest.java | 47 ++++++++++----- .../mka/bear-flac-16bit.mka.audiosink.dump | 58 +++++++++---------- .../mka/bear-flac-24bit.mka.audiosink.dump | 58 +++++++++---------- .../transformer/TransformerAudioRenderer.java | 1 + .../transformer/TransformerBaseRenderer.java | 7 +++ .../TransformerMuxingVideoRenderer.java | 1 + .../TransformerTranscodingVideoRenderer.java | 1 + 9 files changed, 127 insertions(+), 79 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java index 012e4caefe..c347d10a07 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java @@ -1274,7 +1274,8 @@ import java.util.concurrent.atomic.AtomicBoolean; queue.advancePlayingPeriod(); } queue.removeAfter(newPlayingPeriodHolder); - newPlayingPeriodHolder.setRendererOffset(/* rendererPositionOffsetUs= */ 0); + newPlayingPeriodHolder.setRendererOffset( + MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US); enableRenderers(); } } @@ -1307,7 +1308,7 @@ import java.util.concurrent.atomic.AtomicBoolean; MediaPeriodHolder playingMediaPeriod = queue.getPlayingPeriod(); rendererPositionUs = playingMediaPeriod == null - ? periodPositionUs + ? MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US + periodPositionUs : playingMediaPeriod.toRendererTime(periodPositionUs); mediaClock.resetPosition(rendererPositionUs); for (Renderer renderer : renderers) { @@ -1383,7 +1384,7 @@ import java.util.concurrent.atomic.AtomicBoolean; pendingRecoverableRendererError = null; isRebuffering = false; mediaClock.stop(); - rendererPositionUs = 0; + rendererPositionUs = MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US; for (Renderer renderer : renderers) { try { disableRenderer(renderer); @@ -1971,7 +1972,7 @@ import java.util.concurrent.atomic.AtomicBoolean; emptyTrackSelectorResult); mediaPeriodHolder.mediaPeriod.prepare(this, info.startPositionUs); if (queue.getPlayingPeriod() == mediaPeriodHolder) { - resetRendererPosition(mediaPeriodHolder.getStartPositionRendererTime()); + resetRendererPosition(info.startPositionUs); } handleLoadingMediaPeriodChanged(/* loadingTrackSelectionChanged= */ false); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MediaPeriodQueue.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MediaPeriodQueue.java index dcd2ecdc26..0475ebc725 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MediaPeriodQueue.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MediaPeriodQueue.java @@ -39,6 +39,26 @@ import com.google.common.collect.ImmutableList; */ /* package */ final class MediaPeriodQueue { + /** + * Initial renderer position offset used for the first item in the queue, in microseconds. + * + *

Choosing a positive value, larger than any reasonable single media duration, ensures three + * things: + * + *

+ */ + public static final long INITIAL_RENDERER_POSITION_OFFSET_US = 1_000_000_000_000L; + /** * Limits the maximum number of periods to buffer ahead of the current playing period. The * buffering policy normally prevents buffering too far ahead, but the policy could allow too many @@ -165,9 +185,7 @@ import com.google.common.collect.ImmutableList; TrackSelectorResult emptyTrackSelectorResult) { long rendererPositionOffsetUs = loading == null - ? (info.id.isAd() && info.requestedContentPositionUs != C.TIME_UNSET - ? info.requestedContentPositionUs - : 0) + ? INITIAL_RENDERER_POSITION_OFFSET_US : (loading.getRendererOffset() + loading.info.durationUs - info.startPositionUs); MediaPeriodHolder newPeriodHolder = new MediaPeriodHolder( diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MediaPeriodQueueTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MediaPeriodQueueTest.java index 200c57a138..98e766a37e 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MediaPeriodQueueTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MediaPeriodQueueTest.java @@ -499,10 +499,13 @@ public final class MediaPeriodQueueTest { // Change position of first ad (= change duration of playing content before first ad). updateAdPlaybackStateAndTimeline(/* adGroupTimesUs...= */ FIRST_AD_START_TIME_US - 2000); setAdGroupLoaded(/* adGroupIndex= */ 0); - long maxRendererReadPositionUs = FIRST_AD_START_TIME_US - 3000; + long maxRendererReadPositionUs = + MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US + FIRST_AD_START_TIME_US - 3000; boolean changeHandled = mediaPeriodQueue.updateQueuedPeriods( - playbackInfo.timeline, /* rendererPositionUs= */ 0, maxRendererReadPositionUs); + playbackInfo.timeline, + /* rendererPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US, + maxRendererReadPositionUs); assertThat(changeHandled).isTrue(); assertThat(getQueueLength()).isEqualTo(1); @@ -524,10 +527,13 @@ public final class MediaPeriodQueueTest { // Change position of first ad (= change duration of playing content before first ad). updateAdPlaybackStateAndTimeline(/* adGroupTimesUs...= */ FIRST_AD_START_TIME_US - 2000); setAdGroupLoaded(/* adGroupIndex= */ 0); - long maxRendererReadPositionUs = FIRST_AD_START_TIME_US - 1000; + long maxRendererReadPositionUs = + MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US + FIRST_AD_START_TIME_US - 1000; boolean changeHandled = mediaPeriodQueue.updateQueuedPeriods( - playbackInfo.timeline, /* rendererPositionUs= */ 0, maxRendererReadPositionUs); + playbackInfo.timeline, + /* rendererPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US, + maxRendererReadPositionUs); assertThat(changeHandled).isFalse(); assertThat(getQueueLength()).isEqualTo(1); @@ -558,10 +564,13 @@ public final class MediaPeriodQueueTest { .withIsServerSideInserted(/* adGroupIndex= */ 0, /* isServerSideInserted= */ true); updateTimeline(); setAdGroupLoaded(/* adGroupIndex= */ 0); - long maxRendererReadPositionUs = FIRST_AD_START_TIME_US - 1000; + long maxRendererReadPositionUs = + MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US + FIRST_AD_START_TIME_US - 1000; boolean changeHandled = mediaPeriodQueue.updateQueuedPeriods( - playbackInfo.timeline, /* rendererPositionUs= */ 0, maxRendererReadPositionUs); + playbackInfo.timeline, + /* rendererPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US, + maxRendererReadPositionUs); assertThat(changeHandled).isTrue(); assertThat(getQueueLength()).isEqualTo(1); @@ -589,7 +598,9 @@ public final class MediaPeriodQueueTest { setAdGroupLoaded(/* adGroupIndex= */ 1); boolean changeHandled = mediaPeriodQueue.updateQueuedPeriods( - playbackInfo.timeline, /* rendererPositionUs= */ 0, /* maxRendererReadPositionUs= */ 0); + playbackInfo.timeline, + /* rendererPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US, + /* maxRendererReadPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US); assertThat(changeHandled).isTrue(); assertThat(getQueueLength()).isEqualTo(3); @@ -614,11 +625,13 @@ public final class MediaPeriodQueueTest { /* adGroupTimesUs...= */ FIRST_AD_START_TIME_US, SECOND_AD_START_TIME_US - 1000); setAdGroupLoaded(/* adGroupIndex= */ 0); setAdGroupLoaded(/* adGroupIndex= */ 1); + long maxRendererReadPositionUs = + MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US + FIRST_AD_START_TIME_US; boolean changeHandled = mediaPeriodQueue.updateQueuedPeriods( playbackInfo.timeline, - /* rendererPositionUs= */ 0, - /* maxRendererReadPositionUs= */ FIRST_AD_START_TIME_US); + /* rendererPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US, + maxRendererReadPositionUs); assertThat(changeHandled).isFalse(); assertThat(getQueueLength()).isEqualTo(3); @@ -642,11 +655,14 @@ public final class MediaPeriodQueueTest { /* adGroupTimesUs...= */ FIRST_AD_START_TIME_US, SECOND_AD_START_TIME_US - 1000); setAdGroupLoaded(/* adGroupIndex= */ 0); setAdGroupLoaded(/* adGroupIndex= */ 1); - long readingPositionAtStartOfContentBetweenAds = FIRST_AD_START_TIME_US + AD_DURATION_US; + long readingPositionAtStartOfContentBetweenAds = + MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US + + FIRST_AD_START_TIME_US + + AD_DURATION_US; boolean changeHandled = mediaPeriodQueue.updateQueuedPeriods( playbackInfo.timeline, - /* rendererPositionUs= */ 0, + /* rendererPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US, /* maxRendererReadPositionUs= */ readingPositionAtStartOfContentBetweenAds); assertThat(changeHandled).isTrue(); @@ -671,11 +687,14 @@ public final class MediaPeriodQueueTest { /* adGroupTimesUs...= */ FIRST_AD_START_TIME_US, SECOND_AD_START_TIME_US - 1000); setAdGroupLoaded(/* adGroupIndex= */ 0); setAdGroupLoaded(/* adGroupIndex= */ 1); - long readingPositionAtEndOfContentBetweenAds = SECOND_AD_START_TIME_US + AD_DURATION_US; + long readingPositionAtEndOfContentBetweenAds = + MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US + + SECOND_AD_START_TIME_US + + AD_DURATION_US; boolean changeHandled = mediaPeriodQueue.updateQueuedPeriods( playbackInfo.timeline, - /* rendererPositionUs= */ 0, + /* rendererPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US, /* maxRendererReadPositionUs= */ readingPositionAtEndOfContentBetweenAds); assertThat(changeHandled).isFalse(); @@ -703,7 +722,7 @@ public final class MediaPeriodQueueTest { boolean changeHandled = mediaPeriodQueue.updateQueuedPeriods( playbackInfo.timeline, - /* rendererPositionUs= */ 0, + /* rendererPositionUs= */ MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US, /* maxRendererReadPositionUs= */ C.TIME_END_OF_SOURCE); assertThat(changeHandled).isFalse(); diff --git a/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-16bit.mka.audiosink.dump b/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-16bit.mka.audiosink.dump index 044cde306c..b7319b872b 100644 --- a/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-16bit.mka.audiosink.dump +++ b/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-16bit.mka.audiosink.dump @@ -3,89 +3,89 @@ config: channelCount = 2 sampleRate = 48000 buffer: - time = 1000 + time = 1000000001000 data = 1217833679 buffer: - time = 97000 + time = 1000000097000 data = 558614672 buffer: - time = 193000 + time = 1000000193000 data = -709714787 buffer: - time = 289000 + time = 1000000289000 data = 1367870571 buffer: - time = 385000 + time = 1000000385000 data = -141229457 buffer: - time = 481000 + time = 1000000481000 data = 1287758361 buffer: - time = 577000 + time = 1000000577000 data = 1125289147 buffer: - time = 673000 + time = 1000000673000 data = -1677383475 buffer: - time = 769000 + time = 1000000769000 data = 2130742861 buffer: - time = 865000 + time = 1000000865000 data = -1292320253 buffer: - time = 961000 + time = 1000000961000 data = -456587163 buffer: - time = 1057000 + time = 1000001057000 data = 748981534 buffer: - time = 1153000 + time = 1000001153000 data = 1550456016 buffer: - time = 1249000 + time = 1000001249000 data = 1657906039 buffer: - time = 1345000 + time = 1000001345000 data = -762677083 buffer: - time = 1441000 + time = 1000001441000 data = -1343810763 buffer: - time = 1537000 + time = 1000001537000 data = 1137318783 buffer: - time = 1633000 + time = 1000001633000 data = -1891318229 buffer: - time = 1729000 + time = 1000001729000 data = -472068495 buffer: - time = 1825000 + time = 1000001825000 data = 832315001 buffer: - time = 1921000 + time = 1000001921000 data = 2054935175 buffer: - time = 2017000 + time = 1000002017000 data = 57921641 buffer: - time = 2113000 + time = 1000002113000 data = 2132759067 buffer: - time = 2209000 + time = 1000002209000 data = -1742540521 buffer: - time = 2305000 + time = 1000002305000 data = 1657024301 buffer: - time = 2401000 + time = 1000002401000 data = -585080145 buffer: - time = 2497000 + time = 1000002497000 data = 427271397 buffer: - time = 2593000 + time = 1000002593000 data = -364201340 buffer: - time = 2689000 + time = 1000002689000 data = -627965287 diff --git a/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-24bit.mka.audiosink.dump b/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-24bit.mka.audiosink.dump index 319ee311f0..f425e7e2f2 100644 --- a/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-24bit.mka.audiosink.dump +++ b/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-24bit.mka.audiosink.dump @@ -3,89 +3,89 @@ config: channelCount = 2 sampleRate = 48000 buffer: - time = 0 + time = 1000000000000 data = 225023649 buffer: - time = 96000 + time = 1000000096000 data = 455106306 buffer: - time = 192000 + time = 1000000192000 data = 2025727297 buffer: - time = 288000 + time = 1000000288000 data = 758514657 buffer: - time = 384000 + time = 1000000384000 data = 1044986473 buffer: - time = 480000 + time = 1000000480000 data = -2030029695 buffer: - time = 576000 + time = 1000000576000 data = 1907053281 buffer: - time = 672000 + time = 1000000672000 data = -1974954431 buffer: - time = 768000 + time = 1000000768000 data = -206248383 buffer: - time = 864000 + time = 1000000864000 data = 1484984417 buffer: - time = 960000 + time = 1000000960000 data = -1306117439 buffer: - time = 1056000 + time = 1000001056000 data = 692829792 buffer: - time = 1152000 + time = 1000001152000 data = 1070563058 buffer: - time = 1248000 + time = 1000001248000 data = -1444096479 buffer: - time = 1344000 + time = 1000001344000 data = 1753016419 buffer: - time = 1440000 + time = 1000001440000 data = 1947797953 buffer: - time = 1536000 + time = 1000001536000 data = 266121411 buffer: - time = 1632000 + time = 1000001632000 data = 1275494369 buffer: - time = 1728000 + time = 1000001728000 data = 372077825 buffer: - time = 1824000 + time = 1000001824000 data = -993079679 buffer: - time = 1920000 + time = 1000001920000 data = 177307937 buffer: - time = 2016000 + time = 1000002016000 data = 2037083009 buffer: - time = 2112000 + time = 1000002112000 data = -435776287 buffer: - time = 2208000 + time = 1000002208000 data = 1867447329 buffer: - time = 2304000 + time = 1000002304000 data = 1884495937 buffer: - time = 2400000 + time = 1000002400000 data = -804673375 buffer: - time = 2496000 + time = 1000002496000 data = -588531007 buffer: - time = 2592000 + time = 1000002592000 data = -1064642970 buffer: - time = 2688000 + time = 1000002688000 data = -1771406207 diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java index d03ea15c7b..0f6f1089d1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java @@ -279,6 +279,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; int result = readSource(getFormatHolder(), decoderInputBuffer, /* readFlags= */ 0); switch (result) { case C.RESULT_BUFFER_READ: + decoderInputBuffer.timeUs -= streamOffsetUs; mediaClock.updateTimeForTrackType(getTrackType(), decoderInputBuffer.timeUs); decoderInputBuffer.flip(); decoder.queueInputBuffer(decoderInputBuffer); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java index 1b41678d37..00cf5f7560 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java @@ -34,6 +34,7 @@ import androidx.media3.exoplayer.RendererCapabilities; protected final Transformation transformation; protected boolean isRendererStarted; + protected long streamOffsetUs; public TransformerBaseRenderer( int trackType, @@ -46,6 +47,12 @@ import androidx.media3.exoplayer.RendererCapabilities; this.transformation = transformation; } + @Override + protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) + throws ExoPlaybackException { + this.streamOffsetUs = offsetUs; + } + @Override @C.FormatSupport public final int supportsFormat(Format format) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMuxingVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMuxingVideoRenderer.java index fba19a4052..2ecf731d36 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMuxingVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMuxingVideoRenderer.java @@ -117,6 +117,7 @@ import java.nio.ByteBuffer; muxerWrapper.endTrack(getTrackType()); return false; } + buffer.timeUs -= streamOffsetUs; mediaClock.updateTimeForTrackType(getTrackType(), buffer.timeUs); ByteBuffer data = checkNotNull(buffer.data); data.flip(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java index 250887f389..61998b2366 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java @@ -320,6 +320,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; case C.RESULT_FORMAT_READ: throw new IllegalStateException("Format changes are not supported."); case C.RESULT_BUFFER_READ: + decoderInputBuffer.timeUs -= streamOffsetUs; mediaClock.updateTimeForTrackType(getTrackType(), decoderInputBuffer.timeUs); ByteBuffer data = checkNotNull(decoderInputBuffer.data); data.flip();