From da99f9937db2a6fcda3f32c29bc4fdaff026a3ea Mon Sep 17 00:00:00 2001 From: tonihei Date: Tue, 18 Jul 2023 17:07:35 +0100 Subject: [PATCH] Rollback of https://github.com/androidx/media/commit/7996766b225a7df02c2b92ec695d0dfb7bcffc2b *** Original commit *** Rollback of https://github.com/androidx/media/commit/b69b33206e1a6f5f8b192d59f9f8e883cd19faf4 *** Original commit *** Mark output sample as decode-only based on start time We currently do the same check on the input timestamps and expect the output timestamps to match. Some codecs produce samples with modified timestamps and the logic is a lot safer when the comparison with the start time is done on the output side of the codec. Issue: google/ExoPlayer#11000 *** *** PiperOrigin-RevId: 549019403 --- RELEASENOTES.md | 3 + .../media3/exoplayer/BaseRenderer.java | 2 +- .../mediacodec/MediaCodecRenderer.java | 22 +- .../mediacodec/MediaCodecRendererTest.java | 106 +++++++++ .../video/MediaCodecVideoRendererTest.java | 2 +- .../playbackdumps/mp4/sample_opus.mp4.dump | 204 +++++++++--------- 6 files changed, 213 insertions(+), 126 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 5ebf5ef7ad..39f3d0f981 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -16,6 +16,9 @@ * Remove accidentally added `multidex` dependency from all modules ([#499](https://github.com/androidx/media/issues/499)). * ExoPlayer: + * Fix seeking issues in AC4 streams caused by not identifying decode-only + samples correctly + ([#11000](https://github.com/google/ExoPlayer/issues/11000)). * Add suppression of playback on unsuitable audio output devices (e.g. the built-in speaker on Wear OS devices) when this feature is enabled via `ExoPlayer.Builder.setSuppressPlaybackOnUnsuitableOutput`. The playback diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java index 12eb1004e7..2e89df6431 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java @@ -115,7 +115,7 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { state = STATE_ENABLED; onEnabled(joining, mayRenderStartOfStream); replaceStream(formats, stream, startPositionUs, offsetUs); - resetPosition(positionUs, joining); + resetPosition(startPositionUs, joining); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index e192905605..2128c584bf 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -84,7 +84,6 @@ import java.lang.annotation.Target; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.List; /** An abstract renderer that uses {@link MediaCodec} to decode samples for rendering. */ @@ -311,7 +310,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private final DecoderInputBuffer buffer; private final DecoderInputBuffer bypassSampleBuffer; private final BatchBuffer bypassBatchBuffer; - private final ArrayList decodeOnlyPresentationTimestamps; private final MediaCodec.BufferInfo outputBufferInfo; private final ArrayDeque pendingOutputStreamChanges; private final OggOpusAudioPacketizer oggOpusAudioPacketizer; @@ -412,7 +410,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); bypassSampleBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); bypassBatchBuffer = new BatchBuffer(); - decodeOnlyPresentationTimestamps = new ArrayList<>(); outputBufferInfo = new MediaCodec.BufferInfo(); currentPlaybackSpeed = 1f; targetPlaybackSpeed = 1f; @@ -933,7 +930,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { shouldSkipAdaptationWorkaroundOutputBuffer = false; isDecodeOnlyOutputBuffer = false; isLastOutputBuffer = false; - decodeOnlyPresentationTimestamps.clear(); largestQueuedPresentationTimeUs = C.TIME_UNSET; lastBufferInStreamPresentationTimeUs = C.TIME_UNSET; lastProcessedOutputBufferTimeUs = C.TIME_UNSET; @@ -1390,9 +1386,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { c2Mp3TimestampTracker.getLastOutputBufferPresentationTimeUs(inputFormat)); } - if (buffer.isDecodeOnly()) { - decodeOnlyPresentationTimestamps.add(presentationTimeUs); - } if (waitingForFirstSampleInFormat) { if (!pendingOutputStreamChanges.isEmpty()) { pendingOutputStreamChanges.peekLast().formatQueue.add(presentationTimeUs, inputFormat); @@ -1922,7 +1915,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { && largestQueuedPresentationTimeUs != C.TIME_UNSET) { outputBufferInfo.presentationTimeUs = largestQueuedPresentationTimeUs; } - isDecodeOnlyOutputBuffer = isDecodeOnlyBuffer(outputBufferInfo.presentationTimeUs); + isDecodeOnlyOutputBuffer = outputBufferInfo.presentationTimeUs < getLastResetPositionUs(); isLastOutputBuffer = lastBufferInStreamPresentationTimeUs == outputBufferInfo.presentationTimeUs; updateOutputFormatForTime(outputBufferInfo.presentationTimeUs); @@ -2213,19 +2206,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { maybeInitCodecOrBypass(); } - private boolean isDecodeOnlyBuffer(long presentationTimeUs) { - // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would - // box presentationTimeUs, creating a Long object that would need to be garbage collected. - int size = decodeOnlyPresentationTimestamps.size(); - for (int i = 0; i < size; i++) { - if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) { - decodeOnlyPresentationTimestamps.remove(i); - return true; - } - } - return false; - } - @RequiresApi(23) private void updateDrmSessionV23() throws ExoPlaybackException { CryptoConfig cryptoConfig = sourceDrmSession.getCryptoConfig(); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index 58a7f2f2c6..0050c7ade5 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -19,6 +19,9 @@ import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_YES_ import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.spy; @@ -324,6 +327,90 @@ public class MediaCodecRendererTest { inOrder.verify(renderer).onProcessedOutputBuffer(400); } + @Test + public void render_afterEnableWithStartPositionUs_skipsSamplesBeforeStartPositionUs() + throws Exception { + Format format = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(format, /* sampleTimesUs...= */ 0, 100, 200, 300, 400, 500); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 300, + /* offsetUs= */ 0); + renderer.start(); + renderer.setCurrentStreamFinal(); + long positionUs = 0; + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 0, /* isDecodeOnly= */ true); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 100, /* isDecodeOnly= */ true); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 200, /* isDecodeOnly= */ true); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 300, /* isDecodeOnly= */ false); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 400, /* isDecodeOnly= */ false); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 500, /* isDecodeOnly= */ false); + } + + @Test + public void render_afterPositionReset_skipsSamplesBeforeStartPositionUs() throws Exception { + Format format = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(format, /* sampleTimesUs...= */ 0, 100, 200, 300, 400, 500); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 400, + /* offsetUs= */ 0); + renderer.start(); + + renderer.resetPosition(/* positionUs= */ 200); + renderer.setCurrentStreamFinal(); + long positionUs = 0; + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 0, /* isDecodeOnly= */ true); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 100, /* isDecodeOnly= */ true); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 200, /* isDecodeOnly= */ false); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 300, /* isDecodeOnly= */ false); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 400, /* isDecodeOnly= */ false); + verifyProcessOutputBufferDecodeOnly( + inOrder, renderer, /* presentationTimeUs= */ 500, /* isDecodeOnly= */ false); + } + private FakeSampleStream createFakeSampleStream(Format format, long... sampleTimesUs) { ImmutableList.Builder sampleListBuilder = ImmutableList.builder(); @@ -430,4 +517,23 @@ public class MediaCodecRendererTest { /* discardReasons= */ 0); } } + + private static void verifyProcessOutputBufferDecodeOnly( + InOrder inOrder, MediaCodecRenderer renderer, long presentationTimeUs, boolean isDecodeOnly) + throws Exception { + inOrder + .verify(renderer) + .processOutputBuffer( + anyLong(), + anyLong(), + any(), + any(), + anyInt(), + anyInt(), + anyInt(), + eq(presentationTimeUs), + eq(isDecodeOnly), + anyBoolean(), + any()); + } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index c7369c2fb7..a9a67d62a2 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -258,7 +258,7 @@ public class MediaCodecVideoRendererTest { /* positionUs= */ 0, /* joining= */ false, /* mayRenderStartOfStream= */ true, - /* startPositionUs= */ 0, + /* startPositionUs= */ 30_000, /* offsetUs= */ 0); mediaCodecVideoRenderer.start(); diff --git a/libraries/test_data/src/test/assets/playbackdumps/mp4/sample_opus.mp4.dump b/libraries/test_data/src/test/assets/playbackdumps/mp4/sample_opus.mp4.dump index f41592ad83..d8a3ffaafa 100644 --- a/libraries/test_data/src/test/assets/playbackdumps/mp4/sample_opus.mp4.dump +++ b/libraries/test_data/src/test/assets/playbackdumps/mp4/sample_opus.mp4.dump @@ -715,311 +715,309 @@ MediaCodecAdapter (exotest.audio.opus): size = 0 rendered = false AudioSink: - buffer count = 101 + buffer count = 100 config: pcmEncoding = 2 channelCount = 2 sampleRate = 48000 + discontinuity: buffer #0: - time = 999999993500 - data = 1 - buffer #1: time = 1000000003500 data = 1 - buffer #2: + buffer #1: time = 1000000013500 data = 1 - buffer #3: + buffer #2: time = 1000000023500 data = 1 - buffer #4: + buffer #3: time = 1000000033500 data = 1 - buffer #5: + buffer #4: time = 1000000043500 data = 1 - buffer #6: + buffer #5: time = 1000000053500 data = 1 - buffer #7: + buffer #6: time = 1000000063500 data = 1 - buffer #8: + buffer #7: time = 1000000073500 data = 1 - buffer #9: + buffer #8: time = 1000000083500 data = 1 - buffer #10: + buffer #9: time = 1000000093500 data = 1 - buffer #11: + buffer #10: time = 1000000103500 data = 1 - buffer #12: + buffer #11: time = 1000000113500 data = 1 - buffer #13: + buffer #12: time = 1000000123500 data = 1 - buffer #14: + buffer #13: time = 1000000133500 data = 1 - buffer #15: + buffer #14: time = 1000000143500 data = 1 - buffer #16: + buffer #15: time = 1000000153500 data = 1 - buffer #17: + buffer #16: time = 1000000163500 data = 1 - buffer #18: + buffer #17: time = 1000000173500 data = 1 - buffer #19: + buffer #18: time = 1000000183500 data = 1 - buffer #20: + buffer #19: time = 1000000193500 data = 1 - buffer #21: + buffer #20: time = 1000000203500 data = 1 - buffer #22: + buffer #21: time = 1000000213500 data = 1 - buffer #23: + buffer #22: time = 1000000223500 data = 1 - buffer #24: + buffer #23: time = 1000000233500 data = 1 - buffer #25: + buffer #24: time = 1000000243500 data = 1 - buffer #26: + buffer #25: time = 1000000253500 data = 1 - buffer #27: + buffer #26: time = 1000000263500 data = 1 - buffer #28: + buffer #27: time = 1000000273500 data = 1 - buffer #29: + buffer #28: time = 1000000283500 data = 1 - buffer #30: + buffer #29: time = 1000000293500 data = 1 - buffer #31: + buffer #30: time = 1000000303500 data = 1 - buffer #32: + buffer #31: time = 1000000313500 data = 1 - buffer #33: + buffer #32: time = 1000000323500 data = 1 - buffer #34: + buffer #33: time = 1000000333500 data = 1 - buffer #35: + buffer #34: time = 1000000343500 data = 1 - buffer #36: + buffer #35: time = 1000000353500 data = 1 - buffer #37: + buffer #36: time = 1000000363500 data = 1 - buffer #38: + buffer #37: time = 1000000373500 data = 1 - buffer #39: + buffer #38: time = 1000000383500 data = 1 - buffer #40: + buffer #39: time = 1000000393500 data = 1 - buffer #41: + buffer #40: time = 1000000403500 data = 1 - buffer #42: + buffer #41: time = 1000000413500 data = 1 - buffer #43: + buffer #42: time = 1000000423500 data = 1 - buffer #44: + buffer #43: time = 1000000433500 data = 1 - buffer #45: + buffer #44: time = 1000000443500 data = 1 - buffer #46: + buffer #45: time = 1000000453500 data = 1 - buffer #47: + buffer #46: time = 1000000463500 data = 1 - buffer #48: + buffer #47: time = 1000000473500 data = 1 - buffer #49: + buffer #48: time = 1000000483500 data = 1 - buffer #50: + buffer #49: time = 1000000493500 data = 1 - buffer #51: + buffer #50: time = 1000000503500 data = 1 - buffer #52: + buffer #51: time = 1000000513499 data = 1 - buffer #53: + buffer #52: time = 1000000523499 data = 1 - buffer #54: + buffer #53: time = 1000000533500 data = 1 - buffer #55: + buffer #54: time = 1000000543500 data = 1 - buffer #56: + buffer #55: time = 1000000553500 data = 1 - buffer #57: + buffer #56: time = 1000000563500 data = 1 - buffer #58: + buffer #57: time = 1000000573500 data = 1 - buffer #59: + buffer #58: time = 1000000583500 data = 1 - buffer #60: + buffer #59: time = 1000000593500 data = 1 - buffer #61: + buffer #60: time = 1000000603500 data = 1 - buffer #62: + buffer #61: time = 1000000613500 data = 1 - buffer #63: + buffer #62: time = 1000000623500 data = 1 - buffer #64: + buffer #63: time = 1000000633500 data = 1 - buffer #65: + buffer #64: time = 1000000643500 data = 1 - buffer #66: + buffer #65: time = 1000000653500 data = 1 - buffer #67: + buffer #66: time = 1000000663500 data = 1 - buffer #68: + buffer #67: time = 1000000673500 data = 1 - buffer #69: + buffer #68: time = 1000000683500 data = 1 - buffer #70: + buffer #69: time = 1000000693500 data = 1 - buffer #71: + buffer #70: time = 1000000703500 data = 1 - buffer #72: + buffer #71: time = 1000000713500 data = 1 - buffer #73: + buffer #72: time = 1000000723500 data = 1 - buffer #74: + buffer #73: time = 1000000733500 data = 1 - buffer #75: + buffer #74: time = 1000000743500 data = 1 - buffer #76: + buffer #75: time = 1000000753500 data = 1 - buffer #77: + buffer #76: time = 1000000763500 data = 1 - buffer #78: + buffer #77: time = 1000000773500 data = 1 - buffer #79: + buffer #78: time = 1000000783500 data = 1 - buffer #80: + buffer #79: time = 1000000793500 data = 1 - buffer #81: + buffer #80: time = 1000000803500 data = 1 - buffer #82: + buffer #81: time = 1000000813500 data = 1 - buffer #83: + buffer #82: time = 1000000823500 data = 1 - buffer #84: + buffer #83: time = 1000000833500 data = 1 - buffer #85: + buffer #84: time = 1000000843500 data = 1 - buffer #86: + buffer #85: time = 1000000853500 data = 1 - buffer #87: + buffer #86: time = 1000000863500 data = 1 - buffer #88: + buffer #87: time = 1000000873500 data = 1 - buffer #89: + buffer #88: time = 1000000883500 data = 1 - buffer #90: + buffer #89: time = 1000000893500 data = 1 - buffer #91: + buffer #90: time = 1000000903500 data = 1 - buffer #92: + buffer #91: time = 1000000913500 data = 1 - buffer #93: + buffer #92: time = 1000000923500 data = 1 - buffer #94: + buffer #93: time = 1000000933500 data = 1 - buffer #95: + buffer #94: time = 1000000943500 data = 1 - buffer #96: + buffer #95: time = 1000000953500 data = 1 - buffer #97: + buffer #96: time = 1000000963500 data = 1 - buffer #98: + buffer #97: time = 1000000973500 data = 1 - buffer #99: + buffer #98: time = 1000000983500 data = 1 - buffer #100: + buffer #99: time = 1000000993500 data = 1