From 5822d683ead77f2f039c972d2492de0a5d78a8d7 Mon Sep 17 00:00:00 2001 From: tonihei Date: Mon, 27 Feb 2023 12:04:59 +0000 Subject: [PATCH] Ensure output format is updated in sync with stream changes. MediaCodecRenderer currently has two independent paths to trigger events at stream changes: 1. Detection of the last output buffer of the old stream to trigger onProcessedStreamChange and setting the new output stream offset. 2. Detection of the first input buffer of the new stream to trigger onOutputFormatChanged. Both events are identical for most media. However, there are two problematic cases: A. (1) happens after (2). This may happen if the declared media duration is shorter than the actual last sample timestamp. B. (2) is too late and there are output samples between (1) and (2). This can happen if the new media outputs samples with a timestamp less than the first input timestamp. This can be made more robust by: - Keeping a separate formatQueue for each stream to avoid case A. - Force outputting the first format after a stream change to avoid case B. Issue: google/ExoPlayer#8594 PiperOrigin-RevId: 512586838 (cherry picked from commit 3970343846d7bae5d8ae331d74241c50777ce18a) --- RELEASENOTES.md | 6 +- .../mediacodec/MediaCodecRenderer.java | 33 +- .../mediacodec/MediaCodecRendererTest.java | 323 ++++++++++++++++++ 3 files changed, 348 insertions(+), 14 deletions(-) create mode 100644 libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 62210b405d..ff4dbf6f14 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -27,10 +27,12 @@ `AudioProcessors` are active, e.g. for gapless trimming ([#10847](https://github.com/google/ExoPlayer/issues/10847)). * Encapsulate Opus frames in Ogg packets in direct playbacks (offload). -* Video: + * Fix broken gapless MP3 playback on Samsung devices + ([#8594](https://github.com/google/ExoPlayer/issues/8594)). +* Video: * Map HEVC HDR10 format to `HEVCProfileMain10HDR10` instead of `HEVCProfileMain10`. -* Add workaround for a device issue on Chromecast with Google TV and + * Add workaround for a device issue on Chromecast with Google TV and Lenovo M10 FHD Plus that causes 60fps AVC streams to be marked as unsupported ([#10898](https://github.com/google/ExoPlayer/issues/10898)). diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index cad2720ff2..a752432fa0 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -299,7 +299,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private final DecoderInputBuffer buffer; private final DecoderInputBuffer bypassSampleBuffer; private final BatchBuffer bypassBatchBuffer; - private final TimedValueQueue formatQueue; private final ArrayList decodeOnlyPresentationTimestamps; private final MediaCodec.BufferInfo outputBufferInfo; private final ArrayDeque pendingOutputStreamChanges; @@ -359,6 +358,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected DecoderCounters decoderCounters; private OutputStreamInfo outputStreamInfo; private long lastProcessedOutputBufferTimeUs; + private boolean needToNotifyOutputFormatChangeAfterStreamChange; /** * @param trackType The {@link C.TrackType track type} that the renderer handles. @@ -385,7 +385,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); bypassSampleBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); bypassBatchBuffer = new BatchBuffer(); - formatQueue = new TimedValueQueue<>(); decodeOnlyPresentationTimestamps = new ArrayList<>(); outputBufferInfo = new MediaCodec.BufferInfo(); currentPlaybackSpeed = 1f; @@ -597,13 +596,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected final void updateOutputFormatForTime(long presentationTimeUs) throws ExoPlaybackException { boolean outputFormatChanged = false; - @Nullable Format format = formatQueue.pollFloor(presentationTimeUs); - if (format == null && codecOutputMediaFormatChanged) { - // If the codec's output MediaFormat has changed then there should be a corresponding Format - // change, which we've not found. Check the Format queue in case the corresponding - // presentation timestamp is greater than presentationTimeUs, which can happen for some codecs - // [Internal ref: b/162719047]. - format = formatQueue.pollFirst(); + @Nullable Format format = outputStreamInfo.formatQueue.pollFloor(presentationTimeUs); + if (format == null + && needToNotifyOutputFormatChangeAfterStreamChange + && codecOutputMediaFormat != null) { + // After a stream change or after the initial start, there should be an input format change, + // which we've not found. Check the Format queue in case the corresponding presentation + // timestamp is greater than presentationTimeUs, which can happen for some codecs + // [Internal ref: b/162719047 and https://github.com/google/ExoPlayer/issues/8594]. + format = outputStreamInfo.formatQueue.pollFirst(); } if (format != null) { outputFormat = format; @@ -612,6 +613,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { if (outputFormatChanged || (codecOutputMediaFormatChanged && outputFormat != null)) { onOutputFormatChanged(outputFormat, codecOutputMediaFormat); codecOutputMediaFormatChanged = false; + needToNotifyOutputFormatChangeAfterStreamChange = false; } } @@ -668,10 +670,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer { // If there is a format change on the input side still pending propagation to the output, we // need to queue a format next time a buffer is read. This is because we may not read a new // input format after the position reset. - if (formatQueue.size() > 0) { + if (outputStreamInfo.formatQueue.size() > 0) { waitingForFirstSampleInFormat = true; } - formatQueue.clear(); + outputStreamInfo.formatQueue.clear(); pendingOutputStreamChanges.clear(); } @@ -1333,7 +1335,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer { decodeOnlyPresentationTimestamps.add(presentationTimeUs); } if (waitingForFirstSampleInFormat) { - formatQueue.add(presentationTimeUs, inputFormat); + if (!pendingOutputStreamChanges.isEmpty()) { + pendingOutputStreamChanges.peekLast().formatQueue.add(presentationTimeUs, inputFormat); + } else { + outputStreamInfo.formatQueue.add(presentationTimeUs, inputFormat); + } waitingForFirstSampleInFormat = false; } largestQueuedPresentationTimeUs = max(largestQueuedPresentationTimeUs, presentationTimeUs); @@ -2029,6 +2035,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private void setOutputStreamInfo(OutputStreamInfo outputStreamInfo) { this.outputStreamInfo = outputStreamInfo; if (outputStreamInfo.streamOffsetUs != C.TIME_UNSET) { + needToNotifyOutputFormatChangeAfterStreamChange = true; onOutputStreamOffsetUsChanged(outputStreamInfo.streamOffsetUs); } } @@ -2487,12 +2494,14 @@ public abstract class MediaCodecRenderer extends BaseRenderer { public final long previousStreamLastBufferTimeUs; public final long startPositionUs; public final long streamOffsetUs; + public final TimedValueQueue formatQueue; public OutputStreamInfo( long previousStreamLastBufferTimeUs, long startPositionUs, long streamOffsetUs) { this.previousStreamLastBufferTimeUs = previousStreamLastBufferTimeUs; this.startPositionUs = startPositionUs; this.streamOffsetUs = streamOffsetUs; + this.formatQueue = new TimedValueQueue<>(); } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java new file mode 100644 index 0000000000..ac86669ee5 --- /dev/null +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -0,0 +1,323 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.mediacodec; + +import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.spy; + +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.SystemClock; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.exoplayer.DecoderReuseEvaluation; +import androidx.media3.exoplayer.ExoPlaybackException; +import androidx.media3.exoplayer.RendererCapabilities; +import androidx.media3.exoplayer.RendererConfiguration; +import androidx.media3.exoplayer.analytics.PlayerId; +import androidx.media3.exoplayer.drm.DrmSessionEventListener; +import androidx.media3.exoplayer.drm.DrmSessionManager; +import androidx.media3.exoplayer.upstream.DefaultAllocator; +import androidx.media3.test.utils.FakeSampleStream; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; + +/** Unit tests for {@link MediaCodecRenderer} */ +@RunWith(AndroidJUnit4.class) +public class MediaCodecRendererTest { + + @Test + public void render_withReplaceStream_triggersOutputCallbacksInCorrectOrder() throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 400); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(400); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + @Test + public void + render_withReplaceStreamAndBufferBeyondDuration_triggersOutputCallbacksInCorrectOrder() + throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300, 400, 500, 600); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 400); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(400); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + @Test + public void + render_withReplaceStreamAndBufferLessThanStartPosition_triggersOutputCallbacksInCorrectOrder() + throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200, 300, 400); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 200); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(200); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + private FakeSampleStream createFakeSampleStream(Format format, long... sampleTimesUs) { + ImmutableList.Builder sampleListBuilder = + ImmutableList.builder(); + for (long sampleTimeUs : sampleTimesUs) { + sampleListBuilder.add(oneByteSample(sampleTimeUs, C.BUFFER_FLAG_KEY_FRAME)); + } + sampleListBuilder.add(END_OF_STREAM_ITEM); + FakeSampleStream sampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ format, + sampleListBuilder.build()); + sampleStream.writeData(/* startPositionUs= */ 0); + return sampleStream; + } + + private static class TestRenderer extends MediaCodecRenderer { + + public TestRenderer() { + super( + C.TRACK_TYPE_AUDIO, + MediaCodecAdapter.Factory.DEFAULT, + /* mediaCodecSelector= */ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> + Collections.singletonList( + MediaCodecInfo.newInstance( + /* name= */ "name", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ null, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)), + /* enableDecoderFallback= */ false, + /* assumedMinimumCodecOperatingRate= */ 44100); + } + + @Override + public String getName() { + return "test"; + } + + @Override + protected @Capabilities int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format) + throws MediaCodecUtil.DecoderQueryException { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected List getDecoderInfos( + MediaCodecSelector mediaCodecSelector, Format format, boolean requiresSecureDecoder) + throws MediaCodecUtil.DecoderQueryException { + return mediaCodecSelector.getDecoderInfos( + format.sampleMimeType, + /* requiresSecureDecoder= */ false, + /* requiresTunnelingDecoder= */ false); + } + + @Override + protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( + MediaCodecInfo codecInfo, + Format format, + @Nullable MediaCrypto crypto, + float codecOperatingRate) { + return MediaCodecAdapter.Configuration.createForAudioDecoding( + codecInfo, new MediaFormat(), format, crypto); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + @Nullable MediaCodecAdapter codec, + @Nullable ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + int sampleCount, + long bufferPresentationTimeUs, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, + Format format) + throws ExoPlaybackException { + if (bufferPresentationTimeUs <= positionUs) { + // Only release buffers when the position advances far enough for realistic behavior where + // input of buffers to the codec is faster than output. + codec.releaseOutputBuffer(bufferIndex, /* render= */ true); + return true; + } + return false; + } + + @Override + protected DecoderReuseEvaluation canReuseCodec( + MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) { + return new DecoderReuseEvaluation( + codecInfo.name, + oldFormat, + newFormat, + REUSE_RESULT_YES_WITHOUT_RECONFIGURATION, + /* discardReasons= */ 0); + } + } +}