diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 62210b405d..ff4dbf6f14 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -27,10 +27,12 @@ `AudioProcessors` are active, e.g. for gapless trimming ([#10847](https://github.com/google/ExoPlayer/issues/10847)). * Encapsulate Opus frames in Ogg packets in direct playbacks (offload). -* Video: + * Fix broken gapless MP3 playback on Samsung devices + ([#8594](https://github.com/google/ExoPlayer/issues/8594)). +* Video: * Map HEVC HDR10 format to `HEVCProfileMain10HDR10` instead of `HEVCProfileMain10`. -* Add workaround for a device issue on Chromecast with Google TV and + * Add workaround for a device issue on Chromecast with Google TV and Lenovo M10 FHD Plus that causes 60fps AVC streams to be marked as unsupported ([#10898](https://github.com/google/ExoPlayer/issues/10898)). diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index cad2720ff2..a752432fa0 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -299,7 +299,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private final DecoderInputBuffer buffer; private final DecoderInputBuffer bypassSampleBuffer; private final BatchBuffer bypassBatchBuffer; - private final TimedValueQueue formatQueue; private final ArrayList decodeOnlyPresentationTimestamps; private final MediaCodec.BufferInfo outputBufferInfo; private final ArrayDeque pendingOutputStreamChanges; @@ -359,6 +358,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected DecoderCounters decoderCounters; private OutputStreamInfo outputStreamInfo; private long lastProcessedOutputBufferTimeUs; + private boolean needToNotifyOutputFormatChangeAfterStreamChange; /** * @param trackType The {@link C.TrackType track type} that the renderer handles. @@ -385,7 +385,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); bypassSampleBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); bypassBatchBuffer = new BatchBuffer(); - formatQueue = new TimedValueQueue<>(); decodeOnlyPresentationTimestamps = new ArrayList<>(); outputBufferInfo = new MediaCodec.BufferInfo(); currentPlaybackSpeed = 1f; @@ -597,13 +596,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected final void updateOutputFormatForTime(long presentationTimeUs) throws ExoPlaybackException { boolean outputFormatChanged = false; - @Nullable Format format = formatQueue.pollFloor(presentationTimeUs); - if (format == null && codecOutputMediaFormatChanged) { - // If the codec's output MediaFormat has changed then there should be a corresponding Format - // change, which we've not found. Check the Format queue in case the corresponding - // presentation timestamp is greater than presentationTimeUs, which can happen for some codecs - // [Internal ref: b/162719047]. - format = formatQueue.pollFirst(); + @Nullable Format format = outputStreamInfo.formatQueue.pollFloor(presentationTimeUs); + if (format == null + && needToNotifyOutputFormatChangeAfterStreamChange + && codecOutputMediaFormat != null) { + // After a stream change or after the initial start, there should be an input format change, + // which we've not found. Check the Format queue in case the corresponding presentation + // timestamp is greater than presentationTimeUs, which can happen for some codecs + // [Internal ref: b/162719047 and https://github.com/google/ExoPlayer/issues/8594]. + format = outputStreamInfo.formatQueue.pollFirst(); } if (format != null) { outputFormat = format; @@ -612,6 +613,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { if (outputFormatChanged || (codecOutputMediaFormatChanged && outputFormat != null)) { onOutputFormatChanged(outputFormat, codecOutputMediaFormat); codecOutputMediaFormatChanged = false; + needToNotifyOutputFormatChangeAfterStreamChange = false; } } @@ -668,10 +670,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer { // If there is a format change on the input side still pending propagation to the output, we // need to queue a format next time a buffer is read. This is because we may not read a new // input format after the position reset. - if (formatQueue.size() > 0) { + if (outputStreamInfo.formatQueue.size() > 0) { waitingForFirstSampleInFormat = true; } - formatQueue.clear(); + outputStreamInfo.formatQueue.clear(); pendingOutputStreamChanges.clear(); } @@ -1333,7 +1335,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer { decodeOnlyPresentationTimestamps.add(presentationTimeUs); } if (waitingForFirstSampleInFormat) { - formatQueue.add(presentationTimeUs, inputFormat); + if (!pendingOutputStreamChanges.isEmpty()) { + pendingOutputStreamChanges.peekLast().formatQueue.add(presentationTimeUs, inputFormat); + } else { + outputStreamInfo.formatQueue.add(presentationTimeUs, inputFormat); + } waitingForFirstSampleInFormat = false; } largestQueuedPresentationTimeUs = max(largestQueuedPresentationTimeUs, presentationTimeUs); @@ -2029,6 +2035,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private void setOutputStreamInfo(OutputStreamInfo outputStreamInfo) { this.outputStreamInfo = outputStreamInfo; if (outputStreamInfo.streamOffsetUs != C.TIME_UNSET) { + needToNotifyOutputFormatChangeAfterStreamChange = true; onOutputStreamOffsetUsChanged(outputStreamInfo.streamOffsetUs); } } @@ -2487,12 +2494,14 @@ public abstract class MediaCodecRenderer extends BaseRenderer { public final long previousStreamLastBufferTimeUs; public final long startPositionUs; public final long streamOffsetUs; + public final TimedValueQueue formatQueue; public OutputStreamInfo( long previousStreamLastBufferTimeUs, long startPositionUs, long streamOffsetUs) { this.previousStreamLastBufferTimeUs = previousStreamLastBufferTimeUs; this.startPositionUs = startPositionUs; this.streamOffsetUs = streamOffsetUs; + this.formatQueue = new TimedValueQueue<>(); } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java new file mode 100644 index 0000000000..ac86669ee5 --- /dev/null +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -0,0 +1,323 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.mediacodec; + +import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.spy; + +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.SystemClock; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.exoplayer.DecoderReuseEvaluation; +import androidx.media3.exoplayer.ExoPlaybackException; +import androidx.media3.exoplayer.RendererCapabilities; +import androidx.media3.exoplayer.RendererConfiguration; +import androidx.media3.exoplayer.analytics.PlayerId; +import androidx.media3.exoplayer.drm.DrmSessionEventListener; +import androidx.media3.exoplayer.drm.DrmSessionManager; +import androidx.media3.exoplayer.upstream.DefaultAllocator; +import androidx.media3.test.utils.FakeSampleStream; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; + +/** Unit tests for {@link MediaCodecRenderer} */ +@RunWith(AndroidJUnit4.class) +public class MediaCodecRendererTest { + + @Test + public void render_withReplaceStream_triggersOutputCallbacksInCorrectOrder() throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 400); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(400); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + @Test + public void + render_withReplaceStreamAndBufferBeyondDuration_triggersOutputCallbacksInCorrectOrder() + throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300, 400, 500, 600); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 400); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(400); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + @Test + public void + render_withReplaceStreamAndBufferLessThanStartPosition_triggersOutputCallbacksInCorrectOrder() + throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200, 300, 400); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 200); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(200); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + private FakeSampleStream createFakeSampleStream(Format format, long... sampleTimesUs) { + ImmutableList.Builder sampleListBuilder = + ImmutableList.builder(); + for (long sampleTimeUs : sampleTimesUs) { + sampleListBuilder.add(oneByteSample(sampleTimeUs, C.BUFFER_FLAG_KEY_FRAME)); + } + sampleListBuilder.add(END_OF_STREAM_ITEM); + FakeSampleStream sampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ format, + sampleListBuilder.build()); + sampleStream.writeData(/* startPositionUs= */ 0); + return sampleStream; + } + + private static class TestRenderer extends MediaCodecRenderer { + + public TestRenderer() { + super( + C.TRACK_TYPE_AUDIO, + MediaCodecAdapter.Factory.DEFAULT, + /* mediaCodecSelector= */ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> + Collections.singletonList( + MediaCodecInfo.newInstance( + /* name= */ "name", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ null, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)), + /* enableDecoderFallback= */ false, + /* assumedMinimumCodecOperatingRate= */ 44100); + } + + @Override + public String getName() { + return "test"; + } + + @Override + protected @Capabilities int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format) + throws MediaCodecUtil.DecoderQueryException { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected List getDecoderInfos( + MediaCodecSelector mediaCodecSelector, Format format, boolean requiresSecureDecoder) + throws MediaCodecUtil.DecoderQueryException { + return mediaCodecSelector.getDecoderInfos( + format.sampleMimeType, + /* requiresSecureDecoder= */ false, + /* requiresTunnelingDecoder= */ false); + } + + @Override + protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( + MediaCodecInfo codecInfo, + Format format, + @Nullable MediaCrypto crypto, + float codecOperatingRate) { + return MediaCodecAdapter.Configuration.createForAudioDecoding( + codecInfo, new MediaFormat(), format, crypto); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + @Nullable MediaCodecAdapter codec, + @Nullable ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + int sampleCount, + long bufferPresentationTimeUs, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, + Format format) + throws ExoPlaybackException { + if (bufferPresentationTimeUs <= positionUs) { + // Only release buffers when the position advances far enough for realistic behavior where + // input of buffers to the codec is faster than output. + codec.releaseOutputBuffer(bufferIndex, /* render= */ true); + return true; + } + return false; + } + + @Override + protected DecoderReuseEvaluation canReuseCodec( + MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) { + return new DecoderReuseEvaluation( + codecInfo.name, + oldFormat, + newFormat, + REUSE_RESULT_YES_WITHOUT_RECONFIGURATION, + /* discardReasons= */ 0); + } + } +}