diff --git a/RELEASENOTES.md b/RELEASENOTES.md index aca666f2db..1261e56f2b 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -10,6 +10,9 @@ * DataSource: * Audio: * Video: + * Add experimental `ExoPlayer` API to drop late `MediaCodecVideoRenderer` + decoder input buffers that are not depended on. Enable it with + `DefaultRenderersFactory.experimentalSetLateThresholdToDropDecoderInputUs`. * Text: * Metadata: * Image: diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java index a8863194db..4c676c1cc3 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java @@ -110,6 +110,7 @@ public class DefaultRenderersFactory implements RenderersFactory { private boolean enableAudioTrackPlaybackParams; private boolean enableMediaCodecVideoRendererPrewarming; private boolean parseAv1SampleDependencies; + private long lateThresholdToDropDecoderInputUs; /** * @param context A {@link Context}. @@ -120,6 +121,7 @@ public class DefaultRenderersFactory implements RenderersFactory { extensionRendererMode = EXTENSION_RENDERER_MODE_OFF; allowedVideoJoiningTimeMs = DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS; mediaCodecSelector = MediaCodecSelector.DEFAULT; + lateThresholdToDropDecoderInputUs = C.TIME_UNSET; } /** @@ -313,6 +315,24 @@ public class DefaultRenderersFactory implements RenderersFactory { return this; } + /** + * Sets the late threshold for rendered output buffers, in microseconds, after which decoder input + * buffers may be dropped. + * + *

The default value is {@link C#TIME_UNSET} and therefore no input buffers will be dropped due + * to this logic. + * + *

This method is experimental and will be renamed or removed in a future release. + * + * @param lateThresholdToDropDecoderInputUs The threshold. + */ + @CanIgnoreReturnValue + public final DefaultRenderersFactory experimentalSetLateThresholdToDropDecoderInputUs( + long lateThresholdToDropDecoderInputUs) { + this.lateThresholdToDropDecoderInputUs = lateThresholdToDropDecoderInputUs; + return this; + } + @Override public Renderer[] createRenderers( Handler eventHandler, @@ -396,6 +416,7 @@ public class DefaultRenderersFactory implements RenderersFactory { .setEventListener(eventListener) .setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY) .experimentalSetParseAv1SampleDependencies(parseAv1SampleDependencies) + .experimentalSetLateThresholdToDropDecoderInputUs(lateThresholdToDropDecoderInputUs) .build(); out.add(videoRenderer); @@ -800,6 +821,7 @@ public class DefaultRenderersFactory implements RenderersFactory { .setEventListener(eventListener) .setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY) .experimentalSetParseAv1SampleDependencies(parseAv1SampleDependencies) + .experimentalSetLateThresholdToDropDecoderInputUs(lateThresholdToDropDecoderInputUs) .build(); } return null; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index da5496195a..e80e5ee3fd 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -82,6 +82,7 @@ import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.nio.ByteBuffer; import java.util.List; +import java.util.PriorityQueue; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -159,6 +160,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo; @Nullable private final Av1SampleDependencyParser av1SampleDependencyParser; + /** + * The earliest time threshold, in microseconds, after which decoder input buffers may be dropped. + */ + private final long minEarlyUsToDropDecoderInput; + + private final PriorityQueue droppedDecoderInputBufferTimestamps; + private @MonotonicNonNull CodecMaxValues codecMaxValues; private boolean codecNeedsSetOutputSurfaceWorkaround; private boolean codecHandlesHdr10PlusOutOfBandMetadata; @@ -190,6 +198,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer private long periodDurationUs; private boolean pendingVideoSinkInputStreamChange; + private boolean shouldDropDecoderInputBuffers; + /** A builder to create {@link MediaCodecVideoRenderer} instances. */ public static final class Builder { private final Context context; @@ -204,6 +214,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer private float assumedMinimumCodecOperatingRate; @Nullable private VideoSink videoSink; private boolean parseAv1SampleDependencies; + private long lateThresholdToDropDecoderInputUs; /** * Creates a new builder. @@ -215,6 +226,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer this.mediaCodecSelector = MediaCodecSelector.DEFAULT; this.codecAdapterFactory = MediaCodecAdapter.Factory.getDefault(context); this.assumedMinimumCodecOperatingRate = 30; + this.lateThresholdToDropDecoderInputUs = C.TIME_UNSET; } /** Sets the {@link MediaCodecSelector decoder selector}. */ @@ -327,6 +339,22 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer return this; } + /** + * Sets the late threshold for rendered output buffers, in microseconds, after which decoder + * input buffers may be dropped. + * + *

The default value is {@link C#TIME_UNSET} and therefore no input buffers will be dropped + * due to this logic. + * + *

This method is experimental and will be renamed or removed in a future release. + */ + @CanIgnoreReturnValue + public Builder experimentalSetLateThresholdToDropDecoderInputUs( + long lateThresholdToDropDecoderInputUs) { + this.lateThresholdToDropDecoderInputUs = lateThresholdToDropDecoderInputUs; + return this; + } + /** * Builds the {@link MediaCodecVideoRenderer}. Must only be called once per Builder instance. * @@ -546,6 +574,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer periodDurationUs = C.TIME_UNSET; av1SampleDependencyParser = builder.parseAv1SampleDependencies ? new Av1SampleDependencyParser() : null; + droppedDecoderInputBufferTimestamps = new PriorityQueue<>(); + minEarlyUsToDropDecoderInput = + builder.lateThresholdToDropDecoderInputUs != C.TIME_UNSET + ? -builder.lateThresholdToDropDecoderInputUs + : C.TIME_UNSET; } // FrameTimingEvaluator methods @@ -568,6 +601,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer boolean isLastFrame, boolean treatDroppedBuffersAsSkipped) throws ExoPlaybackException { + if (minEarlyUsToDropDecoderInput != C.TIME_UNSET) { + shouldDropDecoderInputBuffers = earlyUs < minEarlyUsToDropDecoderInput; + } return shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastFrame) && maybeDropBuffersToKeyframe(positionUs, treatDroppedBuffersAsSkipped); } @@ -1215,6 +1251,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @Override protected void resetCodecStateForFlush() { super.resetCodecStateForFlush(); + droppedDecoderInputBufferTimestamps.clear(); + shouldDropDecoderInputBuffers = false; buffersInCodecCount = 0; } @@ -1425,8 +1463,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer // block processed. Skipping input buffers before the decoder is not allowed. return false; } - // Skip buffers without sample dependencies that won't be rendered. - if (!isBufferBeforeStartTime(buffer)) { + boolean shouldSkipDecoderInputBuffer = isBufferBeforeStartTime(buffer); + if (!shouldSkipDecoderInputBuffer && !shouldDropDecoderInputBuffers) { return false; } if (buffer.hasSupplementalData()) { @@ -1434,7 +1472,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } if (buffer.notDependedOn()) { buffer.clear(); - decoderCounters.skippedInputBufferCount += 1; + if (shouldSkipDecoderInputBuffer) { + decoderCounters.skippedInputBufferCount += 1; + } else if (shouldDropDecoderInputBuffers) { + droppedDecoderInputBufferTimestamps.add(buffer.timeUs); + } return true; } if (av1SampleDependencyParser != null @@ -1450,7 +1492,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer if (sampleLimitAfterSkippingNonReferenceFrames != readOnlySample.limit() && hasSpaceForNextFrame) { checkNotNull(buffer.data).position(sampleLimitAfterSkippingNonReferenceFrames); - decoderCounters.skippedInputBufferCount += 1; + if (shouldSkipDecoderInputBuffer) { + decoderCounters.skippedInputBufferCount += 1; + } else if (shouldDropDecoderInputBuffers) { + droppedDecoderInputBufferTimestamps.add(buffer.timeUs); + } return true; } return false; @@ -1595,6 +1641,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer long outputStreamOffsetUs = getOutputStreamOffsetUs(); long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; + updateDroppedBufferCountersWithInputBuffers(presentationTimeUs); if (videoSink != null) { // Skip decode-only buffers, e.g. after seeking, immediately. @@ -1867,10 +1914,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer if (treatDroppedBuffersAsSkipped) { decoderCounters.skippedInputBufferCount += droppedSourceBufferCount; decoderCounters.skippedOutputBufferCount += buffersInCodecCount; + decoderCounters.skippedInputBufferCount += droppedDecoderInputBufferTimestamps.size(); } else { decoderCounters.droppedToKeyframeCount++; updateDroppedBufferCounters( - droppedSourceBufferCount, /* droppedDecoderBufferCount= */ buffersInCodecCount); + /* droppedInputBufferCount= */ droppedSourceBufferCount + + droppedDecoderInputBufferTimestamps.size(), + /* droppedDecoderBufferCount= */ buffersInCodecCount); } flushOrReinitializeCodec(); if (videoSink != null) { @@ -1901,6 +1951,23 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } } + /** + * Updates counters to reflect dropped input buffers prior to {@code presentationTimeUs}. + * + * @param presentationTimeUs The presentation timestamp of the last processed output buffer, in + * microseconds. + */ + private void updateDroppedBufferCountersWithInputBuffers(long presentationTimeUs) { + int droppedInputBufferCount = 0; + Long minDroppedDecoderBufferTimeUs; + while ((minDroppedDecoderBufferTimeUs = droppedDecoderInputBufferTimestamps.peek()) != null + && minDroppedDecoderBufferTimeUs < presentationTimeUs) { + droppedInputBufferCount++; + droppedDecoderInputBufferTimestamps.poll(); + } + updateDroppedBufferCounters(droppedInputBufferCount, /* droppedDecoderBufferCount= */ 0); + } + /** * Updates local counters and {@link DecoderCounters} with a new video frame processing offset. * diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 4e412ed169..39d2d6c958 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -15,6 +15,7 @@ */ package androidx.media3.exoplayer.video; +import static android.media.MediaCodec.INFO_TRY_AGAIN_LATER; import static android.view.Display.DEFAULT_DISPLAY; import static androidx.media3.common.util.Util.msToUs; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; @@ -90,6 +91,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.PriorityQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -960,6 +962,395 @@ public class MediaCodecVideoRendererTest { assertThat(currentOutputFormat).isEqualTo(VIDEO_H264); } + @Test + public void render_withLateBufferWithoutDependencies_dropsInputBuffers() throws Exception { + FakeTimeline fakeTimeline = + new FakeTimeline( + new FakeTimeline.TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 1_000_000)); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample( + /* timeUs= */ 20_000))); // Late buffer triggers input buffer dropping. + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new DefaultMediaCodecAdapterFactory( + ApplicationProvider.getApplicationContext(), + () -> { + callbackThread = new HandlerThread("MCVRTest:MediaCodecAsyncAdapter"); + return callbackThread; + }, + () -> { + queueingThread = new HandlerThread("MCVRTest:MediaCodecQueueingThread"); + return queueingThread; + })) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1) + .experimentalSetLateThresholdToDropDecoderInputUs(50_000)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + }; + + mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.setTimeline(fakeTimeline); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0))); + shadowOf(testMainLooper).idle(); + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + DecoderCounters decoderCounters = argumentDecoderCounters.getValue(); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + while (decoderCounters.renderedOutputBufferCount == 0) { + mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000); + } + // Ensure existing buffer will be ~280ms late and new (not yet read) buffers are available + // to be dropped. + int posUs = 300_000; + fakeSampleStream.append( + ImmutableList.of( + oneByteSample(/* timeUs= */ 30_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), // Dropped on input. + oneByteSample(/* timeUs= */ 300_000), // Caught up - render. + oneByteSample(/* timeUs= */ 500_000), // Last buffer is always rendered. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + // Render until the non-dropped frame is reached and then increase time to reach the end. + while (decoderCounters.renderedOutputBufferCount < 2) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + } + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 2_000; + } + + assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(1); + assertThat(decoderCounters.droppedBufferCount).isEqualTo(2); + assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isEqualTo(2); + assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0); + } + + // TODO: b/390604981 - Run the test on older SDK levels to ensure it uses a MediaCodec shadow + // with more than one buffer slot. + @Config(minSdk = 30) + @Test + public void render_afterVeryLateBuffer_doesNotDropInputBuffers() throws Exception { + FakeTimeline fakeTimeline = + new FakeTimeline( + new FakeTimeline.TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 2_000_000)); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 20_000))); // Very late buffer. + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new DefaultMediaCodecAdapterFactory( + ApplicationProvider.getApplicationContext(), + () -> { + callbackThread = new HandlerThread("MCVRTest:MediaCodecAsyncAdapter"); + return callbackThread; + }, + () -> { + queueingThread = new HandlerThread("MCVRTest:MediaCodecQueueingThread"); + return queueingThread; + })) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1) + .experimentalSetLateThresholdToDropDecoderInputUs(50_000)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + }; + + mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.setTimeline(fakeTimeline); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0))); + shadowOf(testMainLooper).idle(); + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + DecoderCounters decoderCounters = argumentDecoderCounters.getValue(); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + while (decoderCounters.renderedOutputBufferCount == 0) { + mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000); + } + // Ensure existing buffer will be 1 second late and new (not yet read) buffers are available + // to be skipped and to skip to in the input stream. + int posUs = 1_020_000; + fakeSampleStream.append( + ImmutableList.of( + oneByteSample(/* timeUs= */ 30_000), // Dropped input buffer when skipping to keyframe. + oneByteSample(/* timeUs= */ 1_020_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_040_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), + oneByteSample(/* timeUs= */ 1_060_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), + oneByteSample(/* timeUs= */ 2_000_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + // Render until the new keyframe has been processed and then increase time to reach the end. + while (decoderCounters.renderedOutputBufferCount < 2) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + maybeIdleAsynchronousMediaCodecAdapterThreads(); + } + while (!mediaCodecVideoRenderer.isEnded()) { + maybeIdleAsynchronousMediaCodecAdapterThreads(); + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 1_000; + } + + assertThat(decoderCounters.renderedOutputBufferCount).isEqualTo(5); + assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(1); + assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(1); + } + + @Test + public void render_afterVeryLateBuffer_countsDroppedInputBuffersCorrectly() throws Exception { + FakeTimeline fakeTimeline = + new FakeTimeline( + new FakeTimeline.TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 3_000_000)); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 20_000))); // Late buffer. + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory()) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1) + .experimentalSetLateThresholdToDropDecoderInputUs(50_000)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + }; + + mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.setTimeline(fakeTimeline); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0))); + shadowOf(testMainLooper).idle(); + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + DecoderCounters decoderCounters = argumentDecoderCounters.getValue(); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + while (decoderCounters.renderedOutputBufferCount == 0) { + mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000); + } + fakeSampleStream.append( + ImmutableList.of( + // Dropped input buffer. + oneByteSample(/* timeUs= */ 500_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), + oneByteSample(300_000), // Render. + oneByteSample(400_000) // Very late buffer. + )); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + // Render until the frame at time 300_000 us is displayed. + int posUs = 300_000; + while (decoderCounters.renderedOutputBufferCount < 2) { + maybeIdleAsynchronousMediaCodecAdapterThreads(); + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + } + fakeSampleStream.append( + ImmutableList.of( + oneByteSample( + /* timeUs= */ 1_000_000), // Dropped input buffer when skipping to keyframe. + oneByteSample(/* timeUs= */ 2_020_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 3_000_000), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + posUs = 2_020_000; + while (decoderCounters.renderedOutputBufferCount + decoderCounters.skippedOutputBufferCount + < 3) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + maybeIdleAsynchronousMediaCodecAdapterThreads(); + } + while (!mediaCodecVideoRenderer.isEnded()) { + maybeIdleAsynchronousMediaCodecAdapterThreads(); + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 1_000; + } + + assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(2); + assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(1); + } + + @Test + public void + render_withLateBufferAndOutOfOrderSamplesWithoutDependencies_dropsInputBuffersAndRendersLast() + throws Exception { + FakeTimeline fakeTimeline = + new FakeTimeline( + new FakeTimeline.TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 1_000_000)); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample( + /* timeUs= */ 20_000))); // Late buffer triggers input buffer dropping. + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory()) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1) + .experimentalSetLateThresholdToDropDecoderInputUs(200_000)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + }; + + mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.setTimeline(fakeTimeline); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0))); + shadowOf(testMainLooper).idle(); + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + DecoderCounters decoderCounters = argumentDecoderCounters.getValue(); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + while (decoderCounters.renderedOutputBufferCount == 0) { + mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000); + } + // Ensure existing buffer will be ~280ms late and new (not yet read) buffers are available + // to be dropped. + int posUs = 300_000; + fakeSampleStream.append( + ImmutableList.of( + oneByteSample(/* timeUs= */ 300_000), // Render. + oneByteSample(/* timeUs= */ 320_000), // Render. + // Drop consecutive input buffers that aren't consecutive output buffers. + oneByteSample(/* timeUs= */ 310_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), + oneByteSample(/* timeUs= */ 330_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), + // Last buffer is always rendered. + oneByteSample(/* timeUs= */ 500_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + // Render until the first frame is reached and then increase time to reach the end. + while (decoderCounters.renderedOutputBufferCount < 2) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + } + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 2_000; + } + + assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(2); + assertThat(decoderCounters.droppedBufferCount).isEqualTo(3); + assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isEqualTo(1); + assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0); + } + @Test public void enable_withMayRenderStartOfStream_rendersFirstFrameBeforeStart() throws Exception { FakeSampleStream fakeSampleStream = @@ -1995,6 +2386,56 @@ public class MediaCodecVideoRendererTest { .build(); } + private static final class ForwardingSynchronousMediaCodecAdapterWithReordering + extends ForwardingSynchronousMediaCodecAdapter { + /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithReordering} instances. */ + public static final class Factory implements MediaCodecAdapter.Factory { + @Override + public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException { + return new ForwardingSynchronousMediaCodecAdapterWithReordering( + new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration)); + } + } + + private final PriorityQueue timestamps; + + ForwardingSynchronousMediaCodecAdapterWithReordering(MediaCodecAdapter adapter) { + super(adapter); + timestamps = new PriorityQueue<>(); + } + + @Override + public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) { + int outputBufferIndex = super.dequeueOutputBufferIndex(bufferInfo); + Long smallestTimestamp = timestamps.peek(); + if (smallestTimestamp != null && outputBufferIndex != INFO_TRY_AGAIN_LATER) { + bufferInfo.presentationTimeUs = smallestTimestamp; + } + return outputBufferIndex; + } + + @Override + public void queueInputBuffer( + int index, int offset, int size, long presentationTimeUs, int flags) { + if ((flags & C.BUFFER_FLAG_END_OF_STREAM) == 0) { + timestamps.add(presentationTimeUs); + } + super.queueInputBuffer(index, offset, size, presentationTimeUs, flags); + } + + @Override + public void releaseOutputBuffer(int index, boolean render) { + timestamps.poll(); + super.releaseOutputBuffer(index, render); + } + + @Override + public void releaseOutputBuffer(int index, long renderTimeStampNs) { + timestamps.poll(); + super.releaseOutputBuffer(index, renderTimeStampNs); + } + } + private static final class ForwardingSynchronousMediaCodecAdapterWithBufferLimit extends ForwardingSynchronousMediaCodecAdapter { /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithBufferLimit} instances. */