diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index aca666f2db..1261e56f2b 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -10,6 +10,9 @@
* DataSource:
* Audio:
* Video:
+ * Add experimental `ExoPlayer` API to drop late `MediaCodecVideoRenderer`
+ decoder input buffers that are not depended on. Enable it with
+ `DefaultRenderersFactory.experimentalSetLateThresholdToDropDecoderInputUs`.
* Text:
* Metadata:
* Image:
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java
index a8863194db..4c676c1cc3 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java
@@ -110,6 +110,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
private boolean enableAudioTrackPlaybackParams;
private boolean enableMediaCodecVideoRendererPrewarming;
private boolean parseAv1SampleDependencies;
+ private long lateThresholdToDropDecoderInputUs;
/**
* @param context A {@link Context}.
@@ -120,6 +121,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
extensionRendererMode = EXTENSION_RENDERER_MODE_OFF;
allowedVideoJoiningTimeMs = DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS;
mediaCodecSelector = MediaCodecSelector.DEFAULT;
+ lateThresholdToDropDecoderInputUs = C.TIME_UNSET;
}
/**
@@ -313,6 +315,24 @@ public class DefaultRenderersFactory implements RenderersFactory {
return this;
}
+ /**
+ * Sets the late threshold for rendered output buffers, in microseconds, after which decoder input
+ * buffers may be dropped.
+ *
+ *
The default value is {@link C#TIME_UNSET} and therefore no input buffers will be dropped due
+ * to this logic.
+ *
+ *
This method is experimental and will be renamed or removed in a future release.
+ *
+ * @param lateThresholdToDropDecoderInputUs The threshold.
+ */
+ @CanIgnoreReturnValue
+ public final DefaultRenderersFactory experimentalSetLateThresholdToDropDecoderInputUs(
+ long lateThresholdToDropDecoderInputUs) {
+ this.lateThresholdToDropDecoderInputUs = lateThresholdToDropDecoderInputUs;
+ return this;
+ }
+
@Override
public Renderer[] createRenderers(
Handler eventHandler,
@@ -396,6 +416,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)
.experimentalSetParseAv1SampleDependencies(parseAv1SampleDependencies)
+ .experimentalSetLateThresholdToDropDecoderInputUs(lateThresholdToDropDecoderInputUs)
.build();
out.add(videoRenderer);
@@ -800,6 +821,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)
.experimentalSetParseAv1SampleDependencies(parseAv1SampleDependencies)
+ .experimentalSetLateThresholdToDropDecoderInputUs(lateThresholdToDropDecoderInputUs)
.build();
}
return null;
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
index da5496195a..e80e5ee3fd 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
@@ -82,6 +82,7 @@ import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.nio.ByteBuffer;
import java.util.List;
+import java.util.PriorityQueue;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@@ -159,6 +160,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo;
@Nullable private final Av1SampleDependencyParser av1SampleDependencyParser;
+ /**
+ * The earliest time threshold, in microseconds, after which decoder input buffers may be dropped.
+ */
+ private final long minEarlyUsToDropDecoderInput;
+
+ private final PriorityQueue droppedDecoderInputBufferTimestamps;
+
private @MonotonicNonNull CodecMaxValues codecMaxValues;
private boolean codecNeedsSetOutputSurfaceWorkaround;
private boolean codecHandlesHdr10PlusOutOfBandMetadata;
@@ -190,6 +198,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private long periodDurationUs;
private boolean pendingVideoSinkInputStreamChange;
+ private boolean shouldDropDecoderInputBuffers;
+
/** A builder to create {@link MediaCodecVideoRenderer} instances. */
public static final class Builder {
private final Context context;
@@ -204,6 +214,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private float assumedMinimumCodecOperatingRate;
@Nullable private VideoSink videoSink;
private boolean parseAv1SampleDependencies;
+ private long lateThresholdToDropDecoderInputUs;
/**
* Creates a new builder.
@@ -215,6 +226,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
this.mediaCodecSelector = MediaCodecSelector.DEFAULT;
this.codecAdapterFactory = MediaCodecAdapter.Factory.getDefault(context);
this.assumedMinimumCodecOperatingRate = 30;
+ this.lateThresholdToDropDecoderInputUs = C.TIME_UNSET;
}
/** Sets the {@link MediaCodecSelector decoder selector}. */
@@ -327,6 +339,22 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
return this;
}
+ /**
+ * Sets the late threshold for rendered output buffers, in microseconds, after which decoder
+ * input buffers may be dropped.
+ *
+ * The default value is {@link C#TIME_UNSET} and therefore no input buffers will be dropped
+ * due to this logic.
+ *
+ *
This method is experimental and will be renamed or removed in a future release.
+ */
+ @CanIgnoreReturnValue
+ public Builder experimentalSetLateThresholdToDropDecoderInputUs(
+ long lateThresholdToDropDecoderInputUs) {
+ this.lateThresholdToDropDecoderInputUs = lateThresholdToDropDecoderInputUs;
+ return this;
+ }
+
/**
* Builds the {@link MediaCodecVideoRenderer}. Must only be called once per Builder instance.
*
@@ -546,6 +574,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
periodDurationUs = C.TIME_UNSET;
av1SampleDependencyParser =
builder.parseAv1SampleDependencies ? new Av1SampleDependencyParser() : null;
+ droppedDecoderInputBufferTimestamps = new PriorityQueue<>();
+ minEarlyUsToDropDecoderInput =
+ builder.lateThresholdToDropDecoderInputUs != C.TIME_UNSET
+ ? -builder.lateThresholdToDropDecoderInputUs
+ : C.TIME_UNSET;
}
// FrameTimingEvaluator methods
@@ -568,6 +601,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
boolean isLastFrame,
boolean treatDroppedBuffersAsSkipped)
throws ExoPlaybackException {
+ if (minEarlyUsToDropDecoderInput != C.TIME_UNSET) {
+ shouldDropDecoderInputBuffers = earlyUs < minEarlyUsToDropDecoderInput;
+ }
return shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastFrame)
&& maybeDropBuffersToKeyframe(positionUs, treatDroppedBuffersAsSkipped);
}
@@ -1215,6 +1251,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
@Override
protected void resetCodecStateForFlush() {
super.resetCodecStateForFlush();
+ droppedDecoderInputBufferTimestamps.clear();
+ shouldDropDecoderInputBuffers = false;
buffersInCodecCount = 0;
}
@@ -1425,8 +1463,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
// block processed. Skipping input buffers before the decoder is not allowed.
return false;
}
- // Skip buffers without sample dependencies that won't be rendered.
- if (!isBufferBeforeStartTime(buffer)) {
+ boolean shouldSkipDecoderInputBuffer = isBufferBeforeStartTime(buffer);
+ if (!shouldSkipDecoderInputBuffer && !shouldDropDecoderInputBuffers) {
return false;
}
if (buffer.hasSupplementalData()) {
@@ -1434,7 +1472,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
}
if (buffer.notDependedOn()) {
buffer.clear();
- decoderCounters.skippedInputBufferCount += 1;
+ if (shouldSkipDecoderInputBuffer) {
+ decoderCounters.skippedInputBufferCount += 1;
+ } else if (shouldDropDecoderInputBuffers) {
+ droppedDecoderInputBufferTimestamps.add(buffer.timeUs);
+ }
return true;
}
if (av1SampleDependencyParser != null
@@ -1450,7 +1492,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
if (sampleLimitAfterSkippingNonReferenceFrames != readOnlySample.limit()
&& hasSpaceForNextFrame) {
checkNotNull(buffer.data).position(sampleLimitAfterSkippingNonReferenceFrames);
- decoderCounters.skippedInputBufferCount += 1;
+ if (shouldSkipDecoderInputBuffer) {
+ decoderCounters.skippedInputBufferCount += 1;
+ } else if (shouldDropDecoderInputBuffers) {
+ droppedDecoderInputBufferTimestamps.add(buffer.timeUs);
+ }
return true;
}
return false;
@@ -1595,6 +1641,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
long outputStreamOffsetUs = getOutputStreamOffsetUs();
long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
+ updateDroppedBufferCountersWithInputBuffers(presentationTimeUs);
if (videoSink != null) {
// Skip decode-only buffers, e.g. after seeking, immediately.
@@ -1867,10 +1914,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
if (treatDroppedBuffersAsSkipped) {
decoderCounters.skippedInputBufferCount += droppedSourceBufferCount;
decoderCounters.skippedOutputBufferCount += buffersInCodecCount;
+ decoderCounters.skippedInputBufferCount += droppedDecoderInputBufferTimestamps.size();
} else {
decoderCounters.droppedToKeyframeCount++;
updateDroppedBufferCounters(
- droppedSourceBufferCount, /* droppedDecoderBufferCount= */ buffersInCodecCount);
+ /* droppedInputBufferCount= */ droppedSourceBufferCount
+ + droppedDecoderInputBufferTimestamps.size(),
+ /* droppedDecoderBufferCount= */ buffersInCodecCount);
}
flushOrReinitializeCodec();
if (videoSink != null) {
@@ -1901,6 +1951,23 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
}
}
+ /**
+ * Updates counters to reflect dropped input buffers prior to {@code presentationTimeUs}.
+ *
+ * @param presentationTimeUs The presentation timestamp of the last processed output buffer, in
+ * microseconds.
+ */
+ private void updateDroppedBufferCountersWithInputBuffers(long presentationTimeUs) {
+ int droppedInputBufferCount = 0;
+ Long minDroppedDecoderBufferTimeUs;
+ while ((minDroppedDecoderBufferTimeUs = droppedDecoderInputBufferTimestamps.peek()) != null
+ && minDroppedDecoderBufferTimeUs < presentationTimeUs) {
+ droppedInputBufferCount++;
+ droppedDecoderInputBufferTimestamps.poll();
+ }
+ updateDroppedBufferCounters(droppedInputBufferCount, /* droppedDecoderBufferCount= */ 0);
+ }
+
/**
* Updates local counters and {@link DecoderCounters} with a new video frame processing offset.
*
diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
index 4e412ed169..39d2d6c958 100644
--- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
+++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
@@ -15,6 +15,7 @@
*/
package androidx.media3.exoplayer.video;
+import static android.media.MediaCodec.INFO_TRY_AGAIN_LATER;
import static android.view.Display.DEFAULT_DISPLAY;
import static androidx.media3.common.util.Util.msToUs;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM;
@@ -90,6 +91,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.PriorityQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@@ -960,6 +962,395 @@ public class MediaCodecVideoRendererTest {
assertThat(currentOutputFormat).isEqualTo(VIDEO_H264);
}
+ @Test
+ public void render_withLateBufferWithoutDependencies_dropsInputBuffers() throws Exception {
+ FakeTimeline fakeTimeline =
+ new FakeTimeline(
+ new FakeTimeline.TimelineWindowDefinition(
+ /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 1_000_000));
+ FakeSampleStream fakeSampleStream =
+ new FakeSampleStream(
+ new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+ /* mediaSourceEventDispatcher= */ null,
+ DrmSessionManager.DRM_UNSUPPORTED,
+ new DrmSessionEventListener.EventDispatcher(),
+ /* initialFormat= */ VIDEO_H264,
+ ImmutableList.of(
+ oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
+ oneByteSample(
+ /* timeUs= */ 20_000))); // Late buffer triggers input buffer dropping.
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ mediaCodecVideoRenderer =
+ new MediaCodecVideoRenderer(
+ new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
+ .setCodecAdapterFactory(
+ new DefaultMediaCodecAdapterFactory(
+ ApplicationProvider.getApplicationContext(),
+ () -> {
+ callbackThread = new HandlerThread("MCVRTest:MediaCodecAsyncAdapter");
+ return callbackThread;
+ },
+ () -> {
+ queueingThread = new HandlerThread("MCVRTest:MediaCodecQueueingThread");
+ return queueingThread;
+ }))
+ .setMediaCodecSelector(mediaCodecSelector)
+ .setAllowedJoiningTimeMs(0)
+ .setEnableDecoderFallback(false)
+ .setEventHandler(new Handler(testMainLooper))
+ .setEventListener(eventListener)
+ .setMaxDroppedFramesToNotify(1)
+ .experimentalSetLateThresholdToDropDecoderInputUs(50_000)) {
+ @Override
+ protected @Capabilities int supportsFormat(
+ MediaCodecSelector mediaCodecSelector, Format format) {
+ return RendererCapabilities.create(C.FORMAT_HANDLED);
+ }
+ };
+
+ mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
+ mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
+ mediaCodecVideoRenderer.setTimeline(fakeTimeline);
+ mediaCodecVideoRenderer.enable(
+ RendererConfiguration.DEFAULT,
+ new Format[] {VIDEO_H264},
+ fakeSampleStream,
+ /* positionUs= */ 0,
+ /* joining= */ false,
+ /* mayRenderStartOfStream= */ true,
+ /* startPositionUs= */ 0,
+ /* offsetUs= */ 0,
+ new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
+ shadowOf(testMainLooper).idle();
+ ArgumentCaptor argumentDecoderCounters =
+ ArgumentCaptor.forClass(DecoderCounters.class);
+ verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
+ DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
+
+ mediaCodecVideoRenderer.start();
+ mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
+ while (decoderCounters.renderedOutputBufferCount == 0) {
+ mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
+ }
+ // Ensure existing buffer will be ~280ms late and new (not yet read) buffers are available
+ // to be dropped.
+ int posUs = 300_000;
+ fakeSampleStream.append(
+ ImmutableList.of(
+ oneByteSample(/* timeUs= */ 30_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), // Dropped on input.
+ oneByteSample(/* timeUs= */ 300_000), // Caught up - render.
+ oneByteSample(/* timeUs= */ 500_000), // Last buffer is always rendered.
+ END_OF_STREAM_ITEM));
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ mediaCodecVideoRenderer.setCurrentStreamFinal();
+ // Render until the non-dropped frame is reached and then increase time to reach the end.
+ while (decoderCounters.renderedOutputBufferCount < 2) {
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ }
+ while (!mediaCodecVideoRenderer.isEnded()) {
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ posUs += 2_000;
+ }
+
+ assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(1);
+ assertThat(decoderCounters.droppedBufferCount).isEqualTo(2);
+ assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isEqualTo(2);
+ assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0);
+ }
+
+ // TODO: b/390604981 - Run the test on older SDK levels to ensure it uses a MediaCodec shadow
+ // with more than one buffer slot.
+ @Config(minSdk = 30)
+ @Test
+ public void render_afterVeryLateBuffer_doesNotDropInputBuffers() throws Exception {
+ FakeTimeline fakeTimeline =
+ new FakeTimeline(
+ new FakeTimeline.TimelineWindowDefinition(
+ /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 2_000_000));
+ FakeSampleStream fakeSampleStream =
+ new FakeSampleStream(
+ new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+ /* mediaSourceEventDispatcher= */ null,
+ DrmSessionManager.DRM_UNSUPPORTED,
+ new DrmSessionEventListener.EventDispatcher(),
+ /* initialFormat= */ VIDEO_H264,
+ ImmutableList.of(
+ oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
+ oneByteSample(/* timeUs= */ 20_000))); // Very late buffer.
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ mediaCodecVideoRenderer =
+ new MediaCodecVideoRenderer(
+ new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
+ .setCodecAdapterFactory(
+ new DefaultMediaCodecAdapterFactory(
+ ApplicationProvider.getApplicationContext(),
+ () -> {
+ callbackThread = new HandlerThread("MCVRTest:MediaCodecAsyncAdapter");
+ return callbackThread;
+ },
+ () -> {
+ queueingThread = new HandlerThread("MCVRTest:MediaCodecQueueingThread");
+ return queueingThread;
+ }))
+ .setMediaCodecSelector(mediaCodecSelector)
+ .setAllowedJoiningTimeMs(0)
+ .setEnableDecoderFallback(false)
+ .setEventHandler(new Handler(testMainLooper))
+ .setEventListener(eventListener)
+ .setMaxDroppedFramesToNotify(1)
+ .experimentalSetLateThresholdToDropDecoderInputUs(50_000)) {
+ @Override
+ protected @Capabilities int supportsFormat(
+ MediaCodecSelector mediaCodecSelector, Format format) {
+ return RendererCapabilities.create(C.FORMAT_HANDLED);
+ }
+ };
+
+ mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
+ mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
+ mediaCodecVideoRenderer.setTimeline(fakeTimeline);
+ mediaCodecVideoRenderer.enable(
+ RendererConfiguration.DEFAULT,
+ new Format[] {VIDEO_H264},
+ fakeSampleStream,
+ /* positionUs= */ 0,
+ /* joining= */ false,
+ /* mayRenderStartOfStream= */ true,
+ /* startPositionUs= */ 0,
+ /* offsetUs= */ 0,
+ new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
+ shadowOf(testMainLooper).idle();
+ ArgumentCaptor argumentDecoderCounters =
+ ArgumentCaptor.forClass(DecoderCounters.class);
+ verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
+ DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
+
+ mediaCodecVideoRenderer.start();
+ mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
+ while (decoderCounters.renderedOutputBufferCount == 0) {
+ mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
+ }
+ // Ensure existing buffer will be 1 second late and new (not yet read) buffers are available
+ // to be skipped and to skip to in the input stream.
+ int posUs = 1_020_000;
+ fakeSampleStream.append(
+ ImmutableList.of(
+ oneByteSample(/* timeUs= */ 30_000), // Dropped input buffer when skipping to keyframe.
+ oneByteSample(/* timeUs= */ 1_020_000, C.BUFFER_FLAG_KEY_FRAME),
+ oneByteSample(/* timeUs= */ 1_040_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+ oneByteSample(/* timeUs= */ 1_060_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+ oneByteSample(/* timeUs= */ 2_000_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+ END_OF_STREAM_ITEM));
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ mediaCodecVideoRenderer.setCurrentStreamFinal();
+ // Render until the new keyframe has been processed and then increase time to reach the end.
+ while (decoderCounters.renderedOutputBufferCount < 2) {
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ maybeIdleAsynchronousMediaCodecAdapterThreads();
+ }
+ while (!mediaCodecVideoRenderer.isEnded()) {
+ maybeIdleAsynchronousMediaCodecAdapterThreads();
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ posUs += 1_000;
+ }
+
+ assertThat(decoderCounters.renderedOutputBufferCount).isEqualTo(5);
+ assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(1);
+ assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(1);
+ }
+
+ @Test
+ public void render_afterVeryLateBuffer_countsDroppedInputBuffersCorrectly() throws Exception {
+ FakeTimeline fakeTimeline =
+ new FakeTimeline(
+ new FakeTimeline.TimelineWindowDefinition(
+ /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 3_000_000));
+ FakeSampleStream fakeSampleStream =
+ new FakeSampleStream(
+ new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+ /* mediaSourceEventDispatcher= */ null,
+ DrmSessionManager.DRM_UNSUPPORTED,
+ new DrmSessionEventListener.EventDispatcher(),
+ /* initialFormat= */ VIDEO_H264,
+ ImmutableList.of(
+ oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
+ oneByteSample(/* timeUs= */ 20_000))); // Late buffer.
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ mediaCodecVideoRenderer =
+ new MediaCodecVideoRenderer(
+ new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
+ .setCodecAdapterFactory(
+ new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory())
+ .setMediaCodecSelector(mediaCodecSelector)
+ .setAllowedJoiningTimeMs(0)
+ .setEnableDecoderFallback(false)
+ .setEventHandler(new Handler(testMainLooper))
+ .setEventListener(eventListener)
+ .setMaxDroppedFramesToNotify(1)
+ .experimentalSetLateThresholdToDropDecoderInputUs(50_000)) {
+ @Override
+ protected @Capabilities int supportsFormat(
+ MediaCodecSelector mediaCodecSelector, Format format) {
+ return RendererCapabilities.create(C.FORMAT_HANDLED);
+ }
+ };
+
+ mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
+ mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
+ mediaCodecVideoRenderer.setTimeline(fakeTimeline);
+ mediaCodecVideoRenderer.enable(
+ RendererConfiguration.DEFAULT,
+ new Format[] {VIDEO_H264},
+ fakeSampleStream,
+ /* positionUs= */ 0,
+ /* joining= */ false,
+ /* mayRenderStartOfStream= */ true,
+ /* startPositionUs= */ 0,
+ /* offsetUs= */ 0,
+ new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
+ shadowOf(testMainLooper).idle();
+ ArgumentCaptor argumentDecoderCounters =
+ ArgumentCaptor.forClass(DecoderCounters.class);
+ verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
+ DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
+
+ mediaCodecVideoRenderer.start();
+ mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
+ while (decoderCounters.renderedOutputBufferCount == 0) {
+ mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
+ }
+ fakeSampleStream.append(
+ ImmutableList.of(
+ // Dropped input buffer.
+ oneByteSample(/* timeUs= */ 500_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+ oneByteSample(300_000), // Render.
+ oneByteSample(400_000) // Very late buffer.
+ ));
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ // Render until the frame at time 300_000 us is displayed.
+ int posUs = 300_000;
+ while (decoderCounters.renderedOutputBufferCount < 2) {
+ maybeIdleAsynchronousMediaCodecAdapterThreads();
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ }
+ fakeSampleStream.append(
+ ImmutableList.of(
+ oneByteSample(
+ /* timeUs= */ 1_000_000), // Dropped input buffer when skipping to keyframe.
+ oneByteSample(/* timeUs= */ 2_020_000, C.BUFFER_FLAG_KEY_FRAME),
+ oneByteSample(/* timeUs= */ 3_000_000),
+ END_OF_STREAM_ITEM));
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ mediaCodecVideoRenderer.setCurrentStreamFinal();
+ posUs = 2_020_000;
+ while (decoderCounters.renderedOutputBufferCount + decoderCounters.skippedOutputBufferCount
+ < 3) {
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ maybeIdleAsynchronousMediaCodecAdapterThreads();
+ }
+ while (!mediaCodecVideoRenderer.isEnded()) {
+ maybeIdleAsynchronousMediaCodecAdapterThreads();
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ posUs += 1_000;
+ }
+
+ assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(2);
+ assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(1);
+ }
+
+ @Test
+ public void
+ render_withLateBufferAndOutOfOrderSamplesWithoutDependencies_dropsInputBuffersAndRendersLast()
+ throws Exception {
+ FakeTimeline fakeTimeline =
+ new FakeTimeline(
+ new FakeTimeline.TimelineWindowDefinition(
+ /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 1_000_000));
+ FakeSampleStream fakeSampleStream =
+ new FakeSampleStream(
+ new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+ /* mediaSourceEventDispatcher= */ null,
+ DrmSessionManager.DRM_UNSUPPORTED,
+ new DrmSessionEventListener.EventDispatcher(),
+ /* initialFormat= */ VIDEO_H264,
+ ImmutableList.of(
+ oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
+ oneByteSample(
+ /* timeUs= */ 20_000))); // Late buffer triggers input buffer dropping.
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ mediaCodecVideoRenderer =
+ new MediaCodecVideoRenderer(
+ new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
+ .setCodecAdapterFactory(
+ new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory())
+ .setMediaCodecSelector(mediaCodecSelector)
+ .setAllowedJoiningTimeMs(0)
+ .setEnableDecoderFallback(false)
+ .setEventHandler(new Handler(testMainLooper))
+ .setEventListener(eventListener)
+ .setMaxDroppedFramesToNotify(1)
+ .experimentalSetLateThresholdToDropDecoderInputUs(200_000)) {
+ @Override
+ protected @Capabilities int supportsFormat(
+ MediaCodecSelector mediaCodecSelector, Format format) {
+ return RendererCapabilities.create(C.FORMAT_HANDLED);
+ }
+ };
+
+ mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
+ mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
+ mediaCodecVideoRenderer.setTimeline(fakeTimeline);
+ mediaCodecVideoRenderer.enable(
+ RendererConfiguration.DEFAULT,
+ new Format[] {VIDEO_H264},
+ fakeSampleStream,
+ /* positionUs= */ 0,
+ /* joining= */ false,
+ /* mayRenderStartOfStream= */ true,
+ /* startPositionUs= */ 0,
+ /* offsetUs= */ 0,
+ new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
+ shadowOf(testMainLooper).idle();
+ ArgumentCaptor argumentDecoderCounters =
+ ArgumentCaptor.forClass(DecoderCounters.class);
+ verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
+ DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
+
+ mediaCodecVideoRenderer.start();
+ mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
+ while (decoderCounters.renderedOutputBufferCount == 0) {
+ mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
+ }
+ // Ensure existing buffer will be ~280ms late and new (not yet read) buffers are available
+ // to be dropped.
+ int posUs = 300_000;
+ fakeSampleStream.append(
+ ImmutableList.of(
+ oneByteSample(/* timeUs= */ 300_000), // Render.
+ oneByteSample(/* timeUs= */ 320_000), // Render.
+ // Drop consecutive input buffers that aren't consecutive output buffers.
+ oneByteSample(/* timeUs= */ 310_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+ oneByteSample(/* timeUs= */ 330_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+ // Last buffer is always rendered.
+ oneByteSample(/* timeUs= */ 500_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+ END_OF_STREAM_ITEM));
+ fakeSampleStream.writeData(/* startPositionUs= */ 0);
+ mediaCodecVideoRenderer.setCurrentStreamFinal();
+ // Render until the first frame is reached and then increase time to reach the end.
+ while (decoderCounters.renderedOutputBufferCount < 2) {
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ }
+ while (!mediaCodecVideoRenderer.isEnded()) {
+ mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+ posUs += 2_000;
+ }
+
+ assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(2);
+ assertThat(decoderCounters.droppedBufferCount).isEqualTo(3);
+ assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isEqualTo(1);
+ assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0);
+ }
+
@Test
public void enable_withMayRenderStartOfStream_rendersFirstFrameBeforeStart() throws Exception {
FakeSampleStream fakeSampleStream =
@@ -1995,6 +2386,56 @@ public class MediaCodecVideoRendererTest {
.build();
}
+ private static final class ForwardingSynchronousMediaCodecAdapterWithReordering
+ extends ForwardingSynchronousMediaCodecAdapter {
+ /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithReordering} instances. */
+ public static final class Factory implements MediaCodecAdapter.Factory {
+ @Override
+ public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException {
+ return new ForwardingSynchronousMediaCodecAdapterWithReordering(
+ new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration));
+ }
+ }
+
+ private final PriorityQueue timestamps;
+
+ ForwardingSynchronousMediaCodecAdapterWithReordering(MediaCodecAdapter adapter) {
+ super(adapter);
+ timestamps = new PriorityQueue<>();
+ }
+
+ @Override
+ public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
+ int outputBufferIndex = super.dequeueOutputBufferIndex(bufferInfo);
+ Long smallestTimestamp = timestamps.peek();
+ if (smallestTimestamp != null && outputBufferIndex != INFO_TRY_AGAIN_LATER) {
+ bufferInfo.presentationTimeUs = smallestTimestamp;
+ }
+ return outputBufferIndex;
+ }
+
+ @Override
+ public void queueInputBuffer(
+ int index, int offset, int size, long presentationTimeUs, int flags) {
+ if ((flags & C.BUFFER_FLAG_END_OF_STREAM) == 0) {
+ timestamps.add(presentationTimeUs);
+ }
+ super.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
+ }
+
+ @Override
+ public void releaseOutputBuffer(int index, boolean render) {
+ timestamps.poll();
+ super.releaseOutputBuffer(index, render);
+ }
+
+ @Override
+ public void releaseOutputBuffer(int index, long renderTimeStampNs) {
+ timestamps.poll();
+ super.releaseOutputBuffer(index, renderTimeStampNs);
+ }
+ }
+
private static final class ForwardingSynchronousMediaCodecAdapterWithBufferLimit
extends ForwardingSynchronousMediaCodecAdapter {
/** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithBufferLimit} instances. */