loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

MCVR: drop decoder input buffers when the decoder is late

Browse Source 
* Add experimentalSetMinEarlyUsToDropDecoderInput to DefaultRenderersFactory
and MediaCodecVideoRenderer
* Enable dropping decoder input buffers inside MCVR.shouldIgnoreFrame
* Track consecutive dropped buffers via priority queue for reordering

PiperOrigin-RevId: 723837356
This commit is contained in:
dancho 2025-02-06 01:36:53 -08:00 committed by Copybara-Service
parent baf46d36d9
commit a56a0bd928
4 changed files with 538 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
RELEASENOTES.md
View File

@ -10,6 +10,9 @@
* DataSource:
* Audio:
* Video:
* Add experimental `ExoPlayer` API to drop late `MediaCodecVideoRenderer`
decoder input buffers that are not depended on. Enable it with
`DefaultRenderersFactory.experimentalSetLateThresholdToDropDecoderInputUs`.
* Text:
* Metadata:
* Image:

22
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java
View File

@ -110,6 +110,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
private boolean enableAudioTrackPlaybackParams;
private boolean enableMediaCodecVideoRendererPrewarming;
private boolean parseAv1SampleDependencies;
private long lateThresholdToDropDecoderInputUs;
/**
* @param context A {@link Context}.
@ -120,6 +121,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
extensionRendererMode = EXTENSION_RENDERER_MODE_OFF;
allowedVideoJoiningTimeMs = DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS;
mediaCodecSelector = MediaCodecSelector.DEFAULT;
lateThresholdToDropDecoderInputUs = C.TIME_UNSET;
}
/**
@ -313,6 +315,24 @@ public class DefaultRenderersFactory implements RenderersFactory {
return this;
}
/**
* Sets the late threshold for rendered output buffers, in microseconds, after which decoder input
* buffers may be dropped.
*
* <p>The default value is {@link C#TIME_UNSET} and therefore no input buffers will be dropped due
* to this logic.
*
* <p>This method is experimental and will be renamed or removed in a future release.
*
* @param lateThresholdToDropDecoderInputUs The threshold.
*/
@CanIgnoreReturnValue
public final DefaultRenderersFactory experimentalSetLateThresholdToDropDecoderInputUs(
long lateThresholdToDropDecoderInputUs) {
this.lateThresholdToDropDecoderInputUs = lateThresholdToDropDecoderInputUs;
return this;
}
@Override
public Renderer[] createRenderers(
Handler eventHandler,
@ -396,6 +416,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)
.experimentalSetParseAv1SampleDependencies(parseAv1SampleDependencies)
.experimentalSetLateThresholdToDropDecoderInputUs(lateThresholdToDropDecoderInputUs)
.build();
out.add(videoRenderer);
@ -800,6 +821,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)
.experimentalSetParseAv1SampleDependencies(parseAv1SampleDependencies)
.experimentalSetLateThresholdToDropDecoderInputUs(lateThresholdToDropDecoderInputUs)
.build();
}
return null;

77
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -82,6 +82,7 @@ import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.PriorityQueue;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@ -159,6 +160,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo;
@Nullable private final Av1SampleDependencyParser av1SampleDependencyParser;
/**
* The earliest time threshold, in microseconds, after which decoder input buffers may be dropped.
*/
private final long minEarlyUsToDropDecoderInput;
private final PriorityQueue<Long> droppedDecoderInputBufferTimestamps;
private @MonotonicNonNull CodecMaxValues codecMaxValues;
private boolean codecNeedsSetOutputSurfaceWorkaround;
private boolean codecHandlesHdr10PlusOutOfBandMetadata;
@ -190,6 +198,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private long periodDurationUs;
private boolean pendingVideoSinkInputStreamChange;
private boolean shouldDropDecoderInputBuffers;
/** A builder to create {@link MediaCodecVideoRenderer} instances. */
public static final class Builder {
private final Context context;
@ -204,6 +214,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private float assumedMinimumCodecOperatingRate;
@Nullable private VideoSink videoSink;
private boolean parseAv1SampleDependencies;
private long lateThresholdToDropDecoderInputUs;
/**
* Creates a new builder.
@ -215,6 +226,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
this.mediaCodecSelector = MediaCodecSelector.DEFAULT;
this.codecAdapterFactory = MediaCodecAdapter.Factory.getDefault(context);
this.assumedMinimumCodecOperatingRate = 30;
this.lateThresholdToDropDecoderInputUs = C.TIME_UNSET;
}
/** Sets the {@link MediaCodecSelector decoder selector}. */
@ -327,6 +339,22 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
return this;
}
/**
* Sets the late threshold for rendered output buffers, in microseconds, after which decoder
* input buffers may be dropped.
*
* <p>The default value is {@link C#TIME_UNSET} and therefore no input buffers will be dropped
* due to this logic.
*
* <p>This method is experimental and will be renamed or removed in a future release.
*/
@CanIgnoreReturnValue
public Builder experimentalSetLateThresholdToDropDecoderInputUs(
long lateThresholdToDropDecoderInputUs) {
this.lateThresholdToDropDecoderInputUs = lateThresholdToDropDecoderInputUs;
return this;
}
/**
* Builds the {@link MediaCodecVideoRenderer}. Must only be called once per Builder instance.
*
@ -546,6 +574,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
periodDurationUs = C.TIME_UNSET;
av1SampleDependencyParser =
builder.parseAv1SampleDependencies ? new Av1SampleDependencyParser() : null;
droppedDecoderInputBufferTimestamps = new PriorityQueue<>();
minEarlyUsToDropDecoderInput =
builder.lateThresholdToDropDecoderInputUs != C.TIME_UNSET
? -builder.lateThresholdToDropDecoderInputUs
: C.TIME_UNSET;
}
// FrameTimingEvaluator methods
@ -568,6 +601,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
boolean isLastFrame,
boolean treatDroppedBuffersAsSkipped)
throws ExoPlaybackException {
if (minEarlyUsToDropDecoderInput != C.TIME_UNSET) {
shouldDropDecoderInputBuffers = earlyUs < minEarlyUsToDropDecoderInput;
}
return shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastFrame)
&& maybeDropBuffersToKeyframe(positionUs, treatDroppedBuffersAsSkipped);
}
@ -1215,6 +1251,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
@Override
protected void resetCodecStateForFlush() {
super.resetCodecStateForFlush();
droppedDecoderInputBufferTimestamps.clear();
shouldDropDecoderInputBuffers = false;
buffersInCodecCount = 0;
}
@ -1425,8 +1463,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
// block processed. Skipping input buffers before the decoder is not allowed.
return false;
}
// Skip buffers without sample dependencies that won't be rendered.
if (!isBufferBeforeStartTime(buffer)) {
boolean shouldSkipDecoderInputBuffer = isBufferBeforeStartTime(buffer);
if (!shouldSkipDecoderInputBuffer && !shouldDropDecoderInputBuffers) {
return false;
}
if (buffer.hasSupplementalData()) {
@ -1434,7 +1472,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
}
if (buffer.notDependedOn()) {
buffer.clear();
decoderCounters.skippedInputBufferCount += 1;
if (shouldSkipDecoderInputBuffer) {
decoderCounters.skippedInputBufferCount += 1;
} else if (shouldDropDecoderInputBuffers) {
droppedDecoderInputBufferTimestamps.add(buffer.timeUs);
}
return true;
}
if (av1SampleDependencyParser != null
@ -1450,7 +1492,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
if (sampleLimitAfterSkippingNonReferenceFrames != readOnlySample.limit()
&& hasSpaceForNextFrame) {
checkNotNull(buffer.data).position(sampleLimitAfterSkippingNonReferenceFrames);
decoderCounters.skippedInputBufferCount += 1;
if (shouldSkipDecoderInputBuffer) {
decoderCounters.skippedInputBufferCount += 1;
} else if (shouldDropDecoderInputBuffers) {
droppedDecoderInputBufferTimestamps.add(buffer.timeUs);
}
return true;
}
return false;
@ -1595,6 +1641,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
long outputStreamOffsetUs = getOutputStreamOffsetUs();
long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
updateDroppedBufferCountersWithInputBuffers(presentationTimeUs);
if (videoSink != null) {
// Skip decode-only buffers, e.g. after seeking, immediately.
@ -1867,10 +1914,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
if (treatDroppedBuffersAsSkipped) {
decoderCounters.skippedInputBufferCount += droppedSourceBufferCount;
decoderCounters.skippedOutputBufferCount += buffersInCodecCount;
decoderCounters.skippedInputBufferCount += droppedDecoderInputBufferTimestamps.size();
} else {
decoderCounters.droppedToKeyframeCount++;
updateDroppedBufferCounters(
droppedSourceBufferCount, /* droppedDecoderBufferCount= */ buffersInCodecCount);
/* droppedInputBufferCount= */ droppedSourceBufferCount
+ droppedDecoderInputBufferTimestamps.size(),
/* droppedDecoderBufferCount= */ buffersInCodecCount);
}
flushOrReinitializeCodec();
if (videoSink != null) {
@ -1901,6 +1951,23 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
}
}
/**
* Updates counters to reflect dropped input buffers prior to {@code presentationTimeUs}.
*
* @param presentationTimeUs The presentation timestamp of the last processed output buffer, in
* microseconds.
*/
private void updateDroppedBufferCountersWithInputBuffers(long presentationTimeUs) {
int droppedInputBufferCount = 0;
Long minDroppedDecoderBufferTimeUs;
while ((minDroppedDecoderBufferTimeUs = droppedDecoderInputBufferTimestamps.peek()) != null
&& minDroppedDecoderBufferTimeUs < presentationTimeUs) {
droppedInputBufferCount++;
droppedDecoderInputBufferTimestamps.poll();
}
updateDroppedBufferCounters(droppedInputBufferCount, /* droppedDecoderBufferCount= */ 0);
}
/**
* Updates local counters and {@link DecoderCounters} with a new video frame processing offset.
*

441
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.exoplayer.video;
import static android.media.MediaCodec.INFO_TRY_AGAIN_LATER;
import static android.view.Display.DEFAULT_DISPLAY;
import static androidx.media3.common.util.Util.msToUs;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM;
@ -90,6 +91,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.PriorityQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@ -960,6 +962,395 @@ public class MediaCodecVideoRendererTest {
assertThat(currentOutputFormat).isEqualTo(VIDEO_H264);
}
@Test
public void render_withLateBufferWithoutDependencies_dropsInputBuffers() throws Exception {
FakeTimeline fakeTimeline =
new FakeTimeline(
new FakeTimeline.TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 1_000_000));
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ VIDEO_H264,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
oneByteSample(
/* timeUs= */ 20_000))); // Late buffer triggers input buffer dropping.
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer =
new MediaCodecVideoRenderer(
new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
.setCodecAdapterFactory(
new DefaultMediaCodecAdapterFactory(
ApplicationProvider.getApplicationContext(),
() -> {
callbackThread = new HandlerThread("MCVRTest:MediaCodecAsyncAdapter");
return callbackThread;
},
() -> {
queueingThread = new HandlerThread("MCVRTest:MediaCodecQueueingThread");
return queueingThread;
}))
.setMediaCodecSelector(mediaCodecSelector)
.setAllowedJoiningTimeMs(0)
.setEnableDecoderFallback(false)
.setEventHandler(new Handler(testMainLooper))
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(1)
.experimentalSetLateThresholdToDropDecoderInputUs(50_000)) {
@Override
protected @Capabilities int supportsFormat(
MediaCodecSelector mediaCodecSelector, Format format) {
return RendererCapabilities.create(C.FORMAT_HANDLED);
}
};
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
mediaCodecVideoRenderer.setTimeline(fakeTimeline);
mediaCodecVideoRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {VIDEO_H264},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
shadowOf(testMainLooper).idle();
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
ArgumentCaptor.forClass(DecoderCounters.class);
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
mediaCodecVideoRenderer.start();
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
while (decoderCounters.renderedOutputBufferCount == 0) {
mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
}
// Ensure existing buffer will be ~280ms late and new (not yet read) buffers are available
// to be dropped.
int posUs = 300_000;
fakeSampleStream.append(
ImmutableList.of(
oneByteSample(/* timeUs= */ 30_000, C.BUFFER_FLAG_NOT_DEPENDED_ON), // Dropped on input.
oneByteSample(/* timeUs= */ 300_000), // Caught up - render.
oneByteSample(/* timeUs= */ 500_000), // Last buffer is always rendered.
END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer.setCurrentStreamFinal();
// Render until the non-dropped frame is reached and then increase time to reach the end.
while (decoderCounters.renderedOutputBufferCount < 2) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
}
while (!mediaCodecVideoRenderer.isEnded()) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
posUs += 2_000;
}
assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(1);
assertThat(decoderCounters.droppedBufferCount).isEqualTo(2);
assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isEqualTo(2);
assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0);
}
// TODO: b/390604981 - Run the test on older SDK levels to ensure it uses a MediaCodec shadow
// with more than one buffer slot.
@Config(minSdk = 30)
@Test
public void render_afterVeryLateBuffer_doesNotDropInputBuffers() throws Exception {
FakeTimeline fakeTimeline =
new FakeTimeline(
new FakeTimeline.TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 2_000_000));
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ VIDEO_H264,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
oneByteSample(/* timeUs= */ 20_000))); // Very late buffer.
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer =
new MediaCodecVideoRenderer(
new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
.setCodecAdapterFactory(
new DefaultMediaCodecAdapterFactory(
ApplicationProvider.getApplicationContext(),
() -> {
callbackThread = new HandlerThread("MCVRTest:MediaCodecAsyncAdapter");
return callbackThread;
},
() -> {
queueingThread = new HandlerThread("MCVRTest:MediaCodecQueueingThread");
return queueingThread;
}))
.setMediaCodecSelector(mediaCodecSelector)
.setAllowedJoiningTimeMs(0)
.setEnableDecoderFallback(false)
.setEventHandler(new Handler(testMainLooper))
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(1)
.experimentalSetLateThresholdToDropDecoderInputUs(50_000)) {
@Override
protected @Capabilities int supportsFormat(
MediaCodecSelector mediaCodecSelector, Format format) {
return RendererCapabilities.create(C.FORMAT_HANDLED);
}
};
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
mediaCodecVideoRenderer.setTimeline(fakeTimeline);
mediaCodecVideoRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {VIDEO_H264},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
shadowOf(testMainLooper).idle();
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
ArgumentCaptor.forClass(DecoderCounters.class);
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
mediaCodecVideoRenderer.start();
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
while (decoderCounters.renderedOutputBufferCount == 0) {
mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
}
// Ensure existing buffer will be 1 second late and new (not yet read) buffers are available
// to be skipped and to skip to in the input stream.
int posUs = 1_020_000;
fakeSampleStream.append(
ImmutableList.of(
oneByteSample(/* timeUs= */ 30_000), // Dropped input buffer when skipping to keyframe.
oneByteSample(/* timeUs= */ 1_020_000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 1_040_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
oneByteSample(/* timeUs= */ 1_060_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
oneByteSample(/* timeUs= */ 2_000_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer.setCurrentStreamFinal();
// Render until the new keyframe has been processed and then increase time to reach the end.
while (decoderCounters.renderedOutputBufferCount < 2) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
maybeIdleAsynchronousMediaCodecAdapterThreads();
}
while (!mediaCodecVideoRenderer.isEnded()) {
maybeIdleAsynchronousMediaCodecAdapterThreads();
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
posUs += 1_000;
}
assertThat(decoderCounters.renderedOutputBufferCount).isEqualTo(5);
assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(1);
assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(1);
}
@Test
public void render_afterVeryLateBuffer_countsDroppedInputBuffersCorrectly() throws Exception {
FakeTimeline fakeTimeline =
new FakeTimeline(
new FakeTimeline.TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 3_000_000));
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ VIDEO_H264,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
oneByteSample(/* timeUs= */ 20_000))); // Late buffer.
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer =
new MediaCodecVideoRenderer(
new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
.setCodecAdapterFactory(
new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory())
.setMediaCodecSelector(mediaCodecSelector)
.setAllowedJoiningTimeMs(0)
.setEnableDecoderFallback(false)
.setEventHandler(new Handler(testMainLooper))
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(1)
.experimentalSetLateThresholdToDropDecoderInputUs(50_000)) {
@Override
protected @Capabilities int supportsFormat(
MediaCodecSelector mediaCodecSelector, Format format) {
return RendererCapabilities.create(C.FORMAT_HANDLED);
}
};
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
mediaCodecVideoRenderer.setTimeline(fakeTimeline);
mediaCodecVideoRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {VIDEO_H264},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
shadowOf(testMainLooper).idle();
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
ArgumentCaptor.forClass(DecoderCounters.class);
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
mediaCodecVideoRenderer.start();
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
while (decoderCounters.renderedOutputBufferCount == 0) {
mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
}
fakeSampleStream.append(
ImmutableList.of(
// Dropped input buffer.
oneByteSample(/* timeUs= */ 500_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
oneByteSample(300_000), // Render.
oneByteSample(400_000) // Very late buffer.
));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
// Render until the frame at time 300_000 us is displayed.
int posUs = 300_000;
while (decoderCounters.renderedOutputBufferCount < 2) {
maybeIdleAsynchronousMediaCodecAdapterThreads();
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
}
fakeSampleStream.append(
ImmutableList.of(
oneByteSample(
/* timeUs= */ 1_000_000), // Dropped input buffer when skipping to keyframe.
oneByteSample(/* timeUs= */ 2_020_000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 3_000_000),
END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer.setCurrentStreamFinal();
posUs = 2_020_000;
while (decoderCounters.renderedOutputBufferCount + decoderCounters.skippedOutputBufferCount
< 3) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
maybeIdleAsynchronousMediaCodecAdapterThreads();
}
while (!mediaCodecVideoRenderer.isEnded()) {
maybeIdleAsynchronousMediaCodecAdapterThreads();
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
posUs += 1_000;
}
assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(2);
assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(1);
}
@Test
public void
render_withLateBufferAndOutOfOrderSamplesWithoutDependencies_dropsInputBuffersAndRendersLast()
throws Exception {
FakeTimeline fakeTimeline =
new FakeTimeline(
new FakeTimeline.TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 1_000_000));
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ VIDEO_H264,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
oneByteSample(
/* timeUs= */ 20_000))); // Late buffer triggers input buffer dropping.
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer =
new MediaCodecVideoRenderer(
new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
.setCodecAdapterFactory(
new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory())
.setMediaCodecSelector(mediaCodecSelector)
.setAllowedJoiningTimeMs(0)
.setEnableDecoderFallback(false)
.setEventHandler(new Handler(testMainLooper))
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(1)
.experimentalSetLateThresholdToDropDecoderInputUs(200_000)) {
@Override
protected @Capabilities int supportsFormat(
MediaCodecSelector mediaCodecSelector, Format format) {
return RendererCapabilities.create(C.FORMAT_HANDLED);
}
};
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
mediaCodecVideoRenderer.setTimeline(fakeTimeline);
mediaCodecVideoRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {VIDEO_H264},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
shadowOf(testMainLooper).idle();
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
ArgumentCaptor.forClass(DecoderCounters.class);
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
mediaCodecVideoRenderer.start();
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
while (decoderCounters.renderedOutputBufferCount == 0) {
mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
}
// Ensure existing buffer will be ~280ms late and new (not yet read) buffers are available
// to be dropped.
int posUs = 300_000;
fakeSampleStream.append(
ImmutableList.of(
oneByteSample(/* timeUs= */ 300_000), // Render.
oneByteSample(/* timeUs= */ 320_000), // Render.
// Drop consecutive input buffers that aren't consecutive output buffers.
oneByteSample(/* timeUs= */ 310_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
oneByteSample(/* timeUs= */ 330_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
// Last buffer is always rendered.
oneByteSample(/* timeUs= */ 500_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer.setCurrentStreamFinal();
// Render until the first frame is reached and then increase time to reach the end.
while (decoderCounters.renderedOutputBufferCount < 2) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
}
while (!mediaCodecVideoRenderer.isEnded()) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
posUs += 2_000;
}
assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(2);
assertThat(decoderCounters.droppedBufferCount).isEqualTo(3);
assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isEqualTo(1);
assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0);
}
@Test
public void enable_withMayRenderStartOfStream_rendersFirstFrameBeforeStart() throws Exception {
FakeSampleStream fakeSampleStream =
@ -1995,6 +2386,56 @@ public class MediaCodecVideoRendererTest {
.build();
}
private static final class ForwardingSynchronousMediaCodecAdapterWithReordering
extends ForwardingSynchronousMediaCodecAdapter {
/** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithReordering} instances. */
public static final class Factory implements MediaCodecAdapter.Factory {
@Override
public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException {
return new ForwardingSynchronousMediaCodecAdapterWithReordering(
new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration));
}
}
private final PriorityQueue<Long> timestamps;
ForwardingSynchronousMediaCodecAdapterWithReordering(MediaCodecAdapter adapter) {
super(adapter);
timestamps = new PriorityQueue<>();
}
@Override
public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
int outputBufferIndex = super.dequeueOutputBufferIndex(bufferInfo);
Long smallestTimestamp = timestamps.peek();
if (smallestTimestamp != null && outputBufferIndex != INFO_TRY_AGAIN_LATER) {
bufferInfo.presentationTimeUs = smallestTimestamp;
}
return outputBufferIndex;
}
@Override
public void queueInputBuffer(
int index, int offset, int size, long presentationTimeUs, int flags) {
if ((flags & C.BUFFER_FLAG_END_OF_STREAM) == 0) {
timestamps.add(presentationTimeUs);
}
super.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
}
@Override
public void releaseOutputBuffer(int index, boolean render) {
timestamps.poll();
super.releaseOutputBuffer(index, render);
}
@Override
public void releaseOutputBuffer(int index, long renderTimeStampNs) {
timestamps.poll();
super.releaseOutputBuffer(index, renderTimeStampNs);
}
}
private static final class ForwardingSynchronousMediaCodecAdapterWithBufferLimit
extends ForwardingSynchronousMediaCodecAdapter {
/** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithBufferLimit} instances. */