loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MediaCodecVideoRenderer skips decoder inputs unused as reference

Browse Source 
During a seek, or when playing a media with clipped start,
MCVR encounters preroll decode-only buffers that are not rendered.
Use C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS to determine
whether a decode-only buffer is unused as reference.
These buffers can be dropped before the decoder.
When this optimization is triggered, increment
decoderCounters.skippedInputBufferCount.

Tested in ExoPlayer demo app on "One hour frame counter (MP4)"
after enabling extractorsFactory.setMp4ExtractorFlags(
    FLAG_READ_WITHIN_GOP_SAMPLE_DEPENDENCIES);
Observe: "sib" increases on each seek.
PiperOrigin-RevId: 650566216
This commit is contained in:
dancho 2024-07-09 03:40:23 -07:00 committed by Copybara-Service
parent 7d4f623b00
commit 6650270a4e
8 changed files with 122 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -19,6 +19,8 @@
* Add `PreloadMediaSource.PreloadControl.onPreloadError` to allow
`PreloadMediaSource.PreloadControl` implementations to take actions when
error occurs.
* `MediaCodecVideoRenderer` avoids decoding samples that are neither
rendered nor used as reference by other samples.
* Transformer:
* Add `SurfaceAssetLoader`, which supports queueing video data to
Transformer via a `Surface`.

9
libraries/common/src/main/java/androidx/media3/common/C.java
View File

@ -620,7 +620,7 @@ public final class C {
* <ul>
* <li>{@link #BUFFER_FLAG_KEY_FRAME}
* <li>{@link #BUFFER_FLAG_END_OF_STREAM}
* <li>{@link #BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}
* <li>{@link #BUFFER_FLAG_NOT_DEPENDED_ON}
* <li>{@link #BUFFER_FLAG_FIRST_SAMPLE}
* <li>{@link #BUFFER_FLAG_LAST_SAMPLE}
* <li>{@link #BUFFER_FLAG_ENCRYPTED}
@ -635,7 +635,7 @@ public final class C {
value = {
BUFFER_FLAG_KEY_FRAME,
BUFFER_FLAG_END_OF_STREAM,
BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS,
BUFFER_FLAG_NOT_DEPENDED_ON,
BUFFER_FLAG_FIRST_SAMPLE,
BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA,
BUFFER_FLAG_LAST_SAMPLE,
@ -650,9 +650,8 @@ public final class C {
@UnstableApi
public static final int BUFFER_FLAG_END_OF_STREAM = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
/** Indicates that future buffers do not depend on the data in this buffer. */
@UnstableApi
public static final int BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS = 1 << 26; // 0x04000000
/** Indicates that no other buffers depend on the data in this buffer. */
@UnstableApi public static final int BUFFER_FLAG_NOT_DEPENDED_ON = 1 << 26; // 0x04000000
/** Indicates that a buffer is known to contain the first media sample of the stream. */
@UnstableApi public static final int BUFFER_FLAG_FIRST_SAMPLE = 1 << 27; // 0x08000000

5
libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java
View File

@ -60,6 +60,11 @@ public abstract class Buffer {
return getFlag(C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA);
}
/** Returns whether the {@link C#BUFFER_FLAG_NOT_DEPENDED_ON} flag is set. */
public final boolean notDependedOn() {
return getFlag(C.BUFFER_FLAG_NOT_DEPENDED_ON);
}
/**
* Replaces this buffer's flags with {@code flags}.
*

18
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
View File

@ -1452,6 +1452,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
return true;
}
if (shouldSkipDecoderInputBuffer(buffer)) {
buffer.clear();
decoderCounters.skippedInputBufferCount += 1;
return true;
}
boolean bufferEncrypted = buffer.isEncrypted();
if (bufferEncrypted) {
buffer.cryptoInfo.increaseClearDataFirstSubSampleBy(adaptiveReconfigurationBytes);
@ -1754,6 +1760,18 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
return 0;
}
/**
* Returns whether the input buffer should be skipped before the decoder.
*
* <p>This can be used to skip decoding of buffers that are not depended on during seeking. See
* {@link C#BUFFER_FLAG_NOT_DEPENDED_ON}.
*
* @param buffer The input buffer.
*/
protected boolean shouldSkipDecoderInputBuffer(DecoderInputBuffer buffer) {
return false;
}
/**
* Returns the presentation time of the last buffer in the stream.
*

23
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -1247,7 +1247,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
@Override
protected int getCodecBufferFlags(DecoderInputBuffer buffer) {
if (Util.SDK_INT >= 34 && tunneling && buffer.timeUs < getLastResetPositionUs()) {
if (Util.SDK_INT >= 34 && tunneling && isBufferBeforeStartTime(buffer)) {
// The buffer likely needs to be dropped because its timestamp is less than the start time.
// We can't decide to do this after decoding because we won't get the buffer back from the
// codec in tunneling mode. This may not work perfectly, e.g. when the codec is doing frame
@ -1257,6 +1257,27 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
return 0;
}
@Override
protected boolean shouldSkipDecoderInputBuffer(DecoderInputBuffer buffer) {
// TODO: b/351164714 - Do not apply this optimization for buffers with timestamp near
// the media duration.
if (hasReadStreamToEnd() || buffer.isLastSample()) {
// Last buffer is always decoded.
return false;
}
if (buffer.isEncrypted()) {
// Commonly used decryption algorithms require updating the initialization vector for each
// block processed. Skipping input buffers before the decoder is not allowed.
return false;
}
// Skip buffers without sample dependencies that won't be rendered.
return isBufferBeforeStartTime(buffer) && buffer.notDependedOn();
}
private boolean isBufferBeforeStartTime(DecoderInputBuffer buffer) {
return buffer.timeUs < getLastResetPositionUs();
}
@Override
protected void onOutputFormatChanged(Format format, @Nullable MediaFormat mediaFormat) {
@Nullable MediaCodecAdapter codec = getCodec();

67
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
View File

@ -356,6 +356,73 @@ public class MediaCodecVideoRendererTest {
assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2);
}
@Test
public void render_withoutSampleDependencies_rendersLastFrameAfterEndOfStream() throws Exception {
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
ArgumentCaptor.forClass(DecoderCounters.class);
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ VIDEO_H264,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
// Second buffer will be skipped before decoder during a seek.
oneByteSample(/* timeUs= */ 10_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
// Last buffer without sample dependencies will be rendered.
oneByteSample(/* timeUs= */ 20_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
// Seek to time after samples.
fakeSampleStream.seekToUs(30_000, /* allowTimeBeyondBuffer= */ true);
mediaCodecVideoRenderer =
new MediaCodecVideoRenderer(
ApplicationProvider.getApplicationContext(),
new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3),
mediaCodecSelector,
/* allowedJoiningTimeMs= */ 0,
/* enableDecoderFallback= */ false,
/* eventHandler= */ new Handler(testMainLooper),
eventListener,
/* maxDroppedFramesToNotify= */ 1);
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
mediaCodecVideoRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {VIDEO_H264},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 30_000,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(new Object()));
mediaCodecVideoRenderer.start();
mediaCodecVideoRenderer.setCurrentStreamFinal();
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
// Call to render has reads all samples including the END_OF_STREAM_ITEM because the
// previous sample is skipped before decoding.
assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue();
int posUs = 30_000;
while (!mediaCodecVideoRenderer.isEnded()) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
posUs += 40_000;
}
shadowOf(testMainLooper).idle();
verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
// First key frame is decoded and skipped as an output buffer.
assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(1);
// Second frame is skipped before the decoder, as an input buffer.
assertThat(argumentDecoderCounters.getValue().skippedInputBufferCount).isEqualTo(1);
// Last frame is rendered.
assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1);
}
@Test
public void
render_withClippingMediaPeriodAndBufferContainingLastAndClippingSamples_rendersLastFrame()

4
libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java
View File

@ -135,7 +135,7 @@ public class FragmentedMp4Extractor implements Extractor {
/**
* Flag to extract additional sample dependency information, and mark output buffers with {@link
* C#BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}.
* C#BUFFER_FLAG_NOT_DEPENDED_ON}.
*
* <p>This class always marks the samples at the start of each group of picture (GOP) with {@link
* C#BUFFER_FLAG_KEY_FRAME}. Usually, key frames can be decoded independently, without depending
@ -1661,7 +1661,7 @@ public class FragmentedMp4Extractor implements Extractor {
@C.BufferFlags int sampleFlags = trackBundle.getCurrentSampleFlags();
if ((flags & FLAG_READ_WITHIN_GOP_SAMPLE_DEPENDENCIES) != 0 && !isSampleDependedOn) {
sampleFlags |= C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS;
sampleFlags |= C.BUFFER_FLAG_NOT_DEPENDED_ON;
}
// Encryption data.

4
libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java
View File

@ -125,7 +125,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
/**
* Flag to extract additional sample dependency information, and mark output buffers with {@link
* C#BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}.
* C#BUFFER_FLAG_NOT_DEPENDED_ON}.
*
* <p>This class always marks the samples at the start of each group of picture (GOP) with {@link
* C#BUFFER_FLAG_KEY_FRAME}. Usually, key frames can be decoded independently, without depending
@ -804,7 +804,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
long timeUs = track.sampleTable.timestampsUs[sampleIndex];
@C.BufferFlags int sampleFlags = track.sampleTable.flags[sampleIndex];
if (!isSampleDependedOn) {
sampleFlags |= C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS;
sampleFlags |= C.BUFFER_FLAG_NOT_DEPENDED_ON;
}
if (trueHdSampleRechunker != null) {
trueHdSampleRechunker.sampleMetadata(