loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

MCVR: limit the number of consecutive dropped input frame headers

Browse Source 
Dropping too many consecutive input buffers reduces the update
frequency of MCVR.shouldDropDecoderInputBuffers and can lead to
dropping too many consecutive input buffers.

Discarding input buffers of type OBU_FRAME_HEADER with
show_existing_frame = 1 saves a smaller amount of resources
than discarding input buffers of type OBU_FRAME.
PiperOrigin-RevId: 730362707

(cherry picked from commit 67e99f46481dbc5fff9ffd04ff13119dac9199bb)
This commit is contained in:
dancho 2025-02-24 02:25:52 -08:00 committed by oceanjules
parent 8ee2532db3
commit 62ef83b68e
5 changed files with 249 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/Av1SampleDependencyParser.java
View File

@ -32,8 +32,9 @@ import java.util.List;
/** An AV1 bitstream parser that identifies frames that are not depended on. */
/* package */ final class Av1SampleDependencyParser {
/**
* When {@link #sampleLimitAfterSkippingNonReferenceFrame(ByteBuffer)} partially skips a temporal
* unit, the decoder input buffer is left with extra reference frames that need to be decoded.
* When {@link #sampleLimitAfterSkippingNonReferenceFrame(ByteBuffer, boolean)} partially skips a
* temporal unit, the decoder input buffer is left with extra reference frames that need to be
* decoded.
*
* <p>The AV1 spec defines {@code NUM_REF_FRAMES = 8} - delaying more than 8 reference frames will
* overwrite the same output slots.
@ -50,20 +51,22 @@ import java.util.List;
* that aren't shown are used as reference, but the shown frame may not be used as reference.
* Frequently, the shown frame is the last frame in the temporal unit.
*
* <p>If the last frame in the temporal unit is a non-reference {@link ObuParser#OBU_FRAME}, this
* method returns a new {@link ByteBuffer#limit()} value that would leave only the frames used as
* reference in the input {@code sample}.
* <p>If the last frame in the temporal unit is a non-reference {@link ObuParser#OBU_FRAME} or
* {@link ObuParser#OBU_FRAME_HEADER}, this method returns a new {@link ByteBuffer#limit()} value
* that would leave only the frames used as reference in the input {@code sample}.
*
* <p>See <a href=https://aomediacodec.github.io/av1-spec/#ordering-of-obus>Ordering of OBUs</a>.
*
* @param sample The sample data for one AV1 temporal unit.
* @param skipFrameHeaders Whether to skip {@link ObuParser#OBU_FRAME_HEADER}.
*/
public int sampleLimitAfterSkippingNonReferenceFrame(ByteBuffer sample) {
public int sampleLimitAfterSkippingNonReferenceFrame(
ByteBuffer sample, boolean skipFrameHeaders) {
List<ObuParser.Obu> obuList = split(sample);
updateSequenceHeaders(obuList);
int skippedFramesCount = 0;
int last = obuList.size() - 1;
while (last >= 0 && canSkipObu(obuList.get(last))) {
while (last >= 0 && canSkipObu(obuList.get(last), skipFrameHeaders)) {
if (obuList.get(last).type == OBU_FRAME || obuList.get(last).type == OBU_FRAME_HEADER) {
skippedFramesCount++;
}
@ -88,10 +91,13 @@ import java.util.List;
sequenceHeader = null;
}
private boolean canSkipObu(ObuParser.Obu obu) {
private boolean canSkipObu(ObuParser.Obu obu, boolean skipFrameHeaders) {
if (obu.type == OBU_TEMPORAL_DELIMITER || obu.type == OBU_PADDING) {
return true;
}
if (obu.type == OBU_FRAME_HEADER && !skipFrameHeaders) {
return false;
}
if ((obu.type == OBU_FRAME || obu.type == OBU_FRAME_HEADER) && sequenceHeader != null) {
FrameHeader frameHeader = FrameHeader.parse(sequenceHeader, obu);
return frameHeader != null && !frameHeader.isDependedOn();

25
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -60,6 +60,7 @@ import androidx.media3.common.util.Size;
import androidx.media3.common.util.TraceUtil;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.container.ObuParser;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.exoplayer.DecoderCounters;
import androidx.media3.exoplayer.DecoderReuseEvaluation;
@ -148,6 +149,18 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
*/
private static final long OFFSET_FROM_PERIOD_END_TO_TREAT_AS_LAST_US = 100_000L;
/**
* The maximum number of consecutive dropped input buffers that allow discarding frame headers.
*
* <p>Discarding input buffers of type {@link ObuParser#OBU_FRAME_HEADER} speeds up decoding by
* not showing already-decoded frames. This is less beneficial than discarding {@link
* ObuParser#OBU_FRAME} which reduces the total number of decoded frames.
*
* <p>Dropping too many consecutive input buffers reduces the update frequency of {@link
* #shouldDropDecoderInputBuffers}, and can harm user experience.
*/
private static final int MAX_CONSECUTIVE_DROPPED_INPUT_BUFFERS_COUNT_TO_DISCARD_HEADER = 0;
private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround;
private static boolean deviceNeedsSetOutputSurfaceWorkaround;
@ -199,6 +212,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private boolean pendingVideoSinkInputStreamChange;
private boolean shouldDropDecoderInputBuffers;
private int consecutiveDroppedInputBufferCount;
/** A builder to create {@link MediaCodecVideoRenderer} instances. */
public static final class Builder {
@ -1254,6 +1268,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
droppedDecoderInputBufferTimestamps.clear();
shouldDropDecoderInputBuffers = false;
buffersInCodecCount = 0;
consecutiveDroppedInputBufferCount = 0;
if (av1SampleDependencyParser != null) {
av1SampleDependencyParser.reset();
}
@ -1436,6 +1451,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
&& buffer.data != null) {
av1SampleDependencyParser.queueInputBuffer(buffer.data);
}
consecutiveDroppedInputBufferCount = 0;
// In tunneling mode the device may do frame rate conversion, so in general we can't keep track
// of the number of buffers in the codec.
if (!tunneling) {
@ -1484,16 +1500,22 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
decoderCounters.skippedInputBufferCount += 1;
} else if (shouldDropDecoderInputBuffers) {
droppedDecoderInputBufferTimestamps.add(buffer.timeUs);
consecutiveDroppedInputBufferCount += 1;
}
return true;
}
if (av1SampleDependencyParser != null
&& checkNotNull(getCodecInfo()).mimeType.equals(MimeTypes.VIDEO_AV1)
&& buffer.data != null) {
boolean skipFrameHeaders =
shouldSkipDecoderInputBuffer
|| consecutiveDroppedInputBufferCount
<= MAX_CONSECUTIVE_DROPPED_INPUT_BUFFERS_COUNT_TO_DISCARD_HEADER;
ByteBuffer readOnlySample = buffer.data.asReadOnlyBuffer();
readOnlySample.flip();
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(readOnlySample);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
readOnlySample, skipFrameHeaders);
boolean hasSpaceForNextFrame =
sampleLimitAfterSkippingNonReferenceFrames + checkNotNull(codecMaxValues).inputSize
< readOnlySample.capacity();
@ -1504,6 +1526,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
decoderCounters.skippedInputBufferCount += 1;
} else if (shouldDropDecoderInputBuffers) {
droppedDecoderInputBufferTimestamps.add(buffer.timeUs);
consecutiveDroppedInputBufferCount += 1;
}
return true;
}

154
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java
View File

@ -15,13 +15,27 @@
*/
package androidx.media3.exoplayer.e2etest;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.os.Handler;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MediaItem.ClippingConfiguration;
import androidx.media3.common.Player;
import androidx.media3.exoplayer.DecoderCounters;
import androidx.media3.exoplayer.DefaultRenderersFactory;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.audio.AudioRendererEventListener;
import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.metadata.MetadataOutput;
import androidx.media3.exoplayer.text.TextOutput;
import androidx.media3.exoplayer.video.MediaCodecVideoRenderer;
import androidx.media3.exoplayer.video.VideoRendererEventListener;
import androidx.media3.test.utils.CapturingRenderersFactory;
import androidx.media3.test.utils.DumpFileAsserts;
import androidx.media3.test.utils.FakeClock;
@ -33,6 +47,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.annotation.Config;
/** End-to-end playback tests using AV1 sample skipping. */
@RunWith(AndroidJUnit4.class)
@ -74,4 +89,143 @@ public class ParseAv1SampleDependenciesPlaybackTest {
playbackOutput,
/* dumpFile= */ "playbackdumps/av1SampleDependencies/clippedMediaItem.dump");
}
// TODO: b/390604981 - Run the test on older SDK levels to ensure it uses a MediaCodec shadow
// with more than one buffer slot.
@Config(minSdk = 30)
@Test
public void playback_withLateThresholdToDropDecoderInput_skipNonReferenceInputSamples()
throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactoryWithLateThresholdToDropDecoderInputUs renderersFactory =
new CapturingRenderersFactoryWithLateThresholdToDropDecoderInputUs(applicationContext);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, renderersFactory)
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.build();
Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1));
player.setVideoSurface(surface);
player.setMediaItem(MediaItem.fromUri(TEST_MP4_URI));
player.prepare();
player.play();
TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
player.release();
surface.release();
DecoderCounters decoderCounters = player.getVideoDecoderCounters();
// Do not assert on a full playback dump as it depends on the number of MediaCodec buffer
// slots, which may change in b/390604981.
// Half of the input buffers are non-reference OBU_FRAME which will be dropped.
// The other half are non-reference OBU_FRAME_HEADER - only the first one may be dropped.
// Which input buffer is dropped first depends on the number of MediaCodec buffer slots.
// This means the asserts cannot be isEqualTo.
assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isAtMost(2);
assertThat(decoderCounters.droppedInputBufferCount).isAtLeast(8);
}
private static final class CapturingRenderersFactoryWithLateThresholdToDropDecoderInputUs
extends CapturingRenderersFactory {
private final Context context;
/**
* Creates an instance.
*
* @param context The {@link Context}.
*/
public CapturingRenderersFactoryWithLateThresholdToDropDecoderInputUs(Context context) {
super(context);
this.context = context;
}
@Override
public Renderer[] createRenderers(
Handler eventHandler,
VideoRendererEventListener videoRendererEventListener,
AudioRendererEventListener audioRendererEventListener,
TextOutput textRendererOutput,
MetadataOutput metadataRendererOutput) {
return new Renderer[] {
new CapturingMediaCodecVideoRenderer(
context,
getMediaCodecAdapterFactory(),
MediaCodecSelector.DEFAULT,
DefaultRenderersFactory.DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS,
/* enableDecoderFallback= */ false,
eventHandler,
videoRendererEventListener,
DefaultRenderersFactory.MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
/* parseAv1SampleDependencies= */ true,
/* lateThresholdToDropDecoderInputUs= */ -100_000_000L)
};
}
/**
* A {@link MediaCodecVideoRenderer} that will not skip or drop buffers due to slow processing.
*/
private static class CapturingMediaCodecVideoRenderer extends MediaCodecVideoRenderer {
private CapturingMediaCodecVideoRenderer(
Context context,
MediaCodecAdapter.Factory codecAdapterFactory,
MediaCodecSelector mediaCodecSelector,
long allowedJoiningTimeMs,
boolean enableDecoderFallback,
@Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify,
boolean parseAv1SampleDependencies,
long lateThresholdToDropDecoderInputUs) {
super(
new Builder(context)
.setCodecAdapterFactory(codecAdapterFactory)
.setMediaCodecSelector(mediaCodecSelector)
.setAllowedJoiningTimeMs(allowedJoiningTimeMs)
.setEnableDecoderFallback(enableDecoderFallback)
.setEventHandler(eventHandler)
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(maxDroppedFramesToNotify)
.experimentalSetParseAv1SampleDependencies(parseAv1SampleDependencies)
.experimentalSetLateThresholdToDropDecoderInputUs(
lateThresholdToDropDecoderInputUs));
}
@Override
protected boolean shouldDropOutputBuffer(
long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
// Do not drop output buffers due to slow processing.
return false;
}
@Override
protected boolean shouldDropBuffersToKeyframe(
long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
// Do not drop output buffers due to slow processing.
return false;
}
@Override
protected boolean shouldSkipBuffersWithIdenticalReleaseTime() {
// Do not skip buffers with identical vsync times as we can't control this from tests.
return false;
}
@Override
protected boolean shouldSkipLateBuffersWhileUsingPlaceholderSurface() {
// Do not skip buffers while using placeholder surface due to slow processing.
return false;
}
@Override
protected boolean shouldForceRenderOutputBuffer(long earlyUs, long elapsedSinceLastRenderUs) {
// An auto-advancing FakeClock can make a lot of progress before
// AsynchronousMediaCodecAdapter produces an output buffer - causing all output buffers to
// be force rendered.
// Force rendering output buffers prevents evaluation of lateThresholdToDropDecoderInputUs.
// Do not allow force rendering of output buffers when testing
// lateThresholdToDropDecoderInputUs.
return false;
}
}
}
}

59
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java
View File

@ -43,6 +43,8 @@ public class Av1SampleDependencyParserTest {
0x32, 0x1A, 0x30, 0xC0, 0x00, 0x1D, 0x66, 0x68, 0x46, 0xC9, 0x38, 0x00, 0x60, 0x10, 0x20,
0x80, 0x20, 0x00, 0x00, 0x01, 0x8B, 0x7A, 0x87, 0xF9, 0xAA, 0x2D, 0x0F, 0x2C);
private static final byte[] frameHeader = createByteArray(0x1A, 0x01, 0xC8);
private static final byte[] temporalDelimiter = createByteArray(0x12, 0x00);
private static final byte[] padding = createByteArray(0x7a, 0x02, 0xFF, 0xFF);
@ -56,7 +58,8 @@ public class Av1SampleDependencyParserTest {
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames)
.isEqualTo(sequenceHeader.length + dependedOnFrame.length);
@ -72,7 +75,8 @@ public class Av1SampleDependencyParserTest {
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(sequenceHeader.length);
}
@ -86,7 +90,8 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(frame);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
frame, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(0);
}
@ -104,7 +109,8 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(0);
}
@ -123,7 +129,8 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames)
.isEqualTo(temporalDelimiter.length + padding.length + dependedOnFrame.length);
@ -135,7 +142,8 @@ public class Av1SampleDependencyParserTest {
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(frame);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
frame, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(notDependedOnFrame.length);
}
@ -152,7 +160,8 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames)
.isEqualTo(notDependedOnFrame.length + notDependedOnFrame.length);
@ -174,7 +183,8 @@ public class Av1SampleDependencyParserTest {
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(sample.limit());
}
@ -189,8 +199,39 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
av1SampleDependencyParser.reset();
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(frame);
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
frame, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(notDependedOnFrame.length);
}
@Test
public void
sampleLimitAfterSkippingNonReferenceFrame_withSkipFrameHeadersTrue_returnsEmptySample() {
ByteBuffer header = ByteBuffer.wrap(sequenceHeader);
ByteBuffer sample = ByteBuffer.wrap(frameHeader);
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrame =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrame).isEqualTo(0);
}
@Test
public void
sampleLimitAfterSkippingNonReferenceFrame_withSkipFrameHeadersFalse_returnsFullSample() {
ByteBuffer header = ByteBuffer.wrap(sequenceHeader);
ByteBuffer sample = ByteBuffer.wrap(frameHeader);
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrame =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
sample, /* skipFrameHeaders= */ false);
assertThat(sampleLimitAfterSkippingNonReferenceFrame).isEqualTo(frameHeader.length);
}
}

7
libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
View File

@ -213,6 +213,13 @@ public class CapturingRenderersFactory implements RenderersFactory, Dumper.Dumpa
/* parseAv1SampleDependencies= */ false);
}
/**
* Returns the {@link CapturingMediaCodecAdapter.Factory} as a {@link MediaCodecAdapter.Factory}.
*/
protected MediaCodecAdapter.Factory getMediaCodecAdapterFactory() {
return mediaCodecAdapterFactory;
}
/**
* A {@link MediaCodecVideoRenderer} that will not skip or drop buffers due to slow processing.
*/