The AV1 spec defines {@code NUM_REF_FRAMES = 8} - delaying more than 8 reference frames will * overwrite the same output slots. @@ -50,20 +51,22 @@ import java.util.List; * that aren't shown are used as reference, but the shown frame may not be used as reference. * Frequently, the shown frame is the last frame in the temporal unit. * - *
If the last frame in the temporal unit is a non-reference {@link ObuParser#OBU_FRAME}, this - * method returns a new {@link ByteBuffer#limit()} value that would leave only the frames used as - * reference in the input {@code sample}. + *
If the last frame in the temporal unit is a non-reference {@link ObuParser#OBU_FRAME} or + * {@link ObuParser#OBU_FRAME_HEADER}, this method returns a new {@link ByteBuffer#limit()} value + * that would leave only the frames used as reference in the input {@code sample}. * *
See Ordering of OBUs.
*
* @param sample The sample data for one AV1 temporal unit.
+ * @param skipFrameHeaders Whether to skip {@link ObuParser#OBU_FRAME_HEADER}.
*/
- public int sampleLimitAfterSkippingNonReferenceFrame(ByteBuffer sample) {
+ public int sampleLimitAfterSkippingNonReferenceFrame(
+ ByteBuffer sample, boolean skipFrameHeaders) {
List Discarding input buffers of type {@link ObuParser#OBU_FRAME_HEADER} speeds up decoding by
+ * not showing already-decoded frames. This is less beneficial than discarding {@link
+ * ObuParser#OBU_FRAME} which reduces the total number of decoded frames.
+ *
+ * Dropping too many consecutive input buffers reduces the update frequency of {@link
+ * #shouldDropDecoderInputBuffers}, and can harm user experience.
+ */
+ private static final int MAX_CONSECUTIVE_DROPPED_INPUT_BUFFERS_COUNT_TO_DISCARD_HEADER = 0;
+
private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround;
private static boolean deviceNeedsSetOutputSurfaceWorkaround;
@@ -199,6 +212,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private boolean pendingVideoSinkInputStreamChange;
private boolean shouldDropDecoderInputBuffers;
+ private int consecutiveDroppedInputBufferCount;
/** A builder to create {@link MediaCodecVideoRenderer} instances. */
public static final class Builder {
@@ -1254,6 +1268,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
droppedDecoderInputBufferTimestamps.clear();
shouldDropDecoderInputBuffers = false;
buffersInCodecCount = 0;
+ consecutiveDroppedInputBufferCount = 0;
if (av1SampleDependencyParser != null) {
av1SampleDependencyParser.reset();
}
@@ -1436,6 +1451,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
&& buffer.data != null) {
av1SampleDependencyParser.queueInputBuffer(buffer.data);
}
+ consecutiveDroppedInputBufferCount = 0;
// In tunneling mode the device may do frame rate conversion, so in general we can't keep track
// of the number of buffers in the codec.
if (!tunneling) {
@@ -1484,16 +1500,22 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
decoderCounters.skippedInputBufferCount += 1;
} else if (shouldDropDecoderInputBuffers) {
droppedDecoderInputBufferTimestamps.add(buffer.timeUs);
+ consecutiveDroppedInputBufferCount += 1;
}
return true;
}
if (av1SampleDependencyParser != null
&& checkNotNull(getCodecInfo()).mimeType.equals(MimeTypes.VIDEO_AV1)
&& buffer.data != null) {
+ boolean skipFrameHeaders =
+ shouldSkipDecoderInputBuffer
+ || consecutiveDroppedInputBufferCount
+ <= MAX_CONSECUTIVE_DROPPED_INPUT_BUFFERS_COUNT_TO_DISCARD_HEADER;
ByteBuffer readOnlySample = buffer.data.asReadOnlyBuffer();
readOnlySample.flip();
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(readOnlySample);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ readOnlySample, skipFrameHeaders);
boolean hasSpaceForNextFrame =
sampleLimitAfterSkippingNonReferenceFrames + checkNotNull(codecMaxValues).inputSize
< readOnlySample.capacity();
@@ -1504,6 +1526,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
decoderCounters.skippedInputBufferCount += 1;
} else if (shouldDropDecoderInputBuffers) {
droppedDecoderInputBufferTimestamps.add(buffer.timeUs);
+ consecutiveDroppedInputBufferCount += 1;
}
return true;
}
diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java
index 202c49e4f6..7a2b56c240 100644
--- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java
+++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java
@@ -15,13 +15,27 @@
*/
package androidx.media3.exoplayer.e2etest;
+import static com.google.common.truth.Truth.assertThat;
+
import android.content.Context;
import android.graphics.SurfaceTexture;
+import android.os.Handler;
import android.view.Surface;
+import androidx.annotation.Nullable;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MediaItem.ClippingConfiguration;
import androidx.media3.common.Player;
+import androidx.media3.exoplayer.DecoderCounters;
+import androidx.media3.exoplayer.DefaultRenderersFactory;
import androidx.media3.exoplayer.ExoPlayer;
+import androidx.media3.exoplayer.Renderer;
+import androidx.media3.exoplayer.audio.AudioRendererEventListener;
+import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
+import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
+import androidx.media3.exoplayer.metadata.MetadataOutput;
+import androidx.media3.exoplayer.text.TextOutput;
+import androidx.media3.exoplayer.video.MediaCodecVideoRenderer;
+import androidx.media3.exoplayer.video.VideoRendererEventListener;
import androidx.media3.test.utils.CapturingRenderersFactory;
import androidx.media3.test.utils.DumpFileAsserts;
import androidx.media3.test.utils.FakeClock;
@@ -33,6 +47,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.robolectric.annotation.Config;
/** End-to-end playback tests using AV1 sample skipping. */
@RunWith(AndroidJUnit4.class)
@@ -74,4 +89,143 @@ public class ParseAv1SampleDependenciesPlaybackTest {
playbackOutput,
/* dumpFile= */ "playbackdumps/av1SampleDependencies/clippedMediaItem.dump");
}
+
+ // TODO: b/390604981 - Run the test on older SDK levels to ensure it uses a MediaCodec shadow
+ // with more than one buffer slot.
+ @Config(minSdk = 30)
+ @Test
+ public void playback_withLateThresholdToDropDecoderInput_skipNonReferenceInputSamples()
+ throws Exception {
+ Context applicationContext = ApplicationProvider.getApplicationContext();
+ CapturingRenderersFactoryWithLateThresholdToDropDecoderInputUs renderersFactory =
+ new CapturingRenderersFactoryWithLateThresholdToDropDecoderInputUs(applicationContext);
+ ExoPlayer player =
+ new ExoPlayer.Builder(applicationContext, renderersFactory)
+ .setClock(new FakeClock(/* isAutoAdvancing= */ true))
+ .build();
+ Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1));
+ player.setVideoSurface(surface);
+ player.setMediaItem(MediaItem.fromUri(TEST_MP4_URI));
+
+ player.prepare();
+ player.play();
+ TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
+ player.release();
+ surface.release();
+ DecoderCounters decoderCounters = player.getVideoDecoderCounters();
+
+ // Do not assert on a full playback dump as it depends on the number of MediaCodec buffer
+ // slots, which may change in b/390604981.
+ // Half of the input buffers are non-reference OBU_FRAME which will be dropped.
+ // The other half are non-reference OBU_FRAME_HEADER - only the first one may be dropped.
+ // Which input buffer is dropped first depends on the number of MediaCodec buffer slots.
+ // This means the asserts cannot be isEqualTo.
+ assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isAtMost(2);
+ assertThat(decoderCounters.droppedInputBufferCount).isAtLeast(8);
+ }
+
+ private static final class CapturingRenderersFactoryWithLateThresholdToDropDecoderInputUs
+ extends CapturingRenderersFactory {
+
+ private final Context context;
+
+ /**
+ * Creates an instance.
+ *
+ * @param context The {@link Context}.
+ */
+ public CapturingRenderersFactoryWithLateThresholdToDropDecoderInputUs(Context context) {
+ super(context);
+ this.context = context;
+ }
+
+ @Override
+ public Renderer[] createRenderers(
+ Handler eventHandler,
+ VideoRendererEventListener videoRendererEventListener,
+ AudioRendererEventListener audioRendererEventListener,
+ TextOutput textRendererOutput,
+ MetadataOutput metadataRendererOutput) {
+ return new Renderer[] {
+ new CapturingMediaCodecVideoRenderer(
+ context,
+ getMediaCodecAdapterFactory(),
+ MediaCodecSelector.DEFAULT,
+ DefaultRenderersFactory.DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS,
+ /* enableDecoderFallback= */ false,
+ eventHandler,
+ videoRendererEventListener,
+ DefaultRenderersFactory.MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
+ /* parseAv1SampleDependencies= */ true,
+ /* lateThresholdToDropDecoderInputUs= */ -100_000_000L)
+ };
+ }
+
+ /**
+ * A {@link MediaCodecVideoRenderer} that will not skip or drop buffers due to slow processing.
+ */
+ private static class CapturingMediaCodecVideoRenderer extends MediaCodecVideoRenderer {
+ private CapturingMediaCodecVideoRenderer(
+ Context context,
+ MediaCodecAdapter.Factory codecAdapterFactory,
+ MediaCodecSelector mediaCodecSelector,
+ long allowedJoiningTimeMs,
+ boolean enableDecoderFallback,
+ @Nullable Handler eventHandler,
+ @Nullable VideoRendererEventListener eventListener,
+ int maxDroppedFramesToNotify,
+ boolean parseAv1SampleDependencies,
+ long lateThresholdToDropDecoderInputUs) {
+ super(
+ new Builder(context)
+ .setCodecAdapterFactory(codecAdapterFactory)
+ .setMediaCodecSelector(mediaCodecSelector)
+ .setAllowedJoiningTimeMs(allowedJoiningTimeMs)
+ .setEnableDecoderFallback(enableDecoderFallback)
+ .setEventHandler(eventHandler)
+ .setEventListener(eventListener)
+ .setMaxDroppedFramesToNotify(maxDroppedFramesToNotify)
+ .experimentalSetParseAv1SampleDependencies(parseAv1SampleDependencies)
+ .experimentalSetLateThresholdToDropDecoderInputUs(
+ lateThresholdToDropDecoderInputUs));
+ }
+
+ @Override
+ protected boolean shouldDropOutputBuffer(
+ long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
+ // Do not drop output buffers due to slow processing.
+ return false;
+ }
+
+ @Override
+ protected boolean shouldDropBuffersToKeyframe(
+ long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
+ // Do not drop output buffers due to slow processing.
+ return false;
+ }
+
+ @Override
+ protected boolean shouldSkipBuffersWithIdenticalReleaseTime() {
+ // Do not skip buffers with identical vsync times as we can't control this from tests.
+ return false;
+ }
+
+ @Override
+ protected boolean shouldSkipLateBuffersWhileUsingPlaceholderSurface() {
+ // Do not skip buffers while using placeholder surface due to slow processing.
+ return false;
+ }
+
+ @Override
+ protected boolean shouldForceRenderOutputBuffer(long earlyUs, long elapsedSinceLastRenderUs) {
+ // An auto-advancing FakeClock can make a lot of progress before
+ // AsynchronousMediaCodecAdapter produces an output buffer - causing all output buffers to
+ // be force rendered.
+ // Force rendering output buffers prevents evaluation of lateThresholdToDropDecoderInputUs.
+ // Do not allow force rendering of output buffers when testing
+ // lateThresholdToDropDecoderInputUs.
+ return false;
+ }
+ }
+ }
}
diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java
index c3a9bb65b9..167ae1d9f6 100644
--- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java
+++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java
@@ -43,6 +43,8 @@ public class Av1SampleDependencyParserTest {
0x32, 0x1A, 0x30, 0xC0, 0x00, 0x1D, 0x66, 0x68, 0x46, 0xC9, 0x38, 0x00, 0x60, 0x10, 0x20,
0x80, 0x20, 0x00, 0x00, 0x01, 0x8B, 0x7A, 0x87, 0xF9, 0xAA, 0x2D, 0x0F, 0x2C);
+ private static final byte[] frameHeader = createByteArray(0x1A, 0x01, 0xC8);
+
private static final byte[] temporalDelimiter = createByteArray(0x12, 0x00);
private static final byte[] padding = createByteArray(0x7a, 0x02, 0xFF, 0xFF);
@@ -56,7 +58,8 @@ public class Av1SampleDependencyParserTest {
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames)
.isEqualTo(sequenceHeader.length + dependedOnFrame.length);
@@ -72,7 +75,8 @@ public class Av1SampleDependencyParserTest {
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(sequenceHeader.length);
}
@@ -86,7 +90,8 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(frame);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ frame, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(0);
}
@@ -104,7 +109,8 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(0);
}
@@ -123,7 +129,8 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames)
.isEqualTo(temporalDelimiter.length + padding.length + dependedOnFrame.length);
@@ -135,7 +142,8 @@ public class Av1SampleDependencyParserTest {
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(frame);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ frame, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(notDependedOnFrame.length);
}
@@ -152,7 +160,8 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames)
.isEqualTo(notDependedOnFrame.length + notDependedOnFrame.length);
@@ -174,7 +183,8 @@ public class Av1SampleDependencyParserTest {
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(sample);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ sample, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(sample.limit());
}
@@ -189,8 +199,39 @@ public class Av1SampleDependencyParserTest {
av1SampleDependencyParser.queueInputBuffer(header);
av1SampleDependencyParser.reset();
int sampleLimitAfterSkippingNonReferenceFrames =
- av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(frame);
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ frame, /* skipFrameHeaders= */ true);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(notDependedOnFrame.length);
}
+
+ @Test
+ public void
+ sampleLimitAfterSkippingNonReferenceFrame_withSkipFrameHeadersTrue_returnsEmptySample() {
+ ByteBuffer header = ByteBuffer.wrap(sequenceHeader);
+ ByteBuffer sample = ByteBuffer.wrap(frameHeader);
+ Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
+
+ av1SampleDependencyParser.queueInputBuffer(header);
+ int sampleLimitAfterSkippingNonReferenceFrame =
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ sample, /* skipFrameHeaders= */ true);
+
+ assertThat(sampleLimitAfterSkippingNonReferenceFrame).isEqualTo(0);
+ }
+
+ @Test
+ public void
+ sampleLimitAfterSkippingNonReferenceFrame_withSkipFrameHeadersFalse_returnsFullSample() {
+ ByteBuffer header = ByteBuffer.wrap(sequenceHeader);
+ ByteBuffer sample = ByteBuffer.wrap(frameHeader);
+ Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
+
+ av1SampleDependencyParser.queueInputBuffer(header);
+ int sampleLimitAfterSkippingNonReferenceFrame =
+ av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(
+ sample, /* skipFrameHeaders= */ false);
+
+ assertThat(sampleLimitAfterSkippingNonReferenceFrame).isEqualTo(frameHeader.length);
+ }
}
diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
index 37399aafcf..f533573392 100644
--- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
+++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
@@ -213,6 +213,13 @@ public class CapturingRenderersFactory implements RenderersFactory, Dumper.Dumpa
/* parseAv1SampleDependencies= */ false);
}
+ /**
+ * Returns the {@link CapturingMediaCodecAdapter.Factory} as a {@link MediaCodecAdapter.Factory}.
+ */
+ protected MediaCodecAdapter.Factory getMediaCodecAdapterFactory() {
+ return mediaCodecAdapterFactory;
+ }
+
/**
* A {@link MediaCodecVideoRenderer} that will not skip or drop buffers due to slow processing.
*/