diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/Av1SampleDependencyParser.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/Av1SampleDependencyParser.java index ed0016408d..b08cf9e3ed 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/Av1SampleDependencyParser.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/Av1SampleDependencyParser.java @@ -83,6 +83,11 @@ import java.util.List; updateSequenceHeaders(split(sample)); } + /** Resets the parser state. */ + public void reset() { + sequenceHeader = null; + } + private boolean canSkipObu(ObuParser.Obu obu) { if (obu.type == OBU_TEMPORAL_DELIMITER || obu.type == OBU_PADDING) { return true; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index e80e5ee3fd..f84b4e74b0 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -1254,6 +1254,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer droppedDecoderInputBufferTimestamps.clear(); shouldDropDecoderInputBuffers = false; buffersInCodecCount = 0; + if (av1SampleDependencyParser != null) { + av1SampleDependencyParser.reset(); + } } @Override @@ -1428,6 +1431,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @CallSuper @Override protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackException { + if (av1SampleDependencyParser != null + && checkNotNull(getCodecInfo()).mimeType.equals(MimeTypes.VIDEO_AV1) + && buffer.data != null) { + av1SampleDependencyParser.queueInputBuffer(buffer.data); + } // In tunneling mode the device may do frame rate conversion, so in general we can't keep track // of the number of buffers in the codec. if (!tunneling) { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java index c928f99b58..c3a9bb65b9 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java @@ -178,4 +178,19 @@ public class Av1SampleDependencyParserTest { assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(sample.limit()); } + + @Test + public void + sampleLimitAfterSkippingNonReferenceFrame_queueSampleHeaderAndReset_returnsFullSample() { + ByteBuffer header = ByteBuffer.wrap(sequenceHeader); + ByteBuffer frame = ByteBuffer.wrap(notDependedOnFrame); + Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser(); + + av1SampleDependencyParser.queueInputBuffer(header); + av1SampleDependencyParser.reset(); + int sampleLimitAfterSkippingNonReferenceFrames = + av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(frame); + + assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(notDependedOnFrame.length); + } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 39d2d6c958..78034fd82e 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -21,7 +21,9 @@ import static androidx.media3.common.util.Util.msToUs; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.format; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.sample; import static androidx.media3.test.utils.FakeTimeline.TimelineWindowDefinition.DEFAULT_WINDOW_OFFSET_IN_FIRST_PERIOD_US; +import static androidx.media3.test.utils.TestUtil.createByteArray; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.anyLong; @@ -108,6 +110,7 @@ import org.robolectric.Shadows; import org.robolectric.annotation.Config; import org.robolectric.shadows.ShadowDisplay; import org.robolectric.shadows.ShadowLooper; +import org.robolectric.shadows.ShadowMediaCodec; import org.robolectric.shadows.ShadowSystemClock; /** Unit test for {@link MediaCodecVideoRenderer}. */ @@ -122,6 +125,13 @@ public class MediaCodecVideoRendererTest { .setHeight(1080) .build(); + private static final Format VIDEO_AV1 = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_AV1) + .setWidth(1920) + .setHeight(1080) + .build(); + private static final TrackGroup TRACK_GROUP_H264 = new TrackGroup(VIDEO_H264); private static final MediaCodecInfo H264_PROFILE8_LEVEL4_HW_MEDIA_CODEC_INFO = @@ -1058,6 +1068,123 @@ public class MediaCodecVideoRendererTest { assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0); } + @Test + public void render_withLateAV1BufferWithoutDependencies_dropsInputBuffers() throws Exception { + // ShadowMediaCodec does not respect the MediaFormat.KEY_MAX_INPUT_SIZE value requested + // so we have to specify large buffers here. + ShadowMediaCodec.addDecoder( + "name", + new ShadowMediaCodec.CodecConfig( + /* inputBufferSize= */ 2_000_000, + /* outputBufferSize= */ 2_000_000, + /* codec= */ (in, out) -> {})); + byte[] syncFrameBytes = + createByteArray( + 0x0A, 0x0E, 0x00, 0x00, 0x00, 0x24, 0xC6, 0xAB, 0xDF, 0x3E, 0xFE, 0x24, 0x04, 0x04, + 0x04, 0x10, 0x32, 0x32, 0x10, 0x00, 0xC8, 0xC6, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, + 0x12, 0x03, 0xCE, 0x0A, 0x5C, 0x9B, 0xB6, 0x7C, 0x34, 0x88, 0x82, 0x3E, 0x0D, 0x3E, + 0xC2, 0x98, 0x91, 0x6A, 0x5C, 0x80, 0x03, 0xCE, 0x0A, 0x5C, 0x9B, 0xB6, 0x7C, 0x48, + 0x35, 0x54, 0xD8, 0x9D, 0x6C, 0x37, 0xD3, 0x4C, 0x4E, 0xD4, 0x6F, 0xF4); + byte[] notDependedOnFrameBytes = + createByteArray( + 0x32, 0x1A, 0x30, 0xC0, 0x00, 0x1D, 0x66, 0x68, 0x46, 0xC9, 0x38, 0x00, 0x60, 0x10, + 0x20, 0x80, 0x20, 0x00, 0x00, 0x01, 0x8B, 0x7A, 0x87, 0xF9, 0xAA, 0x2D, 0x0F, 0x2C); + FakeTimeline fakeTimeline = + new FakeTimeline( + new FakeTimeline.TimelineWindowDefinition( + /* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 1_000_000)); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_AV1, + ImmutableList.of( + sample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME, syncFrameBytes), // First frame + oneByteSample( + /* timeUs= */ 20_000))); // Late buffer triggers input buffer dropping. + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new DefaultMediaCodecAdapterFactory( + ApplicationProvider.getApplicationContext(), + () -> { + callbackThread = new HandlerThread("MCVRTest:MediaCodecAsyncAdapter"); + return callbackThread; + }, + () -> { + queueingThread = new HandlerThread("MCVRTest:MediaCodecQueueingThread"); + return queueingThread; + })) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1) + .experimentalSetLateThresholdToDropDecoderInputUs(50_000) + .experimentalSetParseAv1SampleDependencies(true)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + }; + + mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.setTimeline(fakeTimeline); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_AV1}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0))); + shadowOf(testMainLooper).idle(); + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + DecoderCounters decoderCounters = argumentDecoderCounters.getValue(); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + while (decoderCounters.renderedOutputBufferCount == 0) { + mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000); + } + // Ensure existing buffer will be ~280ms late and new (not yet read) buffers are available + // to be dropped. + int posUs = 300_000; + fakeSampleStream.append( + ImmutableList.of( + sample( + /* timeUs= */ 30_000, /* flags= */ 0, notDependedOnFrameBytes), // Dropped on input. + oneByteSample(/* timeUs= */ 300_000), // Caught up - render. + oneByteSample(/* timeUs= */ 500_000), // Last buffer is always rendered. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + // Render until the non-dropped frame is reached and then increase time to reach the end. + while (decoderCounters.renderedOutputBufferCount < 2) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + } + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 2_000; + } + + assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(1); + assertThat(decoderCounters.droppedBufferCount).isEqualTo(2); + assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isEqualTo(2); + assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0); + } + // TODO: b/390604981 - Run the test on older SDK levels to ensure it uses a MediaCodec shadow // with more than one buffer slot. @Config(minSdk = 30)