loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

MCVR Parse AV1 sequence headers in onQueueInputBuffer

Browse Source 
AV1 random access points (sync samples) contain updated sequence headers
that are needed for later frame parsing.

PiperOrigin-RevId: 725565810
This commit is contained in:
dancho 2025-02-11 03:49:18 -08:00 committed by Copybara-Service
parent 50d4e66308
commit 5510635620
4 changed files with 155 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/Av1SampleDependencyParser.java
View File

@ -83,6 +83,11 @@ import java.util.List;
updateSequenceHeaders(split(sample));
}
/** Resets the parser state. */
public void reset() {
sequenceHeader = null;
}
private boolean canSkipObu(ObuParser.Obu obu) {
if (obu.type == OBU_TEMPORAL_DELIMITER || obu.type == OBU_PADDING) {
return true;

8
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -1254,6 +1254,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
droppedDecoderInputBufferTimestamps.clear();
shouldDropDecoderInputBuffers = false;
buffersInCodecCount = 0;
if (av1SampleDependencyParser != null) {
av1SampleDependencyParser.reset();
}
}
@Override
@ -1428,6 +1431,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
@CallSuper
@Override
protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackException {
if (av1SampleDependencyParser != null
&& checkNotNull(getCodecInfo()).mimeType.equals(MimeTypes.VIDEO_AV1)
&& buffer.data != null) {
av1SampleDependencyParser.queueInputBuffer(buffer.data);
}
// In tunneling mode the device may do frame rate conversion, so in general we can't keep track
// of the number of buffers in the codec.
if (!tunneling) {

15
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/Av1SampleDependencyParserTest.java
View File

@ -178,4 +178,19 @@ public class Av1SampleDependencyParserTest {
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(sample.limit());
}
@Test
public void
sampleLimitAfterSkippingNonReferenceFrame_queueSampleHeaderAndReset_returnsFullSample() {
ByteBuffer header = ByteBuffer.wrap(sequenceHeader);
ByteBuffer frame = ByteBuffer.wrap(notDependedOnFrame);
Av1SampleDependencyParser av1SampleDependencyParser = new Av1SampleDependencyParser();
av1SampleDependencyParser.queueInputBuffer(header);
av1SampleDependencyParser.reset();
int sampleLimitAfterSkippingNonReferenceFrames =
av1SampleDependencyParser.sampleLimitAfterSkippingNonReferenceFrame(frame);
assertThat(sampleLimitAfterSkippingNonReferenceFrames).isEqualTo(notDependedOnFrame.length);
}
}

127
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
View File

@ -21,7 +21,9 @@ import static androidx.media3.common.util.Util.msToUs;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.format;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.sample;
import static androidx.media3.test.utils.FakeTimeline.TimelineWindowDefinition.DEFAULT_WINDOW_OFFSET_IN_FIRST_PERIOD_US;
import static androidx.media3.test.utils.TestUtil.createByteArray;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.anyLong;
@ -108,6 +110,7 @@ import org.robolectric.Shadows;
import org.robolectric.annotation.Config;
import org.robolectric.shadows.ShadowDisplay;
import org.robolectric.shadows.ShadowLooper;
import org.robolectric.shadows.ShadowMediaCodec;
import org.robolectric.shadows.ShadowSystemClock;
/** Unit test for {@link MediaCodecVideoRenderer}. */
@ -122,6 +125,13 @@ public class MediaCodecVideoRendererTest {
.setHeight(1080)
.build();
private static final Format VIDEO_AV1 =
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_AV1)
.setWidth(1920)
.setHeight(1080)
.build();
private static final TrackGroup TRACK_GROUP_H264 = new TrackGroup(VIDEO_H264);
private static final MediaCodecInfo H264_PROFILE8_LEVEL4_HW_MEDIA_CODEC_INFO =
@ -1058,6 +1068,123 @@ public class MediaCodecVideoRendererTest {
assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0);
}
@Test
public void render_withLateAV1BufferWithoutDependencies_dropsInputBuffers() throws Exception {
// ShadowMediaCodec does not respect the MediaFormat.KEY_MAX_INPUT_SIZE value requested
// so we have to specify large buffers here.
ShadowMediaCodec.addDecoder(
"name",
new ShadowMediaCodec.CodecConfig(
/* inputBufferSize= */ 2_000_000,
/* outputBufferSize= */ 2_000_000,
/* codec= */ (in, out) -> {}));
byte[] syncFrameBytes =
createByteArray(
0x0A, 0x0E, 0x00, 0x00, 0x00, 0x24, 0xC6, 0xAB, 0xDF, 0x3E, 0xFE, 0x24, 0x04, 0x04,
0x04, 0x10, 0x32, 0x32, 0x10, 0x00, 0xC8, 0xC6, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00,
0x12, 0x03, 0xCE, 0x0A, 0x5C, 0x9B, 0xB6, 0x7C, 0x34, 0x88, 0x82, 0x3E, 0x0D, 0x3E,
0xC2, 0x98, 0x91, 0x6A, 0x5C, 0x80, 0x03, 0xCE, 0x0A, 0x5C, 0x9B, 0xB6, 0x7C, 0x48,
0x35, 0x54, 0xD8, 0x9D, 0x6C, 0x37, 0xD3, 0x4C, 0x4E, 0xD4, 0x6F, 0xF4);
byte[] notDependedOnFrameBytes =
createByteArray(
0x32, 0x1A, 0x30, 0xC0, 0x00, 0x1D, 0x66, 0x68, 0x46, 0xC9, 0x38, 0x00, 0x60, 0x10,
0x20, 0x80, 0x20, 0x00, 0x00, 0x01, 0x8B, 0x7A, 0x87, 0xF9, 0xAA, 0x2D, 0x0F, 0x2C);
FakeTimeline fakeTimeline =
new FakeTimeline(
new FakeTimeline.TimelineWindowDefinition(
/* isSeekable= */ true, /* isDynamic= */ false, /* durationUs= */ 1_000_000));
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ VIDEO_AV1,
ImmutableList.of(
sample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME, syncFrameBytes), // First frame
oneByteSample(
/* timeUs= */ 20_000))); // Late buffer triggers input buffer dropping.
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer =
new MediaCodecVideoRenderer(
new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext())
.setCodecAdapterFactory(
new DefaultMediaCodecAdapterFactory(
ApplicationProvider.getApplicationContext(),
() -> {
callbackThread = new HandlerThread("MCVRTest:MediaCodecAsyncAdapter");
return callbackThread;
},
() -> {
queueingThread = new HandlerThread("MCVRTest:MediaCodecQueueingThread");
return queueingThread;
}))
.setMediaCodecSelector(mediaCodecSelector)
.setAllowedJoiningTimeMs(0)
.setEnableDecoderFallback(false)
.setEventHandler(new Handler(testMainLooper))
.setEventListener(eventListener)
.setMaxDroppedFramesToNotify(1)
.experimentalSetLateThresholdToDropDecoderInputUs(50_000)
.experimentalSetParseAv1SampleDependencies(true)) {
@Override
protected @Capabilities int supportsFormat(
MediaCodecSelector mediaCodecSelector, Format format) {
return RendererCapabilities.create(C.FORMAT_HANDLED);
}
};
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
mediaCodecVideoRenderer.setTimeline(fakeTimeline);
mediaCodecVideoRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {VIDEO_AV1},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(fakeTimeline.getUidOfPeriod(0)));
shadowOf(testMainLooper).idle();
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
ArgumentCaptor.forClass(DecoderCounters.class);
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
DecoderCounters decoderCounters = argumentDecoderCounters.getValue();
mediaCodecVideoRenderer.start();
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
while (decoderCounters.renderedOutputBufferCount == 0) {
mediaCodecVideoRenderer.render(10_000, SystemClock.elapsedRealtime() * 1000);
}
// Ensure existing buffer will be ~280ms late and new (not yet read) buffers are available
// to be dropped.
int posUs = 300_000;
fakeSampleStream.append(
ImmutableList.of(
sample(
/* timeUs= */ 30_000, /* flags= */ 0, notDependedOnFrameBytes), // Dropped on input.
oneByteSample(/* timeUs= */ 300_000), // Caught up - render.
oneByteSample(/* timeUs= */ 500_000), // Last buffer is always rendered.
END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
mediaCodecVideoRenderer.setCurrentStreamFinal();
// Render until the non-dropped frame is reached and then increase time to reach the end.
while (decoderCounters.renderedOutputBufferCount < 2) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
}
while (!mediaCodecVideoRenderer.isEnded()) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
posUs += 2_000;
}
assertThat(decoderCounters.droppedInputBufferCount).isEqualTo(1);
assertThat(decoderCounters.droppedBufferCount).isEqualTo(2);
assertThat(decoderCounters.maxConsecutiveDroppedBufferCount).isEqualTo(2);
assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(0);
}
// TODO: b/390604981 - Run the test on older SDK levels to ensure it uses a MediaCodec shadow
// with more than one buffer slot.
@Config(minSdk = 30)