From e25bd07a8174757c7a1ab4f8033d0967ade29d79 Mon Sep 17 00:00:00 2001 From: rohks Date: Wed, 1 May 2024 08:20:42 -0700 Subject: [PATCH] Parse initialization data from AV1 tracks Initialization data is not passed to `MediaCodecRenderer` as it is not required for playing AV1 video. See: https://developer.android.com/reference/android/media/MediaCodec#CSD PiperOrigin-RevId: 629729301 --- .../media3/exoplayer/mediacodec/MediaCodecRenderer.java | 9 +++++++++ .../java/androidx/media3/extractor/mp4/AtomParsers.java | 6 ++++++ .../extractordumps/mp4/sample_with_av1c.mp4.0.dump | 2 ++ .../extractordumps/mp4/sample_with_av1c.mp4.1.dump | 2 ++ .../extractordumps/mp4/sample_with_av1c.mp4.2.dump | 2 ++ .../extractordumps/mp4/sample_with_av1c.mp4.3.dump | 2 ++ .../mp4/sample_with_av1c.mp4.unknown_length.dump | 2 ++ .../mp4/sample_with_colr_mdcv_and_clli.mp4.0.dump | 2 ++ .../mp4/sample_with_colr_mdcv_and_clli.mp4.1.dump | 2 ++ .../mp4/sample_with_colr_mdcv_and_clli.mp4.2.dump | 2 ++ .../mp4/sample_with_colr_mdcv_and_clli.mp4.3.dump | 2 ++ ...ample_with_colr_mdcv_and_clli.mp4.unknown_length.dump | 2 ++ .../src/test/assets/muxerdumps/sample_av1.mp4.dump | 2 ++ 13 files changed, 37 insertions(+) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index a0efd57c2f..debaa2b7cb 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -1544,6 +1544,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer { newFormat, PlaybackException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED); } + + // Remove the initialization data from the format if present when dealing with AV1, as it is not + // required for playing AV1 video. + // Reference: https://developer.android.com/reference/android/media/MediaCodec#CSD + if (Objects.equals(newFormat.sampleMimeType, MimeTypes.VIDEO_AV1) + && !newFormat.initializationData.isEmpty()) { + newFormat = newFormat.buildUpon().setInitializationData(null).build(); + } + setSourceDrmSession(formatHolder.drmSession); inputFormat = newFormat; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java index 54daa6b5c0..3b6a3384e7 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java @@ -1227,6 +1227,12 @@ import java.util.Objects; ColorInfo.isoTransferCharacteristicsToColorTransfer(transferCharacteristics); } else if (childAtomType == Atom.TYPE_av1C) { mimeType = MimeTypes.VIDEO_AV1; + + int childAtomBodySize = childAtomSize - Atom.HEADER_SIZE; + byte[] initializationDataChunk = new byte[childAtomBodySize]; + parent.readBytes(initializationDataChunk, /* offset= */ 0, childAtomBodySize); + initializationData = ImmutableList.of(initializationDataChunk); + parent.setPosition(childStartPosition + Atom.HEADER_SIZE); ColorInfo colorInfo = parseAv1c(parent); diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.0.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.0.dump index 2677a1f0ba..d6e8f4bd61 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.0.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.0.dump @@ -23,6 +23,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf60.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 3DFDDB0E sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.1.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.1.dump index 2677a1f0ba..d6e8f4bd61 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.1.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.1.dump @@ -23,6 +23,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf60.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 3DFDDB0E sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.2.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.2.dump index 2677a1f0ba..d6e8f4bd61 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.2.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.2.dump @@ -23,6 +23,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf60.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 3DFDDB0E sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.3.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.3.dump index 2677a1f0ba..d6e8f4bd61 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.3.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.3.dump @@ -23,6 +23,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf60.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 3DFDDB0E sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.unknown_length.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.unknown_length.dump index 2677a1f0ba..d6e8f4bd61 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.unknown_length.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_av1c.mp4.unknown_length.dump @@ -23,6 +23,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf60.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 3DFDDB0E sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.0.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.0.dump index 4ef00b607a..db4c00d469 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.0.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.0.dump @@ -24,6 +24,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf58.76.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 4DF5B288 sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.1.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.1.dump index 4fbfbd773e..bf8b791656 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.1.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.1.dump @@ -24,6 +24,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf58.76.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 4DF5B288 sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.2.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.2.dump index 733957024e..a112e01a52 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.2.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.2.dump @@ -24,6 +24,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf58.76.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 4DF5B288 sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.3.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.3.dump index 538fe086d0..0a9e49fb90 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.3.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.3.dump @@ -24,6 +24,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf58.76.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 4DF5B288 sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.unknown_length.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.unknown_length.dump index 4ef00b607a..db4c00d469 100644 --- a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.unknown_length.dump +++ b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.unknown_length.dump @@ -24,6 +24,8 @@ track 0: lumaBitdepth = 10 chromaBitdepth = 10 metadata = entries=[TSSE: description=null: values=[Lavf58.76.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000] + initializationData: + data = length 20, hash 4DF5B288 sample 0: time = 0 flags = 1 diff --git a/libraries/test_data/src/test/assets/muxerdumps/sample_av1.mp4.dump b/libraries/test_data/src/test/assets/muxerdumps/sample_av1.mp4.dump index 0c0b8c43ff..df405fb466 100644 --- a/libraries/test_data/src/test/assets/muxerdumps/sample_av1.mp4.dump +++ b/libraries/test_data/src/test/assets/muxerdumps/sample_av1.mp4.dump @@ -20,6 +20,8 @@ track 0: lumaBitdepth = 8 chromaBitdepth = 8 metadata = entries=[Mp4Timestamp: creation time=100000000, modification time=500000000, timescale=10000] + initializationData: + data = length 17, hash 54AC4E6D sample 0: time = 0 flags = 1