From 2eab6802c939ddefe660f016e6e70eb2da24b3ca Mon Sep 17 00:00:00 2001 From: kimvde Date: Mon, 29 Jun 2020 13:55:27 +0100 Subject: [PATCH] Parse stbl in FragmentedMp4Extractor This will be necessary to support partially fragmented MP4s. PiperOrigin-RevId: 318798726 --- .../exoplayer2/extractor/mp4/AtomParsers.java | 200 +++++++++--------- .../extractor/mp4/FragmentedMp4Extractor.java | 75 ++++--- .../extractor/mp4/Mp4Extractor.java | 12 +- 3 files changed, 158 insertions(+), 129 deletions(-) diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java index 85c5958401..13ffcc2ff1 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java @@ -37,6 +37,7 @@ import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.AvcConfig; import com.google.android.exoplayer2.video.DolbyVisionConfig; import com.google.android.exoplayer2.video.HevcConfig; +import com.google.common.base.Function; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -87,16 +88,23 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; * * @param moov Moov atom to decode. * @param gaplessInfoHolder Holder to populate with gapless playback information. + * @param duration The duration in units of the timescale declared in the mvhd atom, or {@link + * C#TIME_UNSET} if the duration should be parsed from the tkhd atom. + * @param drmInitData {@link DrmInitData} to be included in the format, or {@code null}. * @param ignoreEditLists Whether to ignore any edit lists in the trak boxes. * @param isQuickTime True for QuickTime media. False otherwise. + * @param modifyTrackFunction A function to apply to the {@link Track Tracks} in the result. * @return A list of {@link TrackSampleTable} instances. * @throws ParserException Thrown if the trak atoms can't be parsed. */ public static List parseTraks( Atom.ContainerAtom moov, GaplessInfoHolder gaplessInfoHolder, + long duration, + @Nullable DrmInitData drmInitData, boolean ignoreEditLists, - boolean isQuickTime) + boolean isQuickTime, + Function modifyTrackFunction) throws ParserException { List trackSampleTables = new ArrayList<>(); for (int i = 0; i < moov.containerChildren.size(); i++) { @@ -106,13 +114,14 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; } @Nullable Track track = - parseTrak( - atom, - moov.getLeafAtomOfType(Atom.TYPE_mvhd), - /* duration= */ C.TIME_UNSET, - /* drmInitData= */ null, - ignoreEditLists, - isQuickTime); + modifyTrackFunction.apply( + parseTrak( + atom, + moov.getLeafAtomOfType(Atom.TYPE_mvhd), + duration, + drmInitData, + ignoreEditLists, + isQuickTime)); if (track == null) { continue; } @@ -121,14 +130,95 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; .getContainerAtomOfType(Atom.TYPE_minf) .getContainerAtomOfType(Atom.TYPE_stbl); TrackSampleTable trackSampleTable = parseStbl(track, stblAtom, gaplessInfoHolder); - if (trackSampleTable.sampleCount == 0) { - continue; - } trackSampleTables.add(trackSampleTable); } return trackSampleTables; } + /** + * Parses a udta atom. + * + * @param udtaAtom The udta (user data) atom to decode. + * @param isQuickTime True for QuickTime media. False otherwise. + * @return Parsed metadata, or null. + */ + @Nullable + public static Metadata parseUdta(Atom.LeafAtom udtaAtom, boolean isQuickTime) { + if (isQuickTime) { + // Meta boxes are regular boxes rather than full boxes in QuickTime. For now, don't try and + // decode one. + return null; + } + ParsableByteArray udtaData = udtaAtom.data; + udtaData.setPosition(Atom.HEADER_SIZE); + while (udtaData.bytesLeft() >= Atom.HEADER_SIZE) { + int atomPosition = udtaData.getPosition(); + int atomSize = udtaData.readInt(); + int atomType = udtaData.readInt(); + if (atomType == Atom.TYPE_meta) { + udtaData.setPosition(atomPosition); + return parseUdtaMeta(udtaData, atomPosition + atomSize); + } + udtaData.setPosition(atomPosition + atomSize); + } + return null; + } + + /** + * Parses a metadata meta atom if it contains metadata with handler 'mdta'. + * + * @param meta The metadata atom to decode. + * @return Parsed metadata, or null. + */ + @Nullable + public static Metadata parseMdtaFromMeta(Atom.ContainerAtom meta) { + @Nullable Atom.LeafAtom hdlrAtom = meta.getLeafAtomOfType(Atom.TYPE_hdlr); + @Nullable Atom.LeafAtom keysAtom = meta.getLeafAtomOfType(Atom.TYPE_keys); + @Nullable Atom.LeafAtom ilstAtom = meta.getLeafAtomOfType(Atom.TYPE_ilst); + if (hdlrAtom == null + || keysAtom == null + || ilstAtom == null + || parseHdlr(hdlrAtom.data) != TYPE_mdta) { + // There isn't enough information to parse the metadata, or the handler type is unexpected. + return null; + } + + // Parse metadata keys. + ParsableByteArray keys = keysAtom.data; + keys.setPosition(Atom.FULL_HEADER_SIZE); + int entryCount = keys.readInt(); + String[] keyNames = new String[entryCount]; + for (int i = 0; i < entryCount; i++) { + int entrySize = keys.readInt(); + keys.skipBytes(4); // keyNamespace + int keySize = entrySize - 8; + keyNames[i] = keys.readString(keySize); + } + + // Parse metadata items. + ParsableByteArray ilst = ilstAtom.data; + ilst.setPosition(Atom.HEADER_SIZE); + ArrayList entries = new ArrayList<>(); + while (ilst.bytesLeft() > Atom.HEADER_SIZE) { + int atomPosition = ilst.getPosition(); + int atomSize = ilst.readInt(); + int keyIndex = ilst.readInt() - 1; + if (keyIndex >= 0 && keyIndex < keyNames.length) { + String key = keyNames[keyIndex]; + @Nullable + Metadata.Entry entry = + MetadataUtil.parseMdtaMetadataEntryFromIlst(ilst, atomPosition + atomSize, key); + if (entry != null) { + entries.add(entry); + } + } else { + Log.w(TAG, "Skipped metadata with unknown key index: " + keyIndex); + } + ilst.setPosition(atomPosition + atomSize); + } + return entries.isEmpty() ? null : new Metadata(entries); + } + /** * Parses a trak atom (defined in ISO/IEC 14496-12). * @@ -143,7 +233,7 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; * @throws ParserException Thrown if the trak atom can't be parsed. */ @Nullable - public static Track parseTrak( + private static Track parseTrak( Atom.ContainerAtom trak, Atom.LeafAtom mvhd, long duration, @@ -201,7 +291,7 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; * @return Sample table described by the stbl atom. * @throws ParserException Thrown if the stbl atom can't be parsed. */ - public static TrackSampleTable parseStbl( + private static TrackSampleTable parseStbl( Track track, Atom.ContainerAtom stblAtom, GaplessInfoHolder gaplessInfoHolder) throws ParserException { SampleSizeBox sampleSizeBox; @@ -561,90 +651,6 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; editedDurationUs); } - /** - * Parses a udta atom. - * - * @param udtaAtom The udta (user data) atom to decode. - * @param isQuickTime True for QuickTime media. False otherwise. - * @return Parsed metadata, or null. - */ - @Nullable - public static Metadata parseUdta(Atom.LeafAtom udtaAtom, boolean isQuickTime) { - if (isQuickTime) { - // Meta boxes are regular boxes rather than full boxes in QuickTime. For now, don't try and - // decode one. - return null; - } - ParsableByteArray udtaData = udtaAtom.data; - udtaData.setPosition(Atom.HEADER_SIZE); - while (udtaData.bytesLeft() >= Atom.HEADER_SIZE) { - int atomPosition = udtaData.getPosition(); - int atomSize = udtaData.readInt(); - int atomType = udtaData.readInt(); - if (atomType == Atom.TYPE_meta) { - udtaData.setPosition(atomPosition); - return parseUdtaMeta(udtaData, atomPosition + atomSize); - } - udtaData.setPosition(atomPosition + atomSize); - } - return null; - } - - /** - * Parses a metadata meta atom if it contains metadata with handler 'mdta'. - * - * @param meta The metadata atom to decode. - * @return Parsed metadata, or null. - */ - @Nullable - public static Metadata parseMdtaFromMeta(Atom.ContainerAtom meta) { - @Nullable Atom.LeafAtom hdlrAtom = meta.getLeafAtomOfType(Atom.TYPE_hdlr); - @Nullable Atom.LeafAtom keysAtom = meta.getLeafAtomOfType(Atom.TYPE_keys); - @Nullable Atom.LeafAtom ilstAtom = meta.getLeafAtomOfType(Atom.TYPE_ilst); - if (hdlrAtom == null - || keysAtom == null - || ilstAtom == null - || parseHdlr(hdlrAtom.data) != TYPE_mdta) { - // There isn't enough information to parse the metadata, or the handler type is unexpected. - return null; - } - - // Parse metadata keys. - ParsableByteArray keys = keysAtom.data; - keys.setPosition(Atom.FULL_HEADER_SIZE); - int entryCount = keys.readInt(); - String[] keyNames = new String[entryCount]; - for (int i = 0; i < entryCount; i++) { - int entrySize = keys.readInt(); - keys.skipBytes(4); // keyNamespace - int keySize = entrySize - 8; - keyNames[i] = keys.readString(keySize); - } - - // Parse metadata items. - ParsableByteArray ilst = ilstAtom.data; - ilst.setPosition(Atom.HEADER_SIZE); - ArrayList entries = new ArrayList<>(); - while (ilst.bytesLeft() > Atom.HEADER_SIZE) { - int atomPosition = ilst.getPosition(); - int atomSize = ilst.readInt(); - int keyIndex = ilst.readInt() - 1; - if (keyIndex >= 0 && keyIndex < keyNames.length) { - String key = keyNames[keyIndex]; - @Nullable - Metadata.Entry entry = - MetadataUtil.parseMdtaMetadataEntryFromIlst(ilst, atomPosition + atomSize, key); - if (entry != null) { - entries.add(entry); - } - } else { - Log.w(TAG, "Skipped metadata with unknown key index: " + keyIndex); - } - ilst.setPosition(atomPosition + atomSize); - } - return entries.isEmpty() ? null : new Metadata(entries); - } - @Nullable private static Metadata parseUdtaMeta(ParsableByteArray meta, int limit) { meta.skipBytes(Atom.FULL_HEADER_SIZE); diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java index 4496d8784e..30a1de8988 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java @@ -15,6 +15,8 @@ */ package com.google.android.exoplayer2.extractor.mp4; +import static com.google.android.exoplayer2.extractor.mp4.AtomParsers.parseTraks; + import android.util.Pair; import android.util.SparseArray; import androidx.annotation.IntDef; @@ -31,6 +33,7 @@ import com.google.android.exoplayer2.extractor.Extractor; import com.google.android.exoplayer2.extractor.ExtractorInput; import com.google.android.exoplayer2.extractor.ExtractorOutput; import com.google.android.exoplayer2.extractor.ExtractorsFactory; +import com.google.android.exoplayer2.extractor.GaplessInfoHolder; import com.google.android.exoplayer2.extractor.PositionHolder; import com.google.android.exoplayer2.extractor.SeekMap; import com.google.android.exoplayer2.extractor.TrackOutput; @@ -479,33 +482,22 @@ public class FragmentedMp4Extractor implements Extractor { } } - // Construction of tracks. - SparseArray tracks = new SparseArray<>(); - int moovContainerChildrenSize = moov.containerChildren.size(); - for (int i = 0; i < moovContainerChildrenSize; i++) { - Atom.ContainerAtom atom = moov.containerChildren.get(i); - if (atom.type == Atom.TYPE_trak) { - @Nullable - Track track = - modifyTrack( - AtomParsers.parseTrak( - atom, - moov.getLeafAtomOfType(Atom.TYPE_mvhd), - duration, - drmInitData, - (flags & FLAG_WORKAROUND_IGNORE_EDIT_LISTS) != 0, - false)); - if (track != null) { - tracks.put(track.id, track); - } - } - } + // Construction of tracks and sample tables. + List trackSampleTables = + parseTraks( + moov, + new GaplessInfoHolder(), + duration, + drmInitData, + /* ignoreEditLists= */ (flags & FLAG_WORKAROUND_IGNORE_EDIT_LISTS) != 0, + /* isQuickTime= */ false, + this::modifyTrack); - int trackCount = tracks.size(); + int trackCount = trackSampleTables.size(); if (trackBundles.size() == 0) { // We need to create the track bundles. for (int i = 0; i < trackCount; i++) { - Track track = tracks.valueAt(i); + Track track = trackSampleTables.get(i).track; TrackBundle trackBundle = new TrackBundle(extractorOutput.track(i, track.type)); trackBundle.init(track, getDefaultSampleValues(defaultSampleValuesArray, track.id)); trackBundles.put(track.id, trackBundle); @@ -516,7 +508,7 @@ public class FragmentedMp4Extractor implements Extractor { } else { Assertions.checkState(trackBundles.size() == trackCount); for (int i = 0; i < trackCount; i++) { - Track track = tracks.valueAt(i); + Track track = trackSampleTables.get(i).track; trackBundles .get(track.id) .init(track, getDefaultSampleValues(defaultSampleValuesArray, track.id)); @@ -1447,13 +1439,34 @@ public class FragmentedMp4Extractor implements Extractor { /** Returns whether the extractor should decode a leaf atom with type {@code atom}. */ private static boolean shouldParseLeafAtom(int atom) { - return atom == Atom.TYPE_hdlr || atom == Atom.TYPE_mdhd || atom == Atom.TYPE_mvhd - || atom == Atom.TYPE_sidx || atom == Atom.TYPE_stsd || atom == Atom.TYPE_tfdt - || atom == Atom.TYPE_tfhd || atom == Atom.TYPE_tkhd || atom == Atom.TYPE_trex - || atom == Atom.TYPE_trun || atom == Atom.TYPE_pssh || atom == Atom.TYPE_saiz - || atom == Atom.TYPE_saio || atom == Atom.TYPE_senc || atom == Atom.TYPE_uuid - || atom == Atom.TYPE_sbgp || atom == Atom.TYPE_sgpd || atom == Atom.TYPE_elst - || atom == Atom.TYPE_mehd || atom == Atom.TYPE_emsg; + return atom == Atom.TYPE_hdlr + || atom == Atom.TYPE_mdhd + || atom == Atom.TYPE_mvhd + || atom == Atom.TYPE_sidx + || atom == Atom.TYPE_stsd + || atom == Atom.TYPE_stts + || atom == Atom.TYPE_ctts + || atom == Atom.TYPE_stsc + || atom == Atom.TYPE_stsz + || atom == Atom.TYPE_stz2 + || atom == Atom.TYPE_stco + || atom == Atom.TYPE_co64 + || atom == Atom.TYPE_stss + || atom == Atom.TYPE_tfdt + || atom == Atom.TYPE_tfhd + || atom == Atom.TYPE_tkhd + || atom == Atom.TYPE_trex + || atom == Atom.TYPE_trun + || atom == Atom.TYPE_pssh + || atom == Atom.TYPE_saiz + || atom == Atom.TYPE_saio + || atom == Atom.TYPE_senc + || atom == Atom.TYPE_uuid + || atom == Atom.TYPE_sbgp + || atom == Atom.TYPE_sgpd + || atom == Atom.TYPE_elst + || atom == Atom.TYPE_mehd + || atom == Atom.TYPE_emsg; } /** Returns whether the extractor should decode a container atom with type {@code atom}. */ diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java index bc822c8212..13668404cf 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java @@ -409,11 +409,21 @@ public final class Mp4Extractor implements Extractor, SeekMap { boolean ignoreEditLists = (flags & FLAG_WORKAROUND_IGNORE_EDIT_LISTS) != 0; List trackSampleTables = - parseTraks(moov, gaplessInfoHolder, ignoreEditLists, isQuickTime); + parseTraks( + moov, + gaplessInfoHolder, + /* duration= */ C.TIME_UNSET, + /* drmInitData= */ null, + ignoreEditLists, + isQuickTime, + /* modifyTrackFunction= */ track -> track); int trackCount = trackSampleTables.size(); for (int i = 0; i < trackCount; i++) { TrackSampleTable trackSampleTable = trackSampleTables.get(i); + if (trackSampleTable.sampleCount == 0) { + continue; + } Track track = trackSampleTable.track; long trackDurationUs = track.durationUs != C.TIME_UNSET ? track.durationUs : trackSampleTable.durationUs;