diff --git a/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java b/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java index 235639f678..c8fd43677d 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java @@ -30,6 +30,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray; import com.google.android.exoplayer2.util.MimeTypes; import com.google.common.util.concurrent.ListenableFuture; import java.util.concurrent.ExecutionException; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -37,9 +38,15 @@ import org.junit.runner.RunWith; @RunWith(AndroidJUnit4.class) public class MetadataRetrieverTest { + private Context context; + + @Before + public void setUp() throws Exception { + context = ApplicationProvider.getApplicationContext(); + } + @Test public void retrieveMetadata_singleMediaItem_outputsExpectedMetadata() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); MediaItem mediaItem = MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4")); @@ -57,7 +64,6 @@ public class MetadataRetrieverTest { @Test public void retrieveMetadata_multipleMediaItems_outputsExpectedMetadata() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); MediaItem mediaItem1 = MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4")); MediaItem mediaItem2 = @@ -85,8 +91,7 @@ public class MetadataRetrieverTest { } @Test - public void retrieveMetadata_motionPhoto_outputsExpectedMetadata() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); + public void retrieveMetadata_heicMotionPhoto_outputsExpectedMetadata() throws Exception { MediaItem mediaItem = MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_MP.heic")); MotionPhoto expectedMotionPhoto = @@ -105,9 +110,21 @@ public class MetadataRetrieverTest { assertThat(trackGroups.get(0).getFormat(0).metadata.get(0)).isEqualTo(expectedMotionPhoto); } + @Test + public void retrieveMetadata_heicStillPhoto_outputsEmptyMetadata() throws Exception { + MediaItem mediaItem = + MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_still_photo.heic")); + + ListenableFuture trackGroupsFuture = retrieveMetadata(context, mediaItem); + TrackGroupArray trackGroups = waitAndGetTrackGroups(trackGroupsFuture); + + assertThat(trackGroups.length).isEqualTo(1); + assertThat(trackGroups.get(0).length).isEqualTo(1); + assertThat(trackGroups.get(0).getFormat(0).metadata).isNull(); + } + @Test public void retrieveMetadata_invalidMediaItem_throwsError() { - Context context = ApplicationProvider.getApplicationContext(); MediaItem mediaItem = MediaItem.fromUri(Uri.parse("asset://android_asset/media/does_not_exist")); diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java index d478eb2b4b..313f1cebe7 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java @@ -16,6 +16,8 @@ package com.google.android.exoplayer2.extractor.mp4; import static com.google.android.exoplayer2.extractor.mp4.AtomParsers.parseTraks; +import static com.google.android.exoplayer2.extractor.mp4.Sniffer.BRAND_HEIC; +import static com.google.android.exoplayer2.extractor.mp4.Sniffer.BRAND_QUICKTIME; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Util.castNonNull; import static java.lang.Math.max; @@ -94,8 +96,15 @@ public final class Mp4Extractor implements Extractor, SeekMap { private static final int STATE_READING_ATOM_PAYLOAD = 1; private static final int STATE_READING_SAMPLE = 2; - /** Brand stored in the ftyp atom for QuickTime media. */ - private static final int BRAND_QUICKTIME = 0x71742020; + /** Supported file types. */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @IntDef({FILE_TYPE_MP4, FILE_TYPE_QUICKTIME, FILE_TYPE_HEIC}) + private @interface FileType {} + + private static final int FILE_TYPE_MP4 = 0; + private static final int FILE_TYPE_QUICKTIME = 1; + private static final int FILE_TYPE_HEIC = 2; /** * When seeking within the source, if the offset is greater than or equal to this value (or the @@ -133,10 +142,12 @@ public final class Mp4Extractor implements Extractor, SeekMap { // Extractor outputs. private @MonotonicNonNull ExtractorOutput extractorOutput; private Mp4Track @MonotonicNonNull [] tracks; + private long @MonotonicNonNull [][] accumulatedSampleSizes; private int firstVideoTrackIndex; private long durationUs; - private boolean isQuickTime; + @FileType private int fileType; + @Nullable private MotionPhoto motionPhoto; /** * Creates a new extractor for unfragmented MP4 streams. @@ -290,6 +301,7 @@ public final class Mp4Extractor implements Extractor, SeekMap { if (atomHeaderBytesRead == 0) { // Read the standard length atom header. if (!input.readFully(atomHeader.getData(), 0, Atom.HEADER_SIZE, true)) { + processEndOfStreamReadingAtomHeader(); return false; } atomHeaderBytesRead = Atom.HEADER_SIZE; @@ -345,14 +357,7 @@ public final class Mp4Extractor implements Extractor, SeekMap { this.atomData = atomData; parserState = STATE_READING_ATOM_PAYLOAD; } else { - if (atomType == Atom.TYPE_mpvd && (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0) { - // There is no need to parse the mpvd atom payload. All the necessary information is in the - // header. - processMpvdBox( - /* atomStartPosition= */ input.getPosition() - atomHeaderBytesRead, - /* atomHeaderSize= */ atomHeaderBytesRead, - atomSize); - } + processUnparsedAtom(input.getPosition() - atomHeaderBytesRead); atomData = null; parserState = STATE_READING_ATOM_PAYLOAD; } @@ -374,7 +379,7 @@ public final class Mp4Extractor implements Extractor, SeekMap { if (atomData != null) { input.readFully(atomData.getData(), atomHeaderBytesRead, (int) atomPayloadSize); if (atomType == Atom.TYPE_ftyp) { - isQuickTime = processFtypAtom(atomData); + fileType = processFtypAtom(atomData); } else if (!containerAtoms.isEmpty()) { containerAtoms.peek().add(new Atom.LeafAtom(atomType, atomData)); } @@ -418,6 +423,7 @@ public final class Mp4Extractor implements Extractor, SeekMap { // Process metadata. @Nullable Metadata udtaMetadata = null; + boolean isQuickTime = fileType == FILE_TYPE_QUICKTIME; GaplessInfoHolder gaplessInfoHolder = new GaplessInfoHolder(); @Nullable Atom.LeafAtom udta = moov.getLeafAtomOfType(Atom.TYPE_udta); if (udta != null) { @@ -655,6 +661,19 @@ public final class Mp4Extractor implements Extractor, SeekMap { } } + /** Processes the end of stream in case there is not atom left to read. */ + private void processEndOfStreamReadingAtomHeader() { + if (fileType == FILE_TYPE_HEIC && (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0) { + // Add image track and prepare media. + ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput); + TrackOutput trackOutput = extractorOutput.track(/* id= */ 0, C.TRACK_TYPE_IMAGE); + @Nullable Metadata metadata = motionPhoto == null ? null : new Metadata(motionPhoto); + trackOutput.format(new Format.Builder().setMetadata(metadata).build()); + extractorOutput.endTracks(); + extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET)); + } + } + /** * Possibly skips the version and flags fields (1+3 byte) of a full meta atom of the {@code * input}. @@ -680,24 +699,18 @@ public final class Mp4Extractor implements Extractor, SeekMap { } } - /** - * Processes the Motion Photo Video Data of an HEIC motion photo following the Google Photos - * Motion Photo File Format V1.1. This consists in adding a track with the motion photo metadata - * and ending playback preparation. - */ - private void processMpvdBox(long atomStartPosition, int atomHeaderSize, long atomSize) { - ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput); - extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET)); - - TrackOutput trackOutput = extractorOutput.track(/* id= */ 0, C.TRACK_TYPE_IMAGE); - MotionPhoto motionPhoto = - new MotionPhoto( - /* photoStartPosition= */ 0, - /* photoSize= */ atomStartPosition, - /* videoStartPosition= */ atomStartPosition + atomHeaderSize, - /* videoSize= */ atomSize - atomHeaderSize); - trackOutput.format(new Format.Builder().setMetadata(new Metadata(motionPhoto)).build()); - extractorOutput.endTracks(); + /** Processes an atom whose payload does not need to be parsed. */ + private void processUnparsedAtom(long atomStartPosition) { + if (atomType == Atom.TYPE_mpvd) { + // The input is an HEIC motion photo following the Google Photos Motion Photo File Format + // V1.1. + motionPhoto = + new MotionPhoto( + /* photoStartPosition= */ 0, + /* photoSize= */ atomStartPosition, + /* videoStartPosition= */ atomStartPosition + atomHeaderBytesRead, + /* videoSize= */ atomSize - atomHeaderBytesRead); + } } /** @@ -779,24 +792,39 @@ public final class Mp4Extractor implements Extractor, SeekMap { } /** - * Process an ftyp atom to determine whether the media is QuickTime. + * Process an ftyp atom to determine the corresponding {@link FileType}. * * @param atomData The ftyp atom data. - * @return Whether the media is QuickTime. + * @return The {@link FileType}. */ - private static boolean processFtypAtom(ParsableByteArray atomData) { + @FileType + private static int processFtypAtom(ParsableByteArray atomData) { atomData.setPosition(Atom.HEADER_SIZE); int majorBrand = atomData.readInt(); - if (majorBrand == BRAND_QUICKTIME) { - return true; + @FileType int fileType = brandToFileType(majorBrand); + if (fileType != FILE_TYPE_MP4) { + return fileType; } atomData.skipBytes(4); // minor_version while (atomData.bytesLeft() > 0) { - if (atomData.readInt() == BRAND_QUICKTIME) { - return true; + fileType = brandToFileType(atomData.readInt()); + if (fileType != FILE_TYPE_MP4) { + return fileType; } } - return false; + return FILE_TYPE_MP4; + } + + @FileType + private static int brandToFileType(int brand) { + switch (brand) { + case BRAND_QUICKTIME: + return FILE_TYPE_QUICKTIME; + case BRAND_HEIC: + return FILE_TYPE_HEIC; + default: + return FILE_TYPE_MP4; + } } /** Returns whether the extractor should decode a leaf atom with type {@code atom}. */ diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java index f830c86edb..409931b1de 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java @@ -26,6 +26,11 @@ import java.io.IOException; */ /* package */ final class Sniffer { + /** Brand stored in the ftyp atom for QuickTime media. */ + public static final int BRAND_QUICKTIME = 0x71742020; + /** Brand stored in the ftyp atom for HEIC media. */ + public static final int BRAND_HEIC = 0x68656963; + /** The maximum number of bytes to peek when sniffing. */ private static final int SEARCH_LENGTH = 4 * 1024; @@ -54,7 +59,7 @@ import java.io.IOException; 0x66347620, // f4v[space] 0x6b646469, // kddi 0x4d345650, // M4VP - 0x71742020, // qt[space][space], Apple QuickTime + BRAND_QUICKTIME, // qt[space][space] 0x4d534e56, // MSNV, Sony PSP 0x64627931, // dby1, Dolby Vision 0x69736d6c, // isml @@ -203,8 +208,7 @@ import java.io.IOException; if (brand >>> 8 == 0x00336770) { // Brand starts with '3gp'. return true; - } else if (brand == 0x68656963 && acceptHeic) { - // Brand is `heic` and HEIC is supported by the extractor. + } else if (brand == BRAND_HEIC && acceptHeic) { return true; } for (int compatibleBrand : COMPATIBLE_BRANDS) { diff --git a/testdata/src/test/assets/media/mp4/sample_still_photo.heic b/testdata/src/test/assets/media/mp4/sample_still_photo.heic new file mode 100644 index 0000000000..0a673803b7 Binary files /dev/null and b/testdata/src/test/assets/media/mp4/sample_still_photo.heic differ