From 3a17dd5fec62a84b3cd9832516394ae8b49209bf Mon Sep 17 00:00:00 2001 From: kimvde Date: Fri, 4 Dec 2020 10:46:46 +0000 Subject: [PATCH] Slomo flattening: get metadata from smta box Retrieve the capture frame rate and the SVC temporal layer count from the smta box instead of the meta box because this is what Samsung do. It is not guaranteed that the meta box will be present and will contain all the necessary info in all slomo files. PiperOrigin-RevId: 345639680 --- .../metadata/mp4/MdtaMetadataEntry.java | 8 -- .../metadata/mp4/SmtaMetadataEntry.java | 107 ++++++++++++++++++ .../metadata/mp4/SmtaMetadataEntryTest.java | 44 +++++++ .../exoplayer2/extractor/mp4/Atom.java | 6 + .../exoplayer2/extractor/mp4/AtomParsers.java | 57 ++++++++-- .../extractor/mp4/MetadataUtil.java | 34 +++--- .../extractor/mp4/Mp4Extractor.java | 17 ++- .../media/mp4/sample_sef_slow_motion.mp4 | Bin 53069 -> 53149 bytes .../mp4/sample_sef_super_slow_motion.mp4 | Bin 64755 -> 64783 bytes 9 files changed, 232 insertions(+), 41 deletions(-) create mode 100644 library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntry.java create mode 100644 library/common/src/test/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntryTest.java diff --git a/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MdtaMetadataEntry.java b/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MdtaMetadataEntry.java index 0f41c46c74..5b2db4945c 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MdtaMetadataEntry.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MdtaMetadataEntry.java @@ -30,14 +30,6 @@ public final class MdtaMetadataEntry implements Metadata.Entry { /** Key for the capture frame rate (in frames per second). */ public static final String KEY_ANDROID_CAPTURE_FPS = "com.android.capture.fps"; - /** Key for the temporal SVC layer count. */ - public static final String KEY_ANDROID_TEMPORAL_LAYER_COUNT = - "com.android.video.temporal_layers_count"; - - /** Type indicator for a 32-bit floating point value. */ - public static final int TYPE_INDICATOR_FLOAT = 23; - /** Type indicator for a 32-bit integer. */ - public static final int TYPE_INDICATOR_INT = 67; /** The metadata key name. */ public final String key; diff --git a/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntry.java b/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntry.java new file mode 100644 index 0000000000..6654a9dbb6 --- /dev/null +++ b/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntry.java @@ -0,0 +1,107 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.metadata.mp4; + +import android.os.Parcel; +import android.os.Parcelable; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.metadata.Metadata; +import com.google.common.primitives.Floats; + +/** + * Stores metadata from the Samsung smta box. + * + *

See [Internal: b/150138465#comment76]. + */ +public final class SmtaMetadataEntry implements Metadata.Entry { + + /** + * The capture frame rate, in fps, or {@link C#RATE_UNSET} if it is unknown. + * + *

If known, the capture frame rate should always be an integer value. + */ + public final float captureFrameRate; + /** The number of layers in the SVC extended frames. */ + public final int svcTemporalLayerCount; + + /** Creates an instance. */ + public SmtaMetadataEntry(float captureFrameRate, int svcTemporalLayerCount) { + this.captureFrameRate = captureFrameRate; + this.svcTemporalLayerCount = svcTemporalLayerCount; + } + + private SmtaMetadataEntry(Parcel in) { + captureFrameRate = in.readFloat(); + svcTemporalLayerCount = in.readInt(); + } + + @Override + public boolean equals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SmtaMetadataEntry other = (SmtaMetadataEntry) obj; + return captureFrameRate == other.captureFrameRate + && svcTemporalLayerCount == other.svcTemporalLayerCount; + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + Floats.hashCode(captureFrameRate); + result = 31 * result + svcTemporalLayerCount; + return result; + } + + @Override + public String toString() { + return "smta: captureFrameRate=" + + captureFrameRate + + ", svcTemporalLayerCount=" + + svcTemporalLayerCount; + } + + // Parcelable implementation. + + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeFloat(captureFrameRate); + dest.writeInt(svcTemporalLayerCount); + } + + @Override + public int describeContents() { + return 0; + } + + public static final Parcelable.Creator CREATOR = + new Parcelable.Creator() { + + @Override + public SmtaMetadataEntry createFromParcel(Parcel in) { + return new SmtaMetadataEntry(in); + } + + @Override + public SmtaMetadataEntry[] newArray(int size) { + return new SmtaMetadataEntry[size]; + } + }; +} diff --git a/library/common/src/test/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntryTest.java b/library/common/src/test/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntryTest.java new file mode 100644 index 0000000000..7cc48a8021 --- /dev/null +++ b/library/common/src/test/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntryTest.java @@ -0,0 +1,44 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.metadata.mp4; + +import static com.google.common.truth.Truth.assertThat; + +import android.os.Parcel; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Test for {@link SmtaMetadataEntry}. */ +@RunWith(AndroidJUnit4.class) +public class SmtaMetadataEntryTest { + + @Test + public void parcelable() { + SmtaMetadataEntry smtaMetadataEntryToParcel = + new SmtaMetadataEntry(/* captureFrameRate= */ 120, /* svcTemporalLayerCount= */ 4); + + Parcel parcel = Parcel.obtain(); + smtaMetadataEntryToParcel.writeToParcel(parcel, /* flags= */ 0); + parcel.setDataPosition(0); + + SmtaMetadataEntry smtaMetadataEntryFromParcel = + SmtaMetadataEntry.CREATOR.createFromParcel(parcel); + assertThat(smtaMetadataEntryFromParcel).isEqualTo(smtaMetadataEntryToParcel); + + parcel.recycle(); + } +} diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java index 1a19358b57..95cd1e2c17 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java @@ -334,6 +334,12 @@ import java.util.List; @SuppressWarnings("ConstantCaseForConstants") public static final int TYPE_meta = 0x6d657461; + @SuppressWarnings("ConstantCaseForConstants") + public static final int TYPE_smta = 0x736d7461; + + @SuppressWarnings("ConstantCaseForConstants") + public static final int TYPE_saut = 0x73617574; + @SuppressWarnings("ConstantCaseForConstants") public static final int TYPE_keys = 0x6b657973; diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java index 551ebc3ea3..2571df954d 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java @@ -31,6 +31,7 @@ import com.google.android.exoplayer2.audio.OpusUtil; import com.google.android.exoplayer2.drm.DrmInitData; import com.google.android.exoplayer2.extractor.GaplessInfoHolder; import com.google.android.exoplayer2.metadata.Metadata; +import com.google.android.exoplayer2.metadata.mp4.SmtaMetadataEntry; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.CodecSpecificDataUtil; import com.google.android.exoplayer2.util.Log; @@ -145,28 +146,30 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; * * @param udtaAtom The udta (user data) atom to decode. * @param isQuickTime True for QuickTime media. False otherwise. - * @return Parsed metadata, or null. + * @return A {@link Pair} containing the metadata from the meta child atom as first value (if + * any), and the metadata from the smta child atom as second value (if any). */ - @Nullable - public static Metadata parseUdta(Atom.LeafAtom udtaAtom, boolean isQuickTime) { - if (isQuickTime) { - // Meta boxes are regular boxes rather than full boxes in QuickTime. For now, don't try and - // decode one. - return null; - } + public static Pair<@NullableType Metadata, @NullableType Metadata> parseUdta( + Atom.LeafAtom udtaAtom, boolean isQuickTime) { ParsableByteArray udtaData = udtaAtom.data; udtaData.setPosition(Atom.HEADER_SIZE); + @Nullable Metadata metaMetadata = null; + @Nullable Metadata smtaMetadata = null; while (udtaData.bytesLeft() >= Atom.HEADER_SIZE) { int atomPosition = udtaData.getPosition(); int atomSize = udtaData.readInt(); int atomType = udtaData.readInt(); - if (atomType == Atom.TYPE_meta) { + // Meta boxes are regular boxes rather than full boxes in QuickTime. Ignore them for now. + if (atomType == Atom.TYPE_meta && !isQuickTime) { udtaData.setPosition(atomPosition); - return parseUdtaMeta(udtaData, atomPosition + atomSize); + metaMetadata = parseUdtaMeta(udtaData, atomPosition + atomSize); + } else if (atomType == Atom.TYPE_smta) { + udtaData.setPosition(atomPosition); + smtaMetadata = parseSmta(udtaData, atomPosition + atomSize); } udtaData.setPosition(atomPosition + atomSize); } - return null; + return Pair.create(metaMetadata, smtaMetadata); } /** @@ -701,6 +704,38 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; return entries.isEmpty() ? null : new Metadata(entries); } + /** + * Parses metadata from a Samsung smta atom. + * + *

See [Internal: b/150138465#comment76]. + */ + @Nullable + private static Metadata parseSmta(ParsableByteArray smta, int limit) { + smta.skipBytes(Atom.FULL_HEADER_SIZE); + while (smta.getPosition() < limit) { + int atomPosition = smta.getPosition(); + int atomSize = smta.readInt(); + int atomType = smta.readInt(); + if (atomType == Atom.TYPE_saut) { + smta.skipBytes(5); // author (4), reserved = 0 (1). + int recordingMode = smta.readUnsignedByte(); + float captureFrameRate; + if (recordingMode == 12) { + captureFrameRate = 240; + } else if (recordingMode == 13) { + captureFrameRate = 120; + } else { + captureFrameRate = C.RATE_UNSET; + } + smta.skipBytes(1); // reserved = 1 (1). + int svcTemporalLayerCount = smta.readUnsignedByte(); + return new Metadata(new SmtaMetadataEntry(captureFrameRate, svcTemporalLayerCount)); + } + smta.setPosition(atomPosition + atomSize); + } + return null; + } + /** * Parses a mvhd atom (defined in ISO/IEC 14496-12), returning the timescale for the movie. * diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/MetadataUtil.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/MetadataUtil.java index 416a63348c..4b00aa6452 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/MetadataUtil.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/MetadataUtil.java @@ -31,8 +31,6 @@ import com.google.android.exoplayer2.metadata.id3.TextInformationFrame; import com.google.android.exoplayer2.metadata.mp4.MdtaMetadataEntry; import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.ParsableByteArray; -import java.util.ArrayList; -import java.util.List; /** Utilities for handling metadata in MP4. */ /* package */ final class MetadataUtil { @@ -290,32 +288,34 @@ import java.util.List; /** Updates a {@link Format.Builder} to include metadata from the provided sources. */ public static void setFormatMetadata( int trackType, - @Nullable Metadata udtaMetadata, + @Nullable Metadata udtaMetaMetadata, @Nullable Metadata mdtaMetadata, + @Nullable Metadata smtaMetadata, Format.Builder formatBuilder, Metadata.Entry... additionalEntries) { Metadata formatMetadata = new Metadata(); if (trackType == C.TRACK_TYPE_AUDIO) { - // We assume all udta metadata is associated with the audio track. - if (udtaMetadata != null) { - formatMetadata = udtaMetadata; + // We assume all meta metadata in the udta box is associated with the audio track. + if (udtaMetaMetadata != null) { + formatMetadata = udtaMetaMetadata; } - } else if (trackType == C.TRACK_TYPE_VIDEO && mdtaMetadata != null) { + } else if (trackType == C.TRACK_TYPE_VIDEO) { // Populate only metadata keys that are known to be specific to video. - List mdtaMetadataEntries = new ArrayList<>(); - for (int i = 0; i < mdtaMetadata.length(); i++) { - Metadata.Entry entry = mdtaMetadata.get(i); - if (entry instanceof MdtaMetadataEntry) { - MdtaMetadataEntry mdtaMetadataEntry = (MdtaMetadataEntry) entry; - if (MdtaMetadataEntry.KEY_ANDROID_CAPTURE_FPS.equals(mdtaMetadataEntry.key) - || MdtaMetadataEntry.KEY_ANDROID_TEMPORAL_LAYER_COUNT.equals(mdtaMetadataEntry.key)) { - mdtaMetadataEntries.add(mdtaMetadataEntry); + if (mdtaMetadata != null) { + for (int i = 0; i < mdtaMetadata.length(); i++) { + Metadata.Entry entry = mdtaMetadata.get(i); + if (entry instanceof MdtaMetadataEntry) { + MdtaMetadataEntry mdtaMetadataEntry = (MdtaMetadataEntry) entry; + if (MdtaMetadataEntry.KEY_ANDROID_CAPTURE_FPS.equals(mdtaMetadataEntry.key)) { + formatMetadata = new Metadata(mdtaMetadataEntry); + break; + } } } } - if (!mdtaMetadataEntries.isEmpty()) { - formatMetadata = new Metadata(mdtaMetadataEntries); + if (smtaMetadata != null) { + formatMetadata = formatMetadata.copyWithAppendedEntriesFrom(smtaMetadata); } } diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java index ed6c948c96..506ceacaa5 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java @@ -23,6 +23,7 @@ import static com.google.android.exoplayer2.util.Util.castNonNull; import static java.lang.Math.max; import static java.lang.Math.min; +import android.util.Pair; import androidx.annotation.IntDef; import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; @@ -53,6 +54,7 @@ import java.lang.annotation.RetentionPolicy; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; +import org.checkerframework.checker.nullness.compatqual.NullableType; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -461,14 +463,18 @@ public final class Mp4Extractor implements Extractor, SeekMap { List tracks = new ArrayList<>(); // Process metadata. - @Nullable Metadata udtaMetadata = null; + @Nullable Metadata udtaMetaMetadata = null; + @Nullable Metadata smtaMetadata = null; boolean isQuickTime = fileType == FILE_TYPE_QUICKTIME; GaplessInfoHolder gaplessInfoHolder = new GaplessInfoHolder(); @Nullable Atom.LeafAtom udta = moov.getLeafAtomOfType(Atom.TYPE_udta); if (udta != null) { - udtaMetadata = AtomParsers.parseUdta(udta, isQuickTime); - if (udtaMetadata != null) { - gaplessInfoHolder.setFromMetadata(udtaMetadata); + Pair<@NullableType Metadata, @NullableType Metadata> udtaMetadata = + AtomParsers.parseUdta(udta, isQuickTime); + udtaMetaMetadata = udtaMetadata.first; + smtaMetadata = udtaMetadata.second; + if (udtaMetaMetadata != null) { + gaplessInfoHolder.setFromMetadata(udtaMetaMetadata); } } @Nullable Metadata mdtaMetadata = null; @@ -517,8 +523,9 @@ public final class Mp4Extractor implements Extractor, SeekMap { MetadataUtil.setFormatGaplessInfo(track.type, gaplessInfoHolder, formatBuilder); MetadataUtil.setFormatMetadata( track.type, - udtaMetadata, + udtaMetaMetadata, mdtaMetadata, + smtaMetadata, formatBuilder, /* additionalEntries...= */ slowMotionMetadataEntries.toArray(new Metadata.Entry[0])); mp4Track.trackOutput.format(formatBuilder.build()); diff --git a/testdata/src/test/assets/media/mp4/sample_sef_slow_motion.mp4 b/testdata/src/test/assets/media/mp4/sample_sef_slow_motion.mp4 index 8b436e0c94c076e595e3f900b5f1c77504b213ff..1440b883c219324aafe3b98dd51ca7a8d6513845 100644 GIT binary patch delta 99 zcmX>*k9qEV<_%YmFmZWJzJ8?GH=s17B$0uEK_J-0$1m75Fh0P?F%rZt&Mis_2?}*} q@(FbX@->Qc!Rmlapj={U35dbK!^i?6i;L2V7(OsCY<54&#svTb-5O~C delta 21 dcmbO`pZV-O<_%YmFme8xeEmrA=2=IXxd3?+3cLUS diff --git a/testdata/src/test/assets/media/mp4/sample_sef_super_slow_motion.mp4 b/testdata/src/test/assets/media/mp4/sample_sef_super_slow_motion.mp4 index ab3b2da1345553a2d6ee4fdf498ec53878b8d955..3999e7129f491afd6ba070122a412627a78341ca 100644 GIT binary patch delta 62 zcmezTlezyF^M+Tq87EDCeY==3U~>E&b!Cm>+>%5F2oNYvEG+>u*%{e@G*59+S`mZX J=H5GL3IO{B6Z!xE delta 34 qcmeDG#r*jv^M+Tq8OtWWzFo{{GCBT^Iy+BsZb>2o!{(KD(i8wSyAR|5