Slomo flattening: get metadata from smta box

Retrieve the capture frame rate and the SVC temporal layer count from
the smta box instead of the meta box because this is what Samsung do. It
is not guaranteed that the meta box will be present and will contain all
the necessary info in all slomo files.

PiperOrigin-RevId: 345639680

library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MdtaMetadataEntry.java

@@ -30,14 +30,6 @@ public final class MdtaMetadataEntry implements Metadata.Entry {
 
   /** Key for the capture frame rate (in frames per second). */
   public static final String KEY_ANDROID_CAPTURE_FPS = "com.android.capture.fps";
-  /** Key for the temporal SVC layer count. */
-  public static final String KEY_ANDROID_TEMPORAL_LAYER_COUNT =
-      "com.android.video.temporal_layers_count";
-
-  /** Type indicator for a 32-bit floating point value. */
-  public static final int TYPE_INDICATOR_FLOAT = 23;
-  /** Type indicator for a 32-bit integer. */
-  public static final int TYPE_INDICATOR_INT = 67;
 
   /** The metadata key name. */
   public final String key;

library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntry.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.metadata.mp4;
+
+import android.os.Parcel;
+import android.os.Parcelable;
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.common.primitives.Floats;
+
+/**
+ * Stores metadata from the Samsung smta box.
+ *
+ * <p>See [Internal: b/150138465#comment76].
+ */
+public final class SmtaMetadataEntry implements Metadata.Entry {
+
+  /**
+   * The capture frame rate, in fps, or {@link C#RATE_UNSET} if it is unknown.
+   *
+   * <p>If known, the capture frame rate should always be an integer value.
+   */
+  public final float captureFrameRate;
+  /** The number of layers in the SVC extended frames. */
+  public final int svcTemporalLayerCount;
+
+  /** Creates an instance. */
+  public SmtaMetadataEntry(float captureFrameRate, int svcTemporalLayerCount) {
+    this.captureFrameRate = captureFrameRate;
+    this.svcTemporalLayerCount = svcTemporalLayerCount;
+  }
+
+  private SmtaMetadataEntry(Parcel in) {
+    captureFrameRate = in.readFloat();
+    svcTemporalLayerCount = in.readInt();
+  }
+
+  @Override
+  public boolean equals(@Nullable Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    SmtaMetadataEntry other = (SmtaMetadataEntry) obj;
+    return captureFrameRate == other.captureFrameRate
+        && svcTemporalLayerCount == other.svcTemporalLayerCount;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = 17;
+    result = 31 * result + Floats.hashCode(captureFrameRate);
+    result = 31 * result + svcTemporalLayerCount;
+    return result;
+  }
+
+  @Override
+  public String toString() {
+    return "smta: captureFrameRate="
+        + captureFrameRate
+        + ", svcTemporalLayerCount="
+        + svcTemporalLayerCount;
+  }
+
+  // Parcelable implementation.
+
+  @Override
+  public void writeToParcel(Parcel dest, int flags) {
+    dest.writeFloat(captureFrameRate);
+    dest.writeInt(svcTemporalLayerCount);
+  }
+
+  @Override
+  public int describeContents() {
+    return 0;
+  }
+
+  public static final Parcelable.Creator<SmtaMetadataEntry> CREATOR =
+      new Parcelable.Creator<SmtaMetadataEntry>() {
+
+        @Override
+        public SmtaMetadataEntry createFromParcel(Parcel in) {
+          return new SmtaMetadataEntry(in);
+        }
+
+        @Override
+        public SmtaMetadataEntry[] newArray(int size) {
+          return new SmtaMetadataEntry[size];
+        }
+      };
+}

library/common/src/test/java/com/google/android/exoplayer2/metadata/mp4/SmtaMetadataEntryTest.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.metadata.mp4;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import android.os.Parcel;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Test for {@link SmtaMetadataEntry}. */
+@RunWith(AndroidJUnit4.class)
+public class SmtaMetadataEntryTest {
+
+  @Test
+  public void parcelable() {
+    SmtaMetadataEntry smtaMetadataEntryToParcel =
+        new SmtaMetadataEntry(/* captureFrameRate= */ 120, /* svcTemporalLayerCount= */ 4);
+
+    Parcel parcel = Parcel.obtain();
+    smtaMetadataEntryToParcel.writeToParcel(parcel, /* flags= */ 0);
+    parcel.setDataPosition(0);
+
+    SmtaMetadataEntry smtaMetadataEntryFromParcel =
+        SmtaMetadataEntry.CREATOR.createFromParcel(parcel);
+    assertThat(smtaMetadataEntryFromParcel).isEqualTo(smtaMetadataEntryToParcel);
+
+    parcel.recycle();
+  }
+}

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java

@@ -334,6 +334,12 @@ import java.util.List;
   @SuppressWarnings("ConstantCaseForConstants")
   public static final int TYPE_meta = 0x6d657461;
 
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_smta = 0x736d7461;
+
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_saut = 0x73617574;
+
   @SuppressWarnings("ConstantCaseForConstants")
   public static final int TYPE_keys = 0x6b657973;
 

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java

@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.audio.OpusUtil;
 import com.google.android.exoplayer2.drm.DrmInitData;
 import com.google.android.exoplayer2.extractor.GaplessInfoHolder;
 import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.android.exoplayer2.metadata.mp4.SmtaMetadataEntry;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.CodecSpecificDataUtil;
 import com.google.android.exoplayer2.util.Log;
@@ -145,28 +146,30 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
    *
    * @param udtaAtom The udta (user data) atom to decode.
    * @param isQuickTime True for QuickTime media. False otherwise.
-   * @return Parsed metadata, or null.
+   * @return A {@link Pair} containing the metadata from the meta child atom as first value (if
+   *     any), and the metadata from the smta child atom as second value (if any).
    */
-  @Nullable
-  public static Metadata parseUdta(Atom.LeafAtom udtaAtom, boolean isQuickTime) {
-    if (isQuickTime) {
-      // Meta boxes are regular boxes rather than full boxes in QuickTime. For now, don't try and
-      // decode one.
-      return null;
-    }
+  public static Pair<@NullableType Metadata, @NullableType Metadata> parseUdta(
+      Atom.LeafAtom udtaAtom, boolean isQuickTime) {
     ParsableByteArray udtaData = udtaAtom.data;
     udtaData.setPosition(Atom.HEADER_SIZE);
+    @Nullable Metadata metaMetadata = null;
+    @Nullable Metadata smtaMetadata = null;
     while (udtaData.bytesLeft() >= Atom.HEADER_SIZE) {
       int atomPosition = udtaData.getPosition();
       int atomSize = udtaData.readInt();
       int atomType = udtaData.readInt();
-      if (atomType == Atom.TYPE_meta) {
+      // Meta boxes are regular boxes rather than full boxes in QuickTime. Ignore them for now.
+      if (atomType == Atom.TYPE_meta && !isQuickTime) {
         udtaData.setPosition(atomPosition);
-        return parseUdtaMeta(udtaData, atomPosition + atomSize);
+        metaMetadata = parseUdtaMeta(udtaData, atomPosition + atomSize);
+      } else if (atomType == Atom.TYPE_smta) {
+        udtaData.setPosition(atomPosition);
+        smtaMetadata = parseSmta(udtaData, atomPosition + atomSize);
       }
       udtaData.setPosition(atomPosition + atomSize);
     }
-    return null;
+    return Pair.create(metaMetadata, smtaMetadata);
   }
 
   /**
@@ -701,6 +704,38 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
     return entries.isEmpty() ? null : new Metadata(entries);
   }
 
+  /**
+   * Parses metadata from a Samsung smta atom.
+   *
+   * <p>See [Internal: b/150138465#comment76].
+   */
+  @Nullable
+  private static Metadata parseSmta(ParsableByteArray smta, int limit) {
+    smta.skipBytes(Atom.FULL_HEADER_SIZE);
+    while (smta.getPosition() < limit) {
+      int atomPosition = smta.getPosition();
+      int atomSize = smta.readInt();
+      int atomType = smta.readInt();
+      if (atomType == Atom.TYPE_saut) {
+        smta.skipBytes(5); // author (4), reserved = 0 (1).
+        int recordingMode = smta.readUnsignedByte();
+        float captureFrameRate;
+        if (recordingMode == 12) {
+          captureFrameRate = 240;
+        } else if (recordingMode == 13) {
+          captureFrameRate = 120;
+        } else {
+          captureFrameRate = C.RATE_UNSET;
+        }
+        smta.skipBytes(1); // reserved = 1 (1).
+        int svcTemporalLayerCount = smta.readUnsignedByte();
+        return new Metadata(new SmtaMetadataEntry(captureFrameRate, svcTemporalLayerCount));
+      }
+      smta.setPosition(atomPosition + atomSize);
+    }
+    return null;
+  }
+
   /**
    * Parses a mvhd atom (defined in ISO/IEC 14496-12), returning the timescale for the movie.
    *

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/MetadataUtil.java

@@ -31,8 +31,6 @@ import com.google.android.exoplayer2.metadata.id3.TextInformationFrame;
 import com.google.android.exoplayer2.metadata.mp4.MdtaMetadataEntry;
 import com.google.android.exoplayer2.util.Log;
 import com.google.android.exoplayer2.util.ParsableByteArray;
-import java.util.ArrayList;
-import java.util.List;
 
 /** Utilities for handling metadata in MP4. */
 /* package */ final class MetadataUtil {
@@ -290,32 +288,34 @@ import java.util.List;
   /** Updates a {@link Format.Builder} to include metadata from the provided sources. */
   public static void setFormatMetadata(
       int trackType,
-      @Nullable Metadata udtaMetadata,
+      @Nullable Metadata udtaMetaMetadata,
       @Nullable Metadata mdtaMetadata,
+      @Nullable Metadata smtaMetadata,
       Format.Builder formatBuilder,
       Metadata.Entry... additionalEntries) {
     Metadata formatMetadata = new Metadata();
 
     if (trackType == C.TRACK_TYPE_AUDIO) {
-      // We assume all udta metadata is associated with the audio track.
-      if (udtaMetadata != null) {
-        formatMetadata = udtaMetadata;
+      // We assume all meta metadata in the udta box is associated with the audio track.
+      if (udtaMetaMetadata != null) {
+        formatMetadata = udtaMetaMetadata;
       }
-    } else if (trackType == C.TRACK_TYPE_VIDEO && mdtaMetadata != null) {
+    } else if (trackType == C.TRACK_TYPE_VIDEO) {
       // Populate only metadata keys that are known to be specific to video.
-      List<MdtaMetadataEntry> mdtaMetadataEntries = new ArrayList<>();
-      for (int i = 0; i < mdtaMetadata.length(); i++) {
-        Metadata.Entry entry = mdtaMetadata.get(i);
-        if (entry instanceof MdtaMetadataEntry) {
-          MdtaMetadataEntry mdtaMetadataEntry = (MdtaMetadataEntry) entry;
-          if (MdtaMetadataEntry.KEY_ANDROID_CAPTURE_FPS.equals(mdtaMetadataEntry.key)
-              || MdtaMetadataEntry.KEY_ANDROID_TEMPORAL_LAYER_COUNT.equals(mdtaMetadataEntry.key)) {
-            mdtaMetadataEntries.add(mdtaMetadataEntry);
+      if (mdtaMetadata != null) {
+        for (int i = 0; i < mdtaMetadata.length(); i++) {
+          Metadata.Entry entry = mdtaMetadata.get(i);
+          if (entry instanceof MdtaMetadataEntry) {
+            MdtaMetadataEntry mdtaMetadataEntry = (MdtaMetadataEntry) entry;
+            if (MdtaMetadataEntry.KEY_ANDROID_CAPTURE_FPS.equals(mdtaMetadataEntry.key)) {
+              formatMetadata = new Metadata(mdtaMetadataEntry);
+              break;
+            }
           }
         }
       }
-      if (!mdtaMetadataEntries.isEmpty()) {
-        formatMetadata = new Metadata(mdtaMetadataEntries);
+      if (smtaMetadata != null) {
+        formatMetadata = formatMetadata.copyWithAppendedEntriesFrom(smtaMetadata);
       }
     }
 

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java

@@ -23,6 +23,7 @@ import static com.google.android.exoplayer2.util.Util.castNonNull;
 import static java.lang.Math.max;
 import static java.lang.Math.min;
 
+import android.util.Pair;
 import androidx.annotation.IntDef;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
@@ -53,6 +54,7 @@ import java.lang.annotation.RetentionPolicy;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.List;
+import org.checkerframework.checker.nullness.compatqual.NullableType;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
@@ -461,14 +463,18 @@ public final class Mp4Extractor implements Extractor, SeekMap {
     List<Mp4Track> tracks = new ArrayList<>();
 
     // Process metadata.
-    @Nullable Metadata udtaMetadata = null;
+    @Nullable Metadata udtaMetaMetadata = null;
+    @Nullable Metadata smtaMetadata = null;
     boolean isQuickTime = fileType == FILE_TYPE_QUICKTIME;
     GaplessInfoHolder gaplessInfoHolder = new GaplessInfoHolder();
     @Nullable Atom.LeafAtom udta = moov.getLeafAtomOfType(Atom.TYPE_udta);
     if (udta != null) {
-      udtaMetadata = AtomParsers.parseUdta(udta, isQuickTime);
-      if (udtaMetadata != null) {
-        gaplessInfoHolder.setFromMetadata(udtaMetadata);
+      Pair<@NullableType Metadata, @NullableType Metadata> udtaMetadata =
+          AtomParsers.parseUdta(udta, isQuickTime);
+      udtaMetaMetadata = udtaMetadata.first;
+      smtaMetadata = udtaMetadata.second;
+      if (udtaMetaMetadata != null) {
+        gaplessInfoHolder.setFromMetadata(udtaMetaMetadata);
       }
     }
     @Nullable Metadata mdtaMetadata = null;
@@ -517,8 +523,9 @@ public final class Mp4Extractor implements Extractor, SeekMap {
       MetadataUtil.setFormatGaplessInfo(track.type, gaplessInfoHolder, formatBuilder);
       MetadataUtil.setFormatMetadata(
           track.type,
-          udtaMetadata,
+          udtaMetaMetadata,
           mdtaMetadata,
+          smtaMetadata,
           formatBuilder,
           /* additionalEntries...= */ slowMotionMetadataEntries.toArray(new Metadata.Entry[0]));
       mp4Track.trackOutput.format(formatBuilder.build());