diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MetadataRetrieverTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MetadataRetrieverTest.java index d709bfd3f9..1ded8940ea 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MetadataRetrieverTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MetadataRetrieverTest.java @@ -25,6 +25,7 @@ import android.content.Context; import android.net.Uri; import androidx.media3.common.C; import androidx.media3.common.MediaItem; +import androidx.media3.common.Metadata; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Util; import androidx.media3.container.MdtaMetadataEntry; @@ -36,6 +37,7 @@ import androidx.media3.extractor.metadata.mp4.SmtaMetadataEntry; import androidx.media3.test.utils.FakeClock; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; import java.util.ArrayList; import java.util.List; @@ -151,7 +153,7 @@ public class MetadataRetrieverTest { } @Test - public void retrieveMetadata_sefSlowMotion_outputsExpectedMetadata() throws Exception { + public void retrieveMetadata_sefSlowMotionAvc_outputsExpectedMetadata() throws Exception { MediaItem mediaItem = MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_sef_slow_motion.mp4")); MdtaMetadataEntry expectedAndroidVersionMetadata = @@ -216,6 +218,60 @@ public class MetadataRetrieverTest { assertThat(trackGroups.get(1).getFormat(0).metadata.get(5)).isEqualTo(expectedMp4TimestampData); } + @Test + public void retrieveMetadata_sefSlowMotionHevc_outputsExpectedMetadata() throws Exception { + MediaItem mediaItem = + MediaItem.fromUri( + Uri.parse("asset://android_asset/media/mp4/sample_sef_slow_motion_hevc.mp4")); + MdtaMetadataEntry expectedAndroidVersionMetadata = + new MdtaMetadataEntry( + /* key= */ "com.android.version", + /* value= */ Util.getUtf8Bytes("13"), + /* localeIndicator= */ 0, + MdtaMetadataEntry.TYPE_INDICATOR_STRING); + SmtaMetadataEntry expectedSmtaEntry = + new SmtaMetadataEntry(/* captureFrameRate= */ 240, /* svcTemporalLayerCount= */ 4); + SlowMotionData expectedSlowMotionData = + new SlowMotionData( + ImmutableList.of( + new SlowMotionData.Segment( + /* startTimeMs= */ 2128, /* endTimeMs= */ 9856, /* speedDivisor= */ 8))); + MdtaMetadataEntry expectedCaptureFpsMdtaEntry = + new MdtaMetadataEntry( + KEY_ANDROID_CAPTURE_FPS, + /* value= */ new byte[] {67, 112, 0, 0}, + /* localeIndicator= */ 0, + /* typeIndicator= */ 23); + + ListenableFuture trackGroupsFuture = + retrieveMetadata(context, mediaItem, clock); + ShadowLooper.idleMainLooper(); + TrackGroupArray trackGroups = trackGroupsFuture.get(TEST_TIMEOUT_SEC, TimeUnit.SECONDS); + + assertThat(trackGroups.length).isEqualTo(2); // Video and audio + + // Video + Metadata videoFormatMetadata = trackGroups.get(0).getFormat(0).metadata; + List videoMetadataEntries = new ArrayList<>(); + for (int i = 0; i < videoFormatMetadata.length(); i++) { + videoMetadataEntries.add(videoFormatMetadata.get(i)); + } + assertThat(videoMetadataEntries).contains(expectedAndroidVersionMetadata); + assertThat(videoMetadataEntries).contains(expectedSlowMotionData); + assertThat(videoMetadataEntries).contains(expectedSmtaEntry); + assertThat(videoMetadataEntries).contains(expectedCaptureFpsMdtaEntry); + + // Audio + Metadata audioFormatMetadata = trackGroups.get(1).getFormat(0).metadata; + List audioMetadataEntries = new ArrayList<>(); + for (int i = 0; i < audioFormatMetadata.length(); i++) { + audioMetadataEntries.add(audioFormatMetadata.get(i)); + } + assertThat(audioMetadataEntries).contains(expectedAndroidVersionMetadata); + assertThat(audioMetadataEntries).contains(expectedSlowMotionData); + assertThat(audioMetadataEntries).contains(expectedSmtaEntry); + } + @Test public void retrieveMetadata_invalidMediaItem_throwsError() { MediaItem mediaItem = diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/metadata/mp4/SmtaMetadataEntry.java b/libraries/extractor/src/main/java/androidx/media3/extractor/metadata/mp4/SmtaMetadataEntry.java index f0a747f3c2..b92fc3e495 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/metadata/mp4/SmtaMetadataEntry.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/metadata/mp4/SmtaMetadataEntry.java @@ -26,7 +26,7 @@ import com.google.common.primitives.Floats; /** * Stores metadata from the Samsung smta box. * - *

See [Internal: b/150138465#comment76]. + *

See [Internal: b/150138465#comment76], [Internal: b/301273734#comment17]. */ @UnstableApi public final class SmtaMetadataEntry implements Metadata.Entry { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Atom.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Atom.java index 8bf730de4d..91d2a1dc6f 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Atom.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Atom.java @@ -359,6 +359,9 @@ import java.util.List; @SuppressWarnings("ConstantCaseForConstants") public static final int TYPE_saut = 0x73617574; + @SuppressWarnings("ConstantCaseForConstants") + public static final int TYPE_srfr = 0x73726672; + @SuppressWarnings("ConstantCaseForConstants") public static final int TYPE_keys = 0x6b657973; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java index 15d5aa0428..a40f9a78b9 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java @@ -46,7 +46,6 @@ import androidx.media3.extractor.GaplessInfoHolder; import androidx.media3.extractor.HevcConfig; import androidx.media3.extractor.OpusUtil; import androidx.media3.extractor.VorbisUtil; -import androidx.media3.extractor.metadata.mp4.SmtaMetadataEntry; import androidx.media3.extractor.mp4.Atom.LeafAtom; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; @@ -177,7 +176,8 @@ import java.util.List; } else if (atomType == Atom.TYPE_smta) { udtaData.setPosition(atomPosition); metadata = - metadata.copyWithAppendedEntriesFrom(parseSmta(udtaData, atomPosition + atomSize)); + metadata.copyWithAppendedEntriesFrom( + SmtaAtomUtil.parseSmta(udtaData, atomPosition + atomSize)); } else if (atomType == Atom.TYPE_xyz) { metadata = metadata.copyWithAppendedEntriesFrom(parseXyz(udtaData)); } @@ -825,37 +825,6 @@ import java.util.List; } } - /** - * Parses metadata from a Samsung smta atom. - * - *

See [Internal: b/150138465#comment76]. - */ - @Nullable - private static Metadata parseSmta(ParsableByteArray smta, int limit) { - smta.skipBytes(Atom.FULL_HEADER_SIZE); - while (smta.getPosition() < limit) { - int atomPosition = smta.getPosition(); - int atomSize = smta.readInt(); - int atomType = smta.readInt(); - if (atomType == Atom.TYPE_saut) { - if (atomSize < 14) { - return null; - } - smta.skipBytes(5); // author (4), reserved = 0 (1). - int recordingMode = smta.readUnsignedByte(); - if (recordingMode != 12 && recordingMode != 13) { - return null; - } - float captureFrameRate = recordingMode == 12 ? 240 : 120; - smta.skipBytes(1); // reserved = 1 (1). - int svcTemporalLayerCount = smta.readUnsignedByte(); - return new Metadata(new SmtaMetadataEntry(captureFrameRate, svcTemporalLayerCount)); - } - smta.setPosition(atomPosition + atomSize); - } - return null; - } - /** * Parses a tkhd atom (defined in ISO/IEC 14496-12). * diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/SmtaAtomUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/SmtaAtomUtil.java new file mode 100644 index 0000000000..e2493b7c04 --- /dev/null +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/SmtaAtomUtil.java @@ -0,0 +1,141 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.extractor.mp4; + +import static java.lang.annotation.ElementType.TYPE_USE; + +import androidx.annotation.IntDef; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Metadata; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.extractor.metadata.mp4.SmtaMetadataEntry; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Utility methods for handling SMTA atoms. + * + *

See [Internal: b/150138465#comment76], [Internal: b/301273734#comment17]. + */ +@UnstableApi +public final class SmtaAtomUtil { + + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef( + open = true, + value = { + NO_VALUE, + CAMCORDER_NORMAL, + CAMCORDER_SINGLE_SUPERSLOW_MOTION, + CAMCORDER_FRC_SUPERSLOW_MOTION, + CAMCORDER_SLOW_MOTION_V2, + CAMCORDER_SLOW_MOTION_V2_120, + CAMCORDER_SLOW_MOTION_V2_HEVC, + CAMCORDER_FRC_SUPERSLOW_MOTION_HEVC, + CAMCORDER_QFRC_SUPERSLOW_MOTION, + }) + private @interface RecordingMode {} + + private static final int NO_VALUE = -1; + private static final int CAMCORDER_NORMAL = 0; + private static final int CAMCORDER_SINGLE_SUPERSLOW_MOTION = 7; + private static final int CAMCORDER_FRC_SUPERSLOW_MOTION = 9; + private static final int CAMCORDER_SLOW_MOTION_V2 = 12; + private static final int CAMCORDER_SLOW_MOTION_V2_120 = 13; + private static final int CAMCORDER_SLOW_MOTION_V2_HEVC = 21; + private static final int CAMCORDER_FRC_SUPERSLOW_MOTION_HEVC = 22; + private static final int CAMCORDER_QFRC_SUPERSLOW_MOTION = 23; + + private SmtaAtomUtil() {} + + /** Parses metadata from a Samsung smta atom. */ + @Nullable + public static Metadata parseSmta(ParsableByteArray smta, int limit) { + smta.skipBytes(Atom.FULL_HEADER_SIZE); + while (smta.getPosition() < limit) { + int atomPosition = smta.getPosition(); + int atomSize = smta.readInt(); + int atomType = smta.readInt(); + if (atomType == Atom.TYPE_saut) { + // Size (4), Type (4), Author (4), Recording mode (2), SVC layer count (2). + if (atomSize < 16) { + return null; + } + smta.skipBytes(4); // Author (4) + + // Each field is stored as a key (1 byte) value (1 byte) pairs. + // The order of the fields is not guaranteed. + @RecordingMode int recordingMode = NO_VALUE; + int svcTemporalLayerCount = 0; + for (int i = 0; i < 2; i++) { + int key = smta.readUnsignedByte(); + int value = smta.readUnsignedByte(); + if (key == 0x00) { // recordingMode key + recordingMode = value; + } else if (key == 0x01) { // svcTemporalLayerCount key + svcTemporalLayerCount = value; + } + } + + int captureFrameRate = getCaptureFrameRate(recordingMode, smta, limit); + if (captureFrameRate == C.RATE_UNSET_INT) { + return null; + } + + return new Metadata(new SmtaMetadataEntry(captureFrameRate, svcTemporalLayerCount)); + } + smta.setPosition(atomPosition + atomSize); + } + return null; + } + + /** + * Returns the capture frame rate for the given recording mode, if supported. + * + *

For {@link #CAMCORDER_SLOW_MOTION_V2_HEVC}, this is done by parsing the Samsung 'srfr' atom. + * + * @return The capture frame rate value, or {@link C#RATE_UNSET_INT} if unavailable. + */ + private static int getCaptureFrameRate( + @RecordingMode int recordingMode, ParsableByteArray smta, int limit) { + // V2 and V2_120 have fixed capture frame rates. + if (recordingMode == CAMCORDER_SLOW_MOTION_V2) { + return 240; + } else if (recordingMode == CAMCORDER_SLOW_MOTION_V2_120) { + return 120; + } else if (recordingMode != CAMCORDER_SLOW_MOTION_V2_HEVC) { + return C.RATE_UNSET_INT; + } + + if (smta.bytesLeft() < Atom.HEADER_SIZE || smta.getPosition() + Atom.HEADER_SIZE > limit) { + return C.RATE_UNSET_INT; + } + + int atomSize = smta.readInt(); + int atomType = smta.readInt(); + if (atomSize < 12 || atomType != Atom.TYPE_srfr) { + return C.RATE_UNSET_INT; + } + // Capture frame rate is in Q16 format. + return smta.readUnsignedFixedPoint1616(); + } +} diff --git a/libraries/test_data/src/test/assets/media/mp4/sample_sef_slow_motion_hevc.mp4 b/libraries/test_data/src/test/assets/media/mp4/sample_sef_slow_motion_hevc.mp4 new file mode 100644 index 0000000000..999ee3d32f Binary files /dev/null and b/libraries/test_data/src/test/assets/media/mp4/sample_sef_slow_motion_hevc.mp4 differ