Add support for flattening SEF files with H.265

PiperOrigin-RevId: 574173120

RELEASENOTES.md

@@ -7,6 +7,7 @@
     *   Add luma and chroma bitdepth to `ColorInfo`
         [#491](https://github.com/androidx/media/pull/491).
 *   Transformer:
+    *   Add support for flattening H.265/HEVC SEF slow motion videos.
 *   Track Selection:
     *   Add `DefaultTrackSelector.Parameters.allowAudioNonSeamlessAdaptiveness`
         to explicitly allow or disallow non-seamless adaptation. The default

libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java

@@ -55,6 +55,9 @@ public final class NalUnitUtil {
   /** Access unit delimiter. */
   public static final int NAL_UNIT_TYPE_AUD = 9;
 
+  /** Prefix NAL unit. */
+  public static final int NAL_UNIT_TYPE_PREFIX = 14;
+
   /** Holds data parsed from a H.264 sequence parameter set NAL unit. */
   public static final class SpsData {
 

libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java

@@ -145,6 +145,17 @@ public final class AndroidTestUtil {
           .setCodecs("avc1.64000D")
           .build();
 
+  public static final String MP4_ASSET_SEF_H265_URI_STRING =
+      "asset:///media/mp4/sample_sef_slow_motion_hevc.mp4";
+  public static final Format MP4_ASSET_SEF_H265_FORMAT =
+      new Format.Builder()
+          .setSampleMimeType(VIDEO_H265)
+          .setWidth(1920)
+          .setHeight(1080)
+          .setFrameRate(30.01679f)
+          .setCodecs("hvc1.1.6.L120.B0")
+          .build();
+
   public static final String MP4_ASSET_BT2020_SDR = "asset:///media/mp4/bt2020-sdr.mp4";
   public static final Format MP4_ASSET_BT2020_SDR_FORMAT =
       new Format.Builder()
@@ -809,6 +820,8 @@ public final class AndroidTestUtil {
         return MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT;
       case MP4_ASSET_SEF_URI_STRING:
         return MP4_ASSET_SEF_FORMAT;
+      case MP4_ASSET_SEF_H265_URI_STRING:
+        return MP4_ASSET_SEF_H265_FORMAT;
       case MP4_ASSET_4K60_PORTRAIT_URI_STRING:
         return MP4_ASSET_4K60_PORTRAIT_FORMAT;
       case MP4_REMOTE_10_SECONDS_URI_STRING:

libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java

@@ -23,6 +23,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_8K24_FORMAT;
 import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_8K24_URI_STRING;
 import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_BT2020_SDR;
 import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_BT2020_SDR_FORMAT;
+import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_SEF_H265_URI_STRING;
 import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_SEF_URI_STRING;
 import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
 import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT;
@@ -250,6 +251,28 @@ public class ExportTest {
     assertThat(result.exportResult.durationMs).isLessThan(950);
   }
 
+  @Test
+  public void exportSefH265() throws Exception {
+    String testId = TAG + "_exportSefH265";
+    Context context = ApplicationProvider.getApplicationContext();
+
+    if (SDK_INT < 25) {
+      // TODO(b/210593256): Remove test skipping after using an in-app muxer that supports B-frames
+      //  before API 25.
+      recordTestSkipped(context, testId, /* reason= */ "API version lacks muxing support");
+      return;
+    }
+
+    Transformer transformer = new Transformer.Builder(context).build();
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_SEF_H265_URI_STRING)))
+            .setFlattenForSlowMotion(true)
+            .build();
+    new TransformerAndroidTestRunner.Builder(context, transformer)
+        .build()
+        .run(testId, editedMediaItem);
+  }
+
   @Test
   public void exportFrameRotation() throws Exception {
     String testId = TAG + "_exportFrameRotation";

libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java

@@ -17,8 +17,10 @@
 package androidx.media3.transformer;
 
 import static androidx.media3.common.util.Assertions.checkArgument;
+import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.container.NalUnitUtil.NAL_START_CODE;
+import static androidx.media3.container.NalUnitUtil.NAL_UNIT_TYPE_PREFIX;
 import static java.lang.Math.min;
 
 import androidx.annotation.Nullable;
@@ -37,14 +39,11 @@ import java.util.List;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
 /**
- * Sample transformer that flattens SEF slow motion video samples.
+ * Sample transformer that flattens SEF slow motion videos in H.264/AVC and H.265/HEVC format using
+ * temporal layers.
  *
- * <p>Such samples follow the ITU-T Recommendation H.264 with temporal SVC.
+ * <p>If the input is not an SEF slow motion video, samples will be unchanged. The mathematical
- *
+ * formulas used in this class are explained in [Internal ref:
- * <p>This transformer leaves the samples received unchanged if the input is not an SEF slow motion
- * video.
- *
- * <p>The mathematical formulas used in this class are explained in [Internal ref:
  * http://go/exoplayer-sef-slomo-video-flattening].
  */
 /* package */ final class SefSlowMotionFlattener {
@@ -67,17 +66,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
   private static final int NAL_START_CODE_LENGTH = NAL_START_CODE.length;
 
-  /**
-   * The nal_unit_type corresponding to a prefix NAL unit (see ITU-T Recommendation H.264 (2016)
-   * table 7-1).
-   */
-  private static final int NAL_UNIT_TYPE_PREFIX = 0x0E;
-
   private final byte[] scratch;
 
   /** The SEF slow motion configuration of the input. */
   @Nullable private final SlowMotionData slowMotionData;
 
+  /** The MIME type of video data stored in input buffers. */
+  private final String mimeType;
+
   /**
    * An iterator iterating over the slow motion segments, pointing at the segment following {@code
    * nextSegmentInfo}, if any.
@@ -125,6 +121,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     lastSamplePresentationTimeUs = C.TIME_UNSET;
     MetadataInfo metadataInfo = getMetadataInfo(format.metadata);
     slowMotionData = metadataInfo.slowMotionData;
+    mimeType = checkNotNull(format.sampleMimeType);
+    if (slowMotionData != null) {
+      checkArgument(
+          mimeType.equals(MimeTypes.VIDEO_H264) || mimeType.equals(MimeTypes.VIDEO_H265),
+          "Unsupported MIME type for SEF slow motion video track: " + mimeType);
+    }
     List<SlowMotionData.Segment> segments =
         slowMotionData != null ? slowMotionData.segments : ImmutableList.of();
     segmentIterator = segments.iterator();
@@ -135,11 +137,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
         segmentIterator.hasNext()
             ? new SegmentInfo(segmentIterator.next(), inputMaxLayer, normalSpeedMaxLayer)
             : null;
-    if (slowMotionData != null) {
-      checkArgument(
-          MimeTypes.VIDEO_H264.equals(format.sampleMimeType),
-          "Unsupported MIME type for SEF slow motion video track: " + format.sampleMimeType);
-    }
   }
 
   /**
@@ -160,13 +157,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
     int originalPosition = buffer.position();
     buffer.position(originalPosition + NAL_START_CODE_LENGTH);
-    buffer.get(scratch, 0, 4); // Read nal_unit_header_svc_extension.
+    buffer.get(scratch, 0, 4);
-    int nalUnitType = scratch[0] & 0x1F;
+    int layer;
-    boolean svcExtensionFlag = ((scratch[1] & 0xFF) >> 7) == 1;
+    if (mimeType.equals(MimeTypes.VIDEO_H264)) {
-    checkState(
+      int nalUnitType = scratch[0] & 0x1F;
-        nalUnitType == NAL_UNIT_TYPE_PREFIX && svcExtensionFlag,
+      boolean svcExtensionFlag = ((scratch[1] & 0xFF) >> 7) == 1;
-        "Missing SVC extension prefix NAL unit.");
+      checkState(
-    int layer = (scratch[3] & 0xFF) >> 5;
+          nalUnitType == NAL_UNIT_TYPE_PREFIX && svcExtensionFlag,
+          "Missing SVC extension prefix NAL unit.");
+      layer = (scratch[3] & 0xFF) >> 5; // temporal_id
+    } else if (mimeType.equals(MimeTypes.VIDEO_H265)) {
+      layer = (scratch[1] & 0x07) - 1; // nuh_temporal_id_plus1
+    } else {
+      throw new IllegalStateException();
+    }
+
     boolean shouldKeepFrame = processCurrentFrame(layer, bufferTimeUs);
     // Update the timestamp regardless of whether the buffer is dropped as the timestamp may be
     // reused for the empty end-of-stream buffer.