From 680eed52f382987089910e9f0157b811f04b0ed4 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Tue, 17 Oct 2023 09:23:21 -0700 Subject: [PATCH] Add support for flattening SEF files with H.265 PiperOrigin-RevId: 574173120 --- RELEASENOTES.md | 1 + .../media3/container/NalUnitUtil.java | 3 + .../media3/transformer/AndroidTestUtil.java | 13 +++++ .../media3/transformer/mh/ExportTest.java | 23 ++++++++ .../transformer/SefSlowMotionFlattener.java | 55 ++++++++++--------- 5 files changed, 70 insertions(+), 25 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1d9f94010e..4039af1450 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -7,6 +7,7 @@ * Add luma and chroma bitdepth to `ColorInfo` [#491](https://github.com/androidx/media/pull/491). * Transformer: + * Add support for flattening H.265/HEVC SEF slow motion videos. * Track Selection: * Add `DefaultTrackSelector.Parameters.allowAudioNonSeamlessAdaptiveness` to explicitly allow or disallow non-seamless adaptation. The default diff --git a/libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java b/libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java index 5da2bc43bf..53045c793a 100644 --- a/libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java +++ b/libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java @@ -55,6 +55,9 @@ public final class NalUnitUtil { /** Access unit delimiter. */ public static final int NAL_UNIT_TYPE_AUD = 9; + /** Prefix NAL unit. */ + public static final int NAL_UNIT_TYPE_PREFIX = 14; + /** Holds data parsed from a H.264 sequence parameter set NAL unit. */ public static final class SpsData { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 5680861935..53cd537dc0 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -145,6 +145,17 @@ public final class AndroidTestUtil { .setCodecs("avc1.64000D") .build(); + public static final String MP4_ASSET_SEF_H265_URI_STRING = + "asset:///media/mp4/sample_sef_slow_motion_hevc.mp4"; + public static final Format MP4_ASSET_SEF_H265_FORMAT = + new Format.Builder() + .setSampleMimeType(VIDEO_H265) + .setWidth(1920) + .setHeight(1080) + .setFrameRate(30.01679f) + .setCodecs("hvc1.1.6.L120.B0") + .build(); + public static final String MP4_ASSET_BT2020_SDR = "asset:///media/mp4/bt2020-sdr.mp4"; public static final Format MP4_ASSET_BT2020_SDR_FORMAT = new Format.Builder() @@ -809,6 +820,8 @@ public final class AndroidTestUtil { return MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT; case MP4_ASSET_SEF_URI_STRING: return MP4_ASSET_SEF_FORMAT; + case MP4_ASSET_SEF_H265_URI_STRING: + return MP4_ASSET_SEF_H265_FORMAT; case MP4_ASSET_4K60_PORTRAIT_URI_STRING: return MP4_ASSET_4K60_PORTRAIT_FORMAT; case MP4_REMOTE_10_SECONDS_URI_STRING: diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java index c579c3f86b..c097a10fd7 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java @@ -23,6 +23,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_8K24_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_8K24_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_BT2020_SDR; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_BT2020_SDR_FORMAT; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_SEF_H265_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_SEF_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT; @@ -250,6 +251,28 @@ public class ExportTest { assertThat(result.exportResult.durationMs).isLessThan(950); } + @Test + public void exportSefH265() throws Exception { + String testId = TAG + "_exportSefH265"; + Context context = ApplicationProvider.getApplicationContext(); + + if (SDK_INT < 25) { + // TODO(b/210593256): Remove test skipping after using an in-app muxer that supports B-frames + // before API 25. + recordTestSkipped(context, testId, /* reason= */ "API version lacks muxing support"); + return; + } + + Transformer transformer = new Transformer.Builder(context).build(); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_SEF_H265_URI_STRING))) + .setFlattenForSlowMotion(true) + .build(); + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + } + @Test public void exportFrameRotation() throws Exception { String testId = TAG + "_exportFrameRotation"; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java index 6c33a03589..0a5150944f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java @@ -17,8 +17,10 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.container.NalUnitUtil.NAL_START_CODE; +import static androidx.media3.container.NalUnitUtil.NAL_UNIT_TYPE_PREFIX; import static java.lang.Math.min; import androidx.annotation.Nullable; @@ -37,14 +39,11 @@ import java.util.List; import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** - * Sample transformer that flattens SEF slow motion video samples. + * Sample transformer that flattens SEF slow motion videos in H.264/AVC and H.265/HEVC format using + * temporal layers. * - *

Such samples follow the ITU-T Recommendation H.264 with temporal SVC. - * - *

This transformer leaves the samples received unchanged if the input is not an SEF slow motion - * video. - * - *

The mathematical formulas used in this class are explained in [Internal ref: + *

If the input is not an SEF slow motion video, samples will be unchanged. The mathematical + * formulas used in this class are explained in [Internal ref: * http://go/exoplayer-sef-slomo-video-flattening]. */ /* package */ final class SefSlowMotionFlattener { @@ -67,17 +66,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private static final int NAL_START_CODE_LENGTH = NAL_START_CODE.length; - /** - * The nal_unit_type corresponding to a prefix NAL unit (see ITU-T Recommendation H.264 (2016) - * table 7-1). - */ - private static final int NAL_UNIT_TYPE_PREFIX = 0x0E; - private final byte[] scratch; /** The SEF slow motion configuration of the input. */ @Nullable private final SlowMotionData slowMotionData; + /** The MIME type of video data stored in input buffers. */ + private final String mimeType; + /** * An iterator iterating over the slow motion segments, pointing at the segment following {@code * nextSegmentInfo}, if any. @@ -125,6 +121,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; lastSamplePresentationTimeUs = C.TIME_UNSET; MetadataInfo metadataInfo = getMetadataInfo(format.metadata); slowMotionData = metadataInfo.slowMotionData; + mimeType = checkNotNull(format.sampleMimeType); + if (slowMotionData != null) { + checkArgument( + mimeType.equals(MimeTypes.VIDEO_H264) || mimeType.equals(MimeTypes.VIDEO_H265), + "Unsupported MIME type for SEF slow motion video track: " + mimeType); + } List segments = slowMotionData != null ? slowMotionData.segments : ImmutableList.of(); segmentIterator = segments.iterator(); @@ -135,11 +137,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; segmentIterator.hasNext() ? new SegmentInfo(segmentIterator.next(), inputMaxLayer, normalSpeedMaxLayer) : null; - if (slowMotionData != null) { - checkArgument( - MimeTypes.VIDEO_H264.equals(format.sampleMimeType), - "Unsupported MIME type for SEF slow motion video track: " + format.sampleMimeType); - } } /** @@ -160,13 +157,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; int originalPosition = buffer.position(); buffer.position(originalPosition + NAL_START_CODE_LENGTH); - buffer.get(scratch, 0, 4); // Read nal_unit_header_svc_extension. - int nalUnitType = scratch[0] & 0x1F; - boolean svcExtensionFlag = ((scratch[1] & 0xFF) >> 7) == 1; - checkState( - nalUnitType == NAL_UNIT_TYPE_PREFIX && svcExtensionFlag, - "Missing SVC extension prefix NAL unit."); - int layer = (scratch[3] & 0xFF) >> 5; + buffer.get(scratch, 0, 4); + int layer; + if (mimeType.equals(MimeTypes.VIDEO_H264)) { + int nalUnitType = scratch[0] & 0x1F; + boolean svcExtensionFlag = ((scratch[1] & 0xFF) >> 7) == 1; + checkState( + nalUnitType == NAL_UNIT_TYPE_PREFIX && svcExtensionFlag, + "Missing SVC extension prefix NAL unit."); + layer = (scratch[3] & 0xFF) >> 5; // temporal_id + } else if (mimeType.equals(MimeTypes.VIDEO_H265)) { + layer = (scratch[1] & 0x07) - 1; // nuh_temporal_id_plus1 + } else { + throw new IllegalStateException(); + } + boolean shouldKeepFrame = processCurrentFrame(layer, bufferTimeUs); // Update the timestamp regardless of whether the buffer is dropped as the timestamp may be // reused for the empty end-of-stream buffer.