loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for flattening SEF files with H.265

Browse Source 
PiperOrigin-RevId: 574173120
This commit is contained in:
andrewlewis 2023-10-17 09:23:21 -07:00 committed by Copybara-Service
parent 009d48a75e
commit 680eed52f3
5 changed files with 70 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
RELEASENOTES.md
View File

@ -7,6 +7,7 @@
* Add luma and chroma bitdepth to `ColorInfo`
[#491](https://github.com/androidx/media/pull/491).
* Transformer:
* Add support for flattening H.265/HEVC SEF slow motion videos.
* Track Selection:
* Add `DefaultTrackSelector.Parameters.allowAudioNonSeamlessAdaptiveness`
to explicitly allow or disallow non-seamless adaptation. The default

3
libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java
View File

@ -55,6 +55,9 @@ public final class NalUnitUtil {
/** Access unit delimiter. */
public static final int NAL_UNIT_TYPE_AUD = 9;
/** Prefix NAL unit. */
public static final int NAL_UNIT_TYPE_PREFIX = 14;
/** Holds data parsed from a H.264 sequence parameter set NAL unit. */
public static final class SpsData {

13
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -145,6 +145,17 @@ public final class AndroidTestUtil {
.setCodecs("avc1.64000D")
.build();
public static final String MP4_ASSET_SEF_H265_URI_STRING =
"asset:///media/mp4/sample_sef_slow_motion_hevc.mp4";
public static final Format MP4_ASSET_SEF_H265_FORMAT =
new Format.Builder()
.setSampleMimeType(VIDEO_H265)
.setWidth(1920)
.setHeight(1080)
.setFrameRate(30.01679f)
.setCodecs("hvc1.1.6.L120.B0")
.build();
public static final String MP4_ASSET_BT2020_SDR = "asset:///media/mp4/bt2020-sdr.mp4";
public static final Format MP4_ASSET_BT2020_SDR_FORMAT =
new Format.Builder()
@ -809,6 +820,8 @@ public final class AndroidTestUtil {
return MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT;
case MP4_ASSET_SEF_URI_STRING:
return MP4_ASSET_SEF_FORMAT;
case MP4_ASSET_SEF_H265_URI_STRING:
return MP4_ASSET_SEF_H265_FORMAT;
case MP4_ASSET_4K60_PORTRAIT_URI_STRING:
return MP4_ASSET_4K60_PORTRAIT_FORMAT;
case MP4_REMOTE_10_SECONDS_URI_STRING:

23
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java
View File

@ -23,6 +23,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_8K24_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_8K24_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_BT2020_SDR;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_BT2020_SDR_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_SEF_H265_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_SEF_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT;
@ -250,6 +251,28 @@ public class ExportTest {
assertThat(result.exportResult.durationMs).isLessThan(950);
}
@Test
public void exportSefH265() throws Exception {
String testId = TAG + "_exportSefH265";
Context context = ApplicationProvider.getApplicationContext();
if (SDK_INT < 25) {
// TODO(b/210593256): Remove test skipping after using an in-app muxer that supports B-frames
// before API 25.
recordTestSkipped(context, testId, /* reason= */ "API version lacks muxing support");
return;
}
Transformer transformer = new Transformer.Builder(context).build();
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_SEF_H265_URI_STRING)))
.setFlattenForSlowMotion(true)
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
}
@Test
public void exportFrameRotation() throws Exception {
String testId = TAG + "_exportFrameRotation";

45
libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java
View File

@ -17,8 +17,10 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.container.NalUnitUtil.NAL_START_CODE;
import static androidx.media3.container.NalUnitUtil.NAL_UNIT_TYPE_PREFIX;
import static java.lang.Math.min;
import androidx.annotation.Nullable;
@ -37,14 +39,11 @@ import java.util.List;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Sample transformer that flattens SEF slow motion video samples.
* Sample transformer that flattens SEF slow motion videos in H.264/AVC and H.265/HEVC format using
* temporal layers.
*
* <p>Such samples follow the ITU-T Recommendation H.264 with temporal SVC.
*
* <p>This transformer leaves the samples received unchanged if the input is not an SEF slow motion
* video.
*
* <p>The mathematical formulas used in this class are explained in [Internal ref:
* <p>If the input is not an SEF slow motion video, samples will be unchanged. The mathematical
* formulas used in this class are explained in [Internal ref:
* http://go/exoplayer-sef-slomo-video-flattening].
*/
/* package */ final class SefSlowMotionFlattener {
@ -67,17 +66,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final int NAL_START_CODE_LENGTH = NAL_START_CODE.length;
/**
* The nal_unit_type corresponding to a prefix NAL unit (see ITU-T Recommendation H.264 (2016)
* table 7-1).
*/
private static final int NAL_UNIT_TYPE_PREFIX = 0x0E;
private final byte[] scratch;
/** The SEF slow motion configuration of the input. */
@Nullable private final SlowMotionData slowMotionData;
/** The MIME type of video data stored in input buffers. */
private final String mimeType;
/**
* An iterator iterating over the slow motion segments, pointing at the segment following {@code
* nextSegmentInfo}, if any.
@ -125,6 +121,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
lastSamplePresentationTimeUs = C.TIME_UNSET;
MetadataInfo metadataInfo = getMetadataInfo(format.metadata);
slowMotionData = metadataInfo.slowMotionData;
mimeType = checkNotNull(format.sampleMimeType);
if (slowMotionData != null) {
checkArgument(
mimeType.equals(MimeTypes.VIDEO_H264) || mimeType.equals(MimeTypes.VIDEO_H265),
"Unsupported MIME type for SEF slow motion video track: " + mimeType);
}
List<SlowMotionData.Segment> segments =
slowMotionData != null ? slowMotionData.segments : ImmutableList.of();
segmentIterator = segments.iterator();
@ -135,11 +137,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
segmentIterator.hasNext()
? new SegmentInfo(segmentIterator.next(), inputMaxLayer, normalSpeedMaxLayer)
: null;
if (slowMotionData != null) {
checkArgument(
MimeTypes.VIDEO_H264.equals(format.sampleMimeType),
"Unsupported MIME type for SEF slow motion video track: " + format.sampleMimeType);
}
}
/**
@ -160,13 +157,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
int originalPosition = buffer.position();
buffer.position(originalPosition + NAL_START_CODE_LENGTH);
buffer.get(scratch, 0, 4); // Read nal_unit_header_svc_extension.
buffer.get(scratch, 0, 4);
int layer;
if (mimeType.equals(MimeTypes.VIDEO_H264)) {
int nalUnitType = scratch[0] & 0x1F;
boolean svcExtensionFlag = ((scratch[1] & 0xFF) >> 7) == 1;
checkState(
nalUnitType == NAL_UNIT_TYPE_PREFIX && svcExtensionFlag,
"Missing SVC extension prefix NAL unit.");
int layer = (scratch[3] & 0xFF) >> 5;
layer = (scratch[3] & 0xFF) >> 5; // temporal_id
} else if (mimeType.equals(MimeTypes.VIDEO_H265)) {
layer = (scratch[1] & 0x07) - 1; // nuh_temporal_id_plus1
} else {
throw new IllegalStateException();
}
boolean shouldKeepFrame = processCurrentFrame(layer, bufferTimeUs);
// Update the timestamp regardless of whether the buffer is dropped as the timestamp may be
// reused for the empty end-of-stream buffer.