From 5fa9985ce6800bef2f03e9b324692f0ff6b01b4b Mon Sep 17 00:00:00 2001 From: ibaker Date: Fri, 12 Jul 2024 07:59:30 -0700 Subject: [PATCH] Add `H264_` prefix to `NalUnitUtil.NAL_UNIT_TYPE_*` constants Also promote all H.265 constants to be public in `NalUnitUtil` with `H265_` prefixes, for consistency. A lot of these names are used in h.265 too (and `NalUnitUtil` handles both), but with different values, so this rename aims to avoid accidentally using an h.264 value in an h.265 context. PiperOrigin-RevId: 651774188 --- .../media3/container/NalUnitUtil.java | 106 ++++++++++++++---- .../media3/extractor/ts/H264Reader.java | 22 ++-- .../media3/extractor/ts/H265Reader.java | 37 +++--- .../media3/transformer/MuxerWrapper.java | 2 +- .../transformer/SefSlowMotionFlattener.java | 4 +- 5 files changed, 111 insertions(+), 60 deletions(-) diff --git a/libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java b/libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java index 3783a63b51..080505e219 100644 --- a/libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java +++ b/libraries/container/src/main/java/androidx/media3/container/NalUnitUtil.java @@ -40,39 +40,97 @@ public final class NalUnitUtil { private static final String TAG = "NalUnitUtil"; - /** Coded slice of a non-IDR picture. */ - public static final int NAL_UNIT_TYPE_NON_IDR = 1; + /** H.264 coded slice of a non-IDR picture. */ + public static final int H264_NAL_UNIT_TYPE_NON_IDR = 1; - /** Coded slice data partition A. */ - public static final int NAL_UNIT_TYPE_PARTITION_A = 2; + /** + * @deprecated Use {@link #H264_NAL_UNIT_TYPE_NON_IDR} + */ + @Deprecated public static final int NAL_UNIT_TYPE_NON_IDR = H264_NAL_UNIT_TYPE_NON_IDR; - /** Coded slice of an IDR picture. */ - public static final int NAL_UNIT_TYPE_IDR = 5; + /** H.264 coded slice data partition A. */ + public static final int H264_NAL_UNIT_TYPE_PARTITION_A = 2; - /** Supplemental enhancement information. */ - public static final int NAL_UNIT_TYPE_SEI = 6; + /** + * @deprecated Use {@link #H264_NAL_UNIT_TYPE_PARTITION_A} + */ + @Deprecated public static final int NAL_UNIT_TYPE_PARTITION_A = H264_NAL_UNIT_TYPE_PARTITION_A; - /** Sequence parameter set. */ - public static final int NAL_UNIT_TYPE_SPS = 7; + /** H.264 coded slice of an IDR picture. */ + public static final int H264_NAL_UNIT_TYPE_IDR = 5; - /** Picture parameter set. */ - public static final int NAL_UNIT_TYPE_PPS = 8; + /** + * @deprecated Use {@link #H264_NAL_UNIT_TYPE_IDR} + */ + @Deprecated public static final int NAL_UNIT_TYPE_IDR = H264_NAL_UNIT_TYPE_IDR; - /** Access unit delimiter. */ - public static final int NAL_UNIT_TYPE_AUD = 9; + /** H.264 supplemental enhancement information. */ + public static final int H264_NAL_UNIT_TYPE_SEI = 6; - /** Prefix NAL unit. */ - public static final int NAL_UNIT_TYPE_PREFIX = 14; + /** + * @deprecated Use {@link #H264_NAL_UNIT_TYPE_SEI} + */ + @Deprecated public static final int NAL_UNIT_TYPE_SEI = H264_NAL_UNIT_TYPE_SEI; - /** H265 video parameter set. */ + /** H.264 sequence parameter set. */ + public static final int H264_NAL_UNIT_TYPE_SPS = 7; + + /** + * @deprecated Use {@link #H264_NAL_UNIT_TYPE_SPS} + */ + @Deprecated public static final int NAL_UNIT_TYPE_SPS = H264_NAL_UNIT_TYPE_SPS; + + /** H.264 picture parameter set. */ + public static final int H264_NAL_UNIT_TYPE_PPS = 8; + + /** + * @deprecated Use {@link #H264_NAL_UNIT_TYPE_PPS} + */ + @Deprecated public static final int NAL_UNIT_TYPE_PPS = H264_NAL_UNIT_TYPE_PPS; + + /** H.264 access unit delimiter. */ + public static final int H264_NAL_UNIT_TYPE_AUD = 9; + + /** + * @deprecated Use {@link #H264_NAL_UNIT_TYPE_AUD} + */ + @Deprecated public static final int NAL_UNIT_TYPE_AUD = H264_NAL_UNIT_TYPE_AUD; + + /** H.264 prefix NAL unit. */ + public static final int H264_NAL_UNIT_TYPE_PREFIX = 14; + + /** + * @deprecated Use {@link #H264_NAL_UNIT_TYPE_PREFIX} + */ + @Deprecated public static final int NAL_UNIT_TYPE_PREFIX = H264_NAL_UNIT_TYPE_PREFIX; + + /** H.265 coded slice segment of a random access skipped leading picture (RASL_R). */ + public static final int H265_NAL_UNIT_TYPE_RASL_R = 9; + + /** H.265 coded slice segment of a broken link access picture (BLA_W_LP). */ + public static final int H265_NAL_UNIT_TYPE_BLA_W_LP = 16; + + /** H.265 coded slice segment of a clean random access picture (CRA_NUT). */ + public static final int H265_NAL_UNIT_TYPE_CRA = 21; + + /** H.265 video parameter set (VPS_NUT). */ public static final int H265_NAL_UNIT_TYPE_VPS = 32; - /** H265 sequence parameter set. */ + /** H.265 sequence parameter set (SPS_NUT). */ public static final int H265_NAL_UNIT_TYPE_SPS = 33; - /** H265 SEI. */ + /** H.265 picture parameter set (PPS_NUT). */ + public static final int H265_NAL_UNIT_TYPE_PPS = 34; + + /** H.265 access unit delimiter (AUD_NUT). */ + public static final int H265_NAL_UNIT_TYPE_AUD = 35; + + /** H.265 prefixed supplemental enhancement information (PREFIX_SEI_NUT). */ public static final int H265_NAL_UNIT_TYPE_PREFIX_SEI = 39; + /** H.265 suffixed supplemental enhancement information (SUFFIX_SEI_NUT). */ + public static final int H265_NAL_UNIT_TYPE_SUFFIX_SEI = 40; + /** Holds data parsed from a H.264 sequence parameter set NAL unit. */ public static final class SpsData { @@ -519,7 +577,7 @@ public final class NalUnitUtil { while (offset + 1 < length) { int value = data.get(offset) & 0xFF; if (consecutiveZeros == 3) { - if (value == 1 && (data.get(offset + 1) & 0x1F) == NAL_UNIT_TYPE_SPS) { + if (value == 1 && (data.get(offset + 1) & 0x1F) == H264_NAL_UNIT_TYPE_SPS) { // Copy from this NAL unit onwards to the start of the buffer. ByteBuffer offsetData = data.duplicate(); offsetData.position(offset - 3); @@ -551,7 +609,7 @@ public final class NalUnitUtil { */ public static boolean isNalUnitSei(@Nullable String mimeType, byte nalUnitHeaderFirstByte) { return (MimeTypes.VIDEO_H264.equals(mimeType) - && (nalUnitHeaderFirstByte & 0x1F) == NAL_UNIT_TYPE_SEI) + && (nalUnitHeaderFirstByte & 0x1F) == H264_NAL_UNIT_TYPE_SEI) || (MimeTypes.VIDEO_H265.equals(mimeType) && ((nalUnitHeaderFirstByte & 0x7E) >> 1) == H265_NAL_UNIT_TYPE_PREFIX_SEI); } @@ -584,16 +642,16 @@ public final class NalUnitUtil { } int nalUnitType = nalUnitHeaderFirstByte & 0x1F; - if (nalUnitType == NAL_UNIT_TYPE_NON_IDR) { + if (nalUnitType == H264_NAL_UNIT_TYPE_NON_IDR) { // For pictures (Video Coding Layer NAL units), we can rely on nal_ref_idc to determine // whether future NAL units depend on it. return false; } - if (nalUnitType == NAL_UNIT_TYPE_AUD) { + if (nalUnitType == H264_NAL_UNIT_TYPE_AUD) { // NAL unit delimiters are not depended on. return false; } - if (nalUnitType == NAL_UNIT_TYPE_PREFIX) { + if (nalUnitType == H264_NAL_UNIT_TYPE_PREFIX) { // Prefix NAL units are only used by Annex G scalable video coding to mark temporal layers. // Rely on nal_ref_idc to identify sample dependencies. return false; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H264Reader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H264Reader.java index b00aad6105..2d14b0bf4a 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H264Reader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H264Reader.java @@ -82,9 +82,9 @@ public final class H264Reader implements ElementaryStreamReader { this.allowNonIdrKeyframes = allowNonIdrKeyframes; this.detectAccessUnits = detectAccessUnits; prefixFlags = new boolean[3]; - sps = new NalUnitTargetBuffer(NalUnitUtil.NAL_UNIT_TYPE_SPS, 128); - pps = new NalUnitTargetBuffer(NalUnitUtil.NAL_UNIT_TYPE_PPS, 128); - sei = new NalUnitTargetBuffer(NalUnitUtil.NAL_UNIT_TYPE_SEI, 128); + sps = new NalUnitTargetBuffer(NalUnitUtil.H264_NAL_UNIT_TYPE_SPS, 128); + pps = new NalUnitTargetBuffer(NalUnitUtil.H264_NAL_UNIT_TYPE_PPS, 128); + sei = new NalUnitTargetBuffer(NalUnitUtil.H264_NAL_UNIT_TYPE_SEI, 128); pesTimeUs = C.TIME_UNSET; seiWrapper = new ParsableByteArray(); } @@ -338,11 +338,11 @@ public final class H264Reader implements ElementaryStreamReader { nalUnitTimeUs = pesTimeUs; nalUnitStartPosition = position; this.randomAccessIndicator = randomAccessIndicator; - if ((allowNonIdrKeyframes && nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_NON_IDR) + if ((allowNonIdrKeyframes && nalUnitType == NalUnitUtil.H264_NAL_UNIT_TYPE_NON_IDR) || (detectAccessUnits - && (nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_IDR - || nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_NON_IDR - || nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_PARTITION_A))) { + && (nalUnitType == NalUnitUtil.H264_NAL_UNIT_TYPE_IDR + || nalUnitType == NalUnitUtil.H264_NAL_UNIT_TYPE_NON_IDR + || nalUnitType == NalUnitUtil.H264_NAL_UNIT_TYPE_PARTITION_A))) { // Store the previous header and prepare to populate the new one. SliceHeaderData newSliceHeader = previousSliceHeader; previousSliceHeader = sliceHeader; @@ -432,7 +432,7 @@ public final class H264Reader implements ElementaryStreamReader { bottomFieldFlagPresent = true; } } - boolean idrPicFlag = nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_IDR; + boolean idrPicFlag = nalUnitType == NalUnitUtil.H264_NAL_UNIT_TYPE_IDR; int idrPicId = 0; if (idrPicFlag) { if (!bitArray.canReadExpGolombCodedNum()) { @@ -486,7 +486,7 @@ public final class H264Reader implements ElementaryStreamReader { } public boolean endNalUnit(long position, int offset, boolean hasOutputFormat) { - if (nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_AUD + if (nalUnitType == NalUnitUtil.H264_NAL_UNIT_TYPE_AUD || (detectAccessUnits && sliceHeader.isFirstVclNalUnitOfPicture(previousSliceHeader))) { // If the NAL unit ending is the start of a new sample, output the previous one. if (hasOutputFormat && readingSample) { @@ -514,8 +514,8 @@ public final class H264Reader implements ElementaryStreamReader { boolean treatIFrameAsKeyframe = allowNonIdrKeyframes ? sliceHeader.isISlice() : randomAccessIndicator; sampleIsKeyframe |= - nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_IDR - || (treatIFrameAsKeyframe && nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_NON_IDR); + nalUnitType == NalUnitUtil.H264_NAL_UNIT_TYPE_IDR + || (treatIFrameAsKeyframe && nalUnitType == NalUnitUtil.H264_NAL_UNIT_TYPE_NON_IDR); } private void outputSample(int offset) { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java index 41f2364313..3b6ba333f7 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java @@ -40,19 +40,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @UnstableApi public final class H265Reader implements ElementaryStreamReader { - private static final String TAG = "H265Reader"; - - // nal_unit_type values from H.265/HEVC (2014) Table 7-1. - private static final int RASL_R = 9; - private static final int BLA_W_LP = 16; - private static final int CRA_NUT = 21; - private static final int VPS_NUT = 32; - private static final int SPS_NUT = 33; - private static final int PPS_NUT = 34; - private static final int AUD_NUT = 35; - private static final int PREFIX_SEI_NUT = 39; - private static final int SUFFIX_SEI_NUT = 40; - private final SeiReader seiReader; private @MonotonicNonNull String formatId; @@ -83,11 +70,11 @@ public final class H265Reader implements ElementaryStreamReader { public H265Reader(SeiReader seiReader) { this.seiReader = seiReader; prefixFlags = new boolean[3]; - vps = new NalUnitTargetBuffer(VPS_NUT, 128); - sps = new NalUnitTargetBuffer(SPS_NUT, 128); - pps = new NalUnitTargetBuffer(PPS_NUT, 128); - prefixSei = new NalUnitTargetBuffer(PREFIX_SEI_NUT, 128); - suffixSei = new NalUnitTargetBuffer(SUFFIX_SEI_NUT, 128); + vps = new NalUnitTargetBuffer(NalUnitUtil.H265_NAL_UNIT_TYPE_VPS, 128); + sps = new NalUnitTargetBuffer(NalUnitUtil.H265_NAL_UNIT_TYPE_SPS, 128); + pps = new NalUnitTargetBuffer(NalUnitUtil.H265_NAL_UNIT_TYPE_PPS, 128); + prefixSei = new NalUnitTargetBuffer(NalUnitUtil.H265_NAL_UNIT_TYPE_PREFIX_SEI, 128); + suffixSei = new NalUnitTargetBuffer(NalUnitUtil.H265_NAL_UNIT_TYPE_SUFFIX_SEI, 128); pesTimeUs = C.TIME_UNSET; seiWrapper = new ParsableByteArray(); } @@ -353,8 +340,11 @@ public final class H265Reader implements ElementaryStreamReader { } // Look for the first slice flag if this NAL unit contains a slice_segment_layer_rbsp. - nalUnitHasKeyframeData = (nalUnitType >= BLA_W_LP && nalUnitType <= CRA_NUT); - lookingForFirstSliceFlag = nalUnitHasKeyframeData || nalUnitType <= RASL_R; + nalUnitHasKeyframeData = + (nalUnitType >= NalUnitUtil.H265_NAL_UNIT_TYPE_BLA_W_LP + && nalUnitType <= NalUnitUtil.H265_NAL_UNIT_TYPE_CRA); + lookingForFirstSliceFlag = + nalUnitHasKeyframeData || nalUnitType <= NalUnitUtil.H265_NAL_UNIT_TYPE_RASL_R; } public void readNalUnitData(byte[] data, int offset, int limit) { @@ -410,12 +400,15 @@ public final class H265Reader implements ElementaryStreamReader { /** Returns whether a NAL unit type is one that occurs before any VCL NAL units in a sample. */ private static boolean isPrefixNalUnit(int nalUnitType) { - return (VPS_NUT <= nalUnitType && nalUnitType <= AUD_NUT) || nalUnitType == PREFIX_SEI_NUT; + return (NalUnitUtil.H265_NAL_UNIT_TYPE_VPS <= nalUnitType + && nalUnitType <= NalUnitUtil.H265_NAL_UNIT_TYPE_AUD) + || nalUnitType == NalUnitUtil.H265_NAL_UNIT_TYPE_PREFIX_SEI; } /** Returns whether a NAL unit type is one that occurs in the VLC body of a sample. */ private static boolean isVclBodyNalUnit(int nalUnitType) { - return nalUnitType < VPS_NUT || nalUnitType == SUFFIX_SEI_NUT; + return nalUnitType < NalUnitUtil.H265_NAL_UNIT_TYPE_VPS + || nalUnitType == NalUnitUtil.H265_NAL_UNIT_TYPE_SUFFIX_SEI; } } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java index 0ce530ff7e..465d1e634a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java @@ -274,7 +274,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } int nalUnitTypeMask = 0x1F; if ((newSps[NalUnitUtil.NAL_START_CODE.length] & nalUnitTypeMask) - != NalUnitUtil.NAL_UNIT_TYPE_SPS) { + != NalUnitUtil.H264_NAL_UNIT_TYPE_SPS) { return null; } // Check that H.264 profile is non-zero. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java index 7cea73a3cf..019eae4253 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SefSlowMotionFlattener.java @@ -19,8 +19,8 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.container.NalUnitUtil.H264_NAL_UNIT_TYPE_PREFIX; import static androidx.media3.container.NalUnitUtil.NAL_START_CODE; -import static androidx.media3.container.NalUnitUtil.NAL_UNIT_TYPE_PREFIX; import static java.lang.Math.min; import androidx.annotation.Nullable; @@ -162,7 +162,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; int nalUnitType = scratch[0] & 0x1F; boolean svcExtensionFlag = ((scratch[1] & 0xFF) >> 7) == 1; checkState( - nalUnitType == NAL_UNIT_TYPE_PREFIX && svcExtensionFlag, + nalUnitType == H264_NAL_UNIT_TYPE_PREFIX && svcExtensionFlag, "Missing SVC extension prefix NAL unit."); layer = (scratch[3] & 0xFF) >> 5; // temporal_id } else if (mimeType.equals(MimeTypes.VIDEO_H265)) {