From 296074fbea3992a7e3c3a5e4897dcbfabd58756b Mon Sep 17 00:00:00 2001 From: Marcus Wichelmann Date: Mon, 13 Sep 2021 10:27:55 +0200 Subject: [PATCH 1/4] Extend SPS parsing when building the initial MP4 HevcConfig and include the PAR for propagating it into the Format --- .../util/CodecSpecificDataUtil.java | 25 +- .../android/exoplayer2/util/NalUnitUtil.java | 267 +++++++++++++++++- .../exoplayer2/util/NalUnitUtilTest.java | 7 +- .../exoplayer2/extractor/mp4/AtomParsers.java | 3 + .../exoplayer2/extractor/ts/H264Reader.java | 8 +- .../exoplayer2/extractor/ts/H265Reader.java | 22 +- .../android/exoplayer2/video/AvcConfig.java | 5 +- .../android/exoplayer2/video/HevcConfig.java | 38 ++- .../source/rtsp/RtspMediaTrack.java | 4 +- 9 files changed, 322 insertions(+), 57 deletions(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/CodecSpecificDataUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/CodecSpecificDataUtil.java index 0f8edb4acd..8d310bd361 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/CodecSpecificDataUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/CodecSpecificDataUtil.java @@ -86,28 +86,11 @@ public final class CodecSpecificDataUtil { } /** - * Returns an RFC 6381 HEVC codec string based on the SPS NAL unit read from the provided bit - * array. The position of the bit array must be the start of an SPS NALU (nal_unit_header), and - * the position may be modified by this method. + * Builds a RFC 6381 HEVC codec string using the provided parameters. */ - public static String buildHevcCodecStringFromSps(ParsableNalUnitBitArray bitArray) { - // Skip nal_unit_header, sps_video_parameter_set_id, sps_max_sub_layers_minus1 and - // sps_temporal_id_nesting_flag. - bitArray.skipBits(16 + 4 + 3 + 1); - int generalProfileSpace = bitArray.readBits(2); - boolean generalTierFlag = bitArray.readBit(); - int generalProfileIdc = bitArray.readBits(5); - int generalProfileCompatibilityFlags = 0; - for (int i = 0; i < 32; i++) { - if (bitArray.readBit()) { - generalProfileCompatibilityFlags |= (1 << i); - } - } - int[] constraintBytes = new int[6]; - for (int i = 0; i < constraintBytes.length; ++i) { - constraintBytes[i] = bitArray.readBits(8); - } - int generalLevelIdc = bitArray.readBits(8); + public static String buildHevcCodecString( + int generalProfileSpace, boolean generalTierFlag, int generalProfileIdc, + int generalProfileCompatibilityFlags, int[] constraintBytes, int generalLevelIdc) { StringBuilder builder = new StringBuilder( Util.formatInvariant( diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java index 1fd44011e0..4b267e0bfa 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java @@ -15,6 +15,8 @@ */ package com.google.android.exoplayer2.util; +import static java.lang.Math.min; + import androidx.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; @@ -24,7 +26,7 @@ public final class NalUnitUtil { private static final String TAG = "NalUnitUtil"; - /** Holds data parsed from a sequence parameter set NAL unit. */ + /** Holds data parsed from a H.264 sequence parameter set NAL unit. */ public static final class SpsData { public final int profileIdc; @@ -71,6 +73,44 @@ public final class NalUnitUtil { } } + /** Holds data parsed from a H.265 sequence parameter set NAL unit. */ + public static final class H265SpsData { + + public final int generalProfileSpace; + public final boolean generalTierFlag; + public final int generalProfileIdc; + public final int generalProfileCompatibilityFlags; + public final int[] constraintBytes; + public final int generalLevelIdc; + public final int seqParameterSetId; + public final int picWidthInLumaSamples; + public final int picHeightInLumaSamples; + public final float pixelWidthHeightRatio; + + public H265SpsData( + int generalProfileSpace, + boolean generalTierFlag, + int generalProfileIdc, + int generalProfileCompatibilityFlags, + int[] constraintBytes, + int generalLevelIdc, + int seqParameterSetId, + int picWidthInLumaSamples, + int picHeightInLumaSamples, + float pixelWidthHeightRatio) { + this.generalProfileSpace = generalProfileSpace; + this.generalTierFlag = generalTierFlag; + this.generalProfileIdc = generalProfileIdc; + this.generalProfileCompatibilityFlags = generalProfileCompatibilityFlags; + this.constraintBytes = constraintBytes; + this.generalLevelIdc = generalLevelIdc; + this.seqParameterSetId = seqParameterSetId; + this.picWidthInLumaSamples = picWidthInLumaSamples; + this.picHeightInLumaSamples = picHeightInLumaSamples; + this.pixelWidthHeightRatio = pixelWidthHeightRatio; + } + } + /** Holds data parsed from a picture parameter set NAL unit. */ public static final class PpsData { @@ -252,17 +292,16 @@ public final class NalUnitUtil { } /** - * Parses an SPS NAL unit using the syntax defined in ITU-T Recommendation H.264 (2013) subsection - * 7.3.2.1.1. + * Parses a SPS NAL unit payload using the syntax defined in ITU-T Recommendation H.264 (2013) + * subsection 7.3.2.1.1. * * @param nalData A buffer containing escaped SPS data. - * @param nalOffset The offset of the NAL unit header in {@code nalData}. + * @param nalOffset The offset of the NAL unit in {@code nalData}. * @param nalLimit The limit of the NAL unit in {@code nalData}. * @return A parsed representation of the SPS data. */ - public static SpsData parseSpsNalUnit(byte[] nalData, int nalOffset, int nalLimit) { + public static SpsData parseSpsNalUnitPayload(byte[] nalData, int nalOffset, int nalLimit) { ParsableNalUnitBitArray data = new ParsableNalUnitBitArray(nalData, nalOffset, nalLimit); - data.skipBits(8); // nal_unit int profileIdc = data.readBits(8); int constraintsFlagsAndReservedZero2Bits = data.readBits(8); int levelIdc = data.readBits(8); @@ -387,17 +426,166 @@ public final class NalUnitUtil { } /** - * Parses a PPS NAL unit using the syntax defined in ITU-T Recommendation H.264 (2013) subsection - * 7.3.2.2. + * Parses a H.265 SPS NAL unit payload using the syntax defined in ITU-T Recommendation H.265 (2019) + * subsection 7.3.2.2.1. + * + * @param nalData A buffer containing escaped SPS data. + * @param nalOffset The offset of the NAL unit in {@code nalData}. + * @param nalLimit The limit of the NAL unit in {@code nalData}. + * @return A parsed representation of the SPS data. + */ + public static H265SpsData parseH265SpsNalUnitPayload(byte[] nalData, int nalOffset, int nalLimit) { + ParsableNalUnitBitArray data = new ParsableNalUnitBitArray(nalData, nalOffset, nalLimit); + // Skip sps_video_parameter_set_id. + data.skipBits(8 + 4); + int maxSubLayersMinus1 = data.readBits(3); + data.skipBit(); // sps_temporal_id_nesting_flag + int generalProfileSpace = data.readBits(2); + boolean generalTierFlag = data.readBit(); + int generalProfileIdc = data.readBits(5); + int generalProfileCompatibilityFlags = 0; + for (int i = 0; i < 32; i++) { + if (data.readBit()) { + generalProfileCompatibilityFlags |= (1 << i); + } + } + int[] constraintBytes = new int[6]; + for (int i = 0; i < constraintBytes.length; ++i) { + constraintBytes[i] = data.readBits(8); + } + int generalLevelIdc = data.readBits(8); + int toSkip = 0; + for (int i = 0; i < maxSubLayersMinus1; i++) { + if (data.readBit()) { // sub_layer_profile_present_flag[i] + toSkip += 89; + } + if (data.readBit()) { // sub_layer_level_present_flag[i] + toSkip += 8; + } + } + data.skipBits(toSkip); + if (maxSubLayersMinus1 > 0) { + data.skipBits(2 * (8 - maxSubLayersMinus1)); + } + int seqParameterSetId = data.readUnsignedExpGolombCodedInt(); + int chromaFormatIdc = data.readUnsignedExpGolombCodedInt(); + if (chromaFormatIdc == 3) { + data.skipBit(); // separate_colour_plane_flag + } + int picWidthInLumaSamples = data.readUnsignedExpGolombCodedInt(); + int picHeightInLumaSamples = data.readUnsignedExpGolombCodedInt(); + if (data.readBit()) { // conformance_window_flag + int confWinLeftOffset = data.readUnsignedExpGolombCodedInt(); + int confWinRightOffset = data.readUnsignedExpGolombCodedInt(); + int confWinTopOffset = data.readUnsignedExpGolombCodedInt(); + int confWinBottomOffset = data.readUnsignedExpGolombCodedInt(); + // H.265/HEVC (2014) Table 6-1 + int subWidthC = chromaFormatIdc == 1 || chromaFormatIdc == 2 ? 2 : 1; + int subHeightC = chromaFormatIdc == 1 ? 2 : 1; + picWidthInLumaSamples -= subWidthC * (confWinLeftOffset + confWinRightOffset); + picHeightInLumaSamples -= subHeightC * (confWinTopOffset + confWinBottomOffset); + } + data.readUnsignedExpGolombCodedInt(); // bit_depth_luma_minus8 + data.readUnsignedExpGolombCodedInt(); // bit_depth_chroma_minus8 + int log2MaxPicOrderCntLsbMinus4 = data.readUnsignedExpGolombCodedInt(); + // for (i = sps_sub_layer_ordering_info_present_flag ? 0 : sps_max_sub_layers_minus1; ...) + for (int i = data.readBit() ? 0 : maxSubLayersMinus1; i <= maxSubLayersMinus1; i++) { + data.readUnsignedExpGolombCodedInt(); // sps_max_dec_pic_buffering_minus1[i] + data.readUnsignedExpGolombCodedInt(); // sps_max_num_reorder_pics[i] + data.readUnsignedExpGolombCodedInt(); // sps_max_latency_increase_plus1[i] + } + data.readUnsignedExpGolombCodedInt(); // log2_min_luma_coding_block_size_minus3 + data.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_luma_coding_block_size + data.readUnsignedExpGolombCodedInt(); // log2_min_luma_transform_block_size_minus2 + data.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_luma_transform_block_size + data.readUnsignedExpGolombCodedInt(); // max_transform_hierarchy_depth_inter + data.readUnsignedExpGolombCodedInt(); // max_transform_hierarchy_depth_intra + // if (scaling_list_enabled_flag) { if (sps_scaling_list_data_present_flag) {...}} + boolean scalingListEnabled = data.readBit(); + if (scalingListEnabled && data.readBit()) { + skipH265ScalingList(data); + } + data.skipBits(2); // amp_enabled_flag (1), sample_adaptive_offset_enabled_flag (1) + if (data.readBit()) { // pcm_enabled_flag + // pcm_sample_bit_depth_luma_minus1 (4), pcm_sample_bit_depth_chroma_minus1 (4) + data.skipBits(8); + data.readUnsignedExpGolombCodedInt(); // log2_min_pcm_luma_coding_block_size_minus3 + data.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_pcm_luma_coding_block_size + data.skipBit(); // pcm_loop_filter_disabled_flag + } + // Skips all short term reference picture sets. + skipShortTermRefPicSets(data); + if (data.readBit()) { // long_term_ref_pics_present_flag + // num_long_term_ref_pics_sps + for (int i = 0; i < data.readUnsignedExpGolombCodedInt(); i++) { + int ltRefPicPocLsbSpsLength = log2MaxPicOrderCntLsbMinus4 + 4; + // lt_ref_pic_poc_lsb_sps[i], used_by_curr_pic_lt_sps_flag[i] + data.skipBits(ltRefPicPocLsbSpsLength + 1); + } + } + data.skipBits(2); // sps_temporal_mvp_enabled_flag, strong_intra_smoothing_enabled_flag + float pixelWidthHeightRatio = 1; + if (data.readBit()) { // vui_parameters_present_flag + if (data.readBit()) { // aspect_ratio_info_present_flag + int aspectRatioIdc = data.readBits(8); + if (aspectRatioIdc == NalUnitUtil.EXTENDED_SAR) { + int sarWidth = data.readBits(16); + int sarHeight = data.readBits(16); + if (sarWidth != 0 && sarHeight != 0) { + pixelWidthHeightRatio = (float) sarWidth / sarHeight; + } + } else if (aspectRatioIdc < NalUnitUtil.ASPECT_RATIO_IDC_VALUES.length) { + pixelWidthHeightRatio = NalUnitUtil.ASPECT_RATIO_IDC_VALUES[aspectRatioIdc]; + } else { + Log.w(TAG, "Unexpected aspect_ratio_idc value: " + aspectRatioIdc); + } + } + if (data.readBit()) { // overscan_info_present_flag + data.skipBit(); // overscan_appropriate_flag + } + if (data.readBit()) { // video_signal_type_present_flag + data.skipBits(4); // video_format, video_full_range_flag + if (data.readBit()) { // colour_description_present_flag + // colour_primaries, transfer_characteristics, matrix_coeffs + data.skipBits(24); + } + } + if (data.readBit()) { // chroma_loc_info_present_flag + data.readUnsignedExpGolombCodedInt(); // chroma_sample_loc_type_top_field + data.readUnsignedExpGolombCodedInt(); // chroma_sample_loc_type_bottom_field + } + data.skipBit(); // neutral_chroma_indication_flag + if (data.readBit()) { // field_seq_flag + // field_seq_flag equal to 1 indicates that the coded video sequence conveys pictures that + // represent fields, which means that frame height is double the picture height. + picHeightInLumaSamples *= 2; + } + } + + return new H265SpsData( + generalProfileSpace, + generalTierFlag, + generalProfileIdc, + generalProfileCompatibilityFlags, + constraintBytes, + generalLevelIdc, + seqParameterSetId, + picWidthInLumaSamples, + picHeightInLumaSamples, + pixelWidthHeightRatio); + } + + /** + * Parses a PPS NAL unit payload using the syntax defined in ITU-T Recommendation H.264 (2013) + * subsection 7.3.2.2. * * @param nalData A buffer containing escaped PPS data. - * @param nalOffset The offset of the NAL unit header in {@code nalData}. + * @param nalOffset The offset of the NAL unit in {@code nalData}. * @param nalLimit The limit of the NAL unit in {@code nalData}. * @return A parsed representation of the PPS data. */ - public static PpsData parsePpsNalUnit(byte[] nalData, int nalOffset, int nalLimit) { + public static PpsData parsePpsNalUnitPayload(byte[] nalData, int nalOffset, int nalLimit) { ParsableNalUnitBitArray data = new ParsableNalUnitBitArray(nalData, nalOffset, nalLimit); - data.skipBits(8); // nal_unit int picParameterSetId = data.readUnsignedExpGolombCodedInt(); int seqParameterSetId = data.readUnsignedExpGolombCodedInt(); data.skipBit(); // entropy_coding_mode_flag @@ -516,6 +704,63 @@ public final class NalUnitUtil { } } + private static void skipH265ScalingList(ParsableNalUnitBitArray bitArray) { + for (int sizeId = 0; sizeId < 4; sizeId++) { + for (int matrixId = 0; matrixId < 6; matrixId += sizeId == 3 ? 3 : 1) { + if (!bitArray.readBit()) { // scaling_list_pred_mode_flag[sizeId][matrixId] + // scaling_list_pred_matrix_id_delta[sizeId][matrixId] + bitArray.readUnsignedExpGolombCodedInt(); + } else { + int coefNum = min(64, 1 << (4 + (sizeId << 1))); + if (sizeId > 1) { + // scaling_list_dc_coef_minus8[sizeId - 2][matrixId] + bitArray.readSignedExpGolombCodedInt(); + } + for (int i = 0; i < coefNum; i++) { + bitArray.readSignedExpGolombCodedInt(); // scaling_list_delta_coef + } + } + } + } + } + + private static void skipShortTermRefPicSets(ParsableNalUnitBitArray bitArray) { + int numShortTermRefPicSets = bitArray.readUnsignedExpGolombCodedInt(); + boolean interRefPicSetPredictionFlag = false; + int numNegativePics; + int numPositivePics; + // As this method applies in a SPS, the only element of NumDeltaPocs accessed is the previous + // one, so we just keep track of that rather than storing the whole array. + // RefRpsIdx = stRpsIdx - (delta_idx_minus1 + 1) and delta_idx_minus1 is always zero in SPS. + int previousNumDeltaPocs = 0; + for (int stRpsIdx = 0; stRpsIdx < numShortTermRefPicSets; stRpsIdx++) { + if (stRpsIdx != 0) { + interRefPicSetPredictionFlag = bitArray.readBit(); + } + if (interRefPicSetPredictionFlag) { + bitArray.skipBit(); // delta_rps_sign + bitArray.readUnsignedExpGolombCodedInt(); // abs_delta_rps_minus1 + for (int j = 0; j <= previousNumDeltaPocs; j++) { + if (bitArray.readBit()) { // used_by_curr_pic_flag[j] + bitArray.skipBit(); // use_delta_flag[j] + } + } + } else { + numNegativePics = bitArray.readUnsignedExpGolombCodedInt(); + numPositivePics = bitArray.readUnsignedExpGolombCodedInt(); + previousNumDeltaPocs = numNegativePics + numPositivePics; + for (int i = 0; i < numNegativePics; i++) { + bitArray.readUnsignedExpGolombCodedInt(); // delta_poc_s0_minus1[i] + bitArray.skipBit(); // used_by_curr_pic_s0_flag[i] + } + for (int i = 0; i < numPositivePics; i++) { + bitArray.readUnsignedExpGolombCodedInt(); // delta_poc_s1_minus1[i] + bitArray.skipBit(); // used_by_curr_pic_s1_flag[i] + } + } + } + } + private NalUnitUtil() { // Prevent instantiation. } diff --git a/library/common/src/test/java/com/google/android/exoplayer2/util/NalUnitUtilTest.java b/library/common/src/test/java/com/google/android/exoplayer2/util/NalUnitUtilTest.java index 4d9fc89a03..1d202c6eb6 100644 --- a/library/common/src/test/java/com/google/android/exoplayer2/util/NalUnitUtilTest.java +++ b/library/common/src/test/java/com/google/android/exoplayer2/util/NalUnitUtilTest.java @@ -33,7 +33,7 @@ public final class NalUnitUtilTest { createByteArray( 0x00, 0x00, 0x01, 0x67, 0x4D, 0x40, 0x16, 0xEC, 0xA0, 0x50, 0x17, 0xFC, 0xB8, 0x08, 0x80, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x0F, 0x47, 0x8B, 0x16, 0xCB); - private static final int SPS_TEST_DATA_OFFSET = 3; + private static final int SPS_TEST_DATA_OFFSET = 4; @Test public void findNalUnit() { @@ -121,9 +121,10 @@ public final class NalUnitUtilTest { } @Test - public void parseSpsNalUnit() { + public void parseSpsNalUnitPayload() { NalUnitUtil.SpsData data = - NalUnitUtil.parseSpsNalUnit(SPS_TEST_DATA, SPS_TEST_DATA_OFFSET, SPS_TEST_DATA.length); + NalUnitUtil.parseSpsNalUnitPayload( + SPS_TEST_DATA, SPS_TEST_DATA_OFFSET, SPS_TEST_DATA.length); assertThat(data.width).isEqualTo(640); assertThat(data.height).isEqualTo(360); assertThat(data.deltaPicOrderAlwaysZeroFlag).isFalse(); diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java index f6ed6f7796..fcb0db2c94 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/AtomParsers.java @@ -1139,6 +1139,9 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; HevcConfig hevcConfig = HevcConfig.parse(parent); initializationData = hevcConfig.initializationData; out.nalUnitLengthFieldLength = hevcConfig.nalUnitLengthFieldLength; + if (!pixelWidthHeightRatioFromPasp) { + pixelWidthHeightRatio = hevcConfig.pixelWidthHeightRatio; + } codecs = hevcConfig.codecs; } else if (childAtomType == Atom.TYPE_dvcC || childAtomType == Atom.TYPE_dvvC) { @Nullable DolbyVisionConfig dolbyVisionConfig = DolbyVisionConfig.parse(parent); diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java index 6790747ac9..8bdc00cc65 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java @@ -200,8 +200,8 @@ public final class H264Reader implements ElementaryStreamReader { List initializationData = new ArrayList<>(); initializationData.add(Arrays.copyOf(sps.nalData, sps.nalLength)); initializationData.add(Arrays.copyOf(pps.nalData, pps.nalLength)); - NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnit(sps.nalData, 3, sps.nalLength); - NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnit(pps.nalData, 3, pps.nalLength); + NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnitPayload(sps.nalData, 4, sps.nalLength); + NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnitPayload(pps.nalData, 4, pps.nalLength); String codecs = CodecSpecificDataUtil.buildAvcCodecString( spsData.profileIdc, @@ -224,11 +224,11 @@ public final class H264Reader implements ElementaryStreamReader { pps.reset(); } } else if (sps.isCompleted()) { - NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnit(sps.nalData, 3, sps.nalLength); + NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnitPayload(sps.nalData, 4, sps.nalLength); sampleReader.putSps(spsData); sps.reset(); } else if (pps.isCompleted()) { - NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnit(pps.nalData, 3, pps.nalLength); + NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnitPayload(pps.nalData, 4, pps.nalLength); sampleReader.putPps(ppsData); pps.reset(); } diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H265Reader.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H265Reader.java index 9dccd88280..18e4d2edd4 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H265Reader.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H265Reader.java @@ -243,10 +243,20 @@ public final class H265Reader implements ElementaryStreamReader { bitArray.skipBits(40 + 4); // NAL header, sps_video_parameter_set_id int maxSubLayersMinus1 = bitArray.readBits(3); bitArray.skipBit(); // sps_temporal_id_nesting_flag - - // profile_tier_level(1, sps_max_sub_layers_minus1) - bitArray.skipBits(88); // if (profilePresentFlag) {...} - bitArray.skipBits(8); // general_level_idc + int generalProfileSpace = bitArray.readBits(2); + boolean generalTierFlag = bitArray.readBit(); + int generalProfileIdc = bitArray.readBits(5); + int generalProfileCompatibilityFlags = 0; + for (int i = 0; i < 32; i++) { + if (bitArray.readBit()) { + generalProfileCompatibilityFlags |= (1 << i); + } + } + int[] constraintBytes = new int[6]; + for (int i = 0; i < constraintBytes.length; ++i) { + constraintBytes[i] = bitArray.readBits(8); + } + int generalLevelIdc = bitArray.readBits(8); int toSkip = 0; for (int i = 0; i < maxSubLayersMinus1; i++) { if (bitArray.readBit()) { // sub_layer_profile_present_flag[i] @@ -359,7 +369,9 @@ public final class H265Reader implements ElementaryStreamReader { // Parse the SPS to derive an RFC 6381 codecs string. bitArray.reset(sps.nalData, 0, sps.nalLength); bitArray.skipBits(24); // Skip start code. - String codecs = CodecSpecificDataUtil.buildHevcCodecStringFromSps(bitArray); + String codecs = CodecSpecificDataUtil.buildHevcCodecString( + generalProfileSpace, generalTierFlag, generalProfileIdc, + generalProfileCompatibilityFlags, constraintBytes, generalLevelIdc); return new Format.Builder() .setId(formatId) diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/video/AvcConfig.java b/library/extractor/src/main/java/com/google/android/exoplayer2/video/AvcConfig.java index a6c5577749..8f46a4afb7 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/video/AvcConfig.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/video/AvcConfig.java @@ -66,9 +66,8 @@ public final class AvcConfig { @Nullable String codecs = null; if (numSequenceParameterSets > 0) { byte[] sps = initializationData.get(0); - SpsData spsData = - NalUnitUtil.parseSpsNalUnit( - initializationData.get(0), nalUnitLengthFieldLength, sps.length); + SpsData spsData = NalUnitUtil.parseSpsNalUnitPayload(sps, + nalUnitLengthFieldLength + 1, sps.length); width = spsData.width; height = spsData.height; pixelWidthAspectRatio = spsData.pixelWidthAspectRatio; diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java b/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java index 567485ba92..a29b17fda1 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java @@ -16,11 +16,11 @@ package com.google.android.exoplayer2.video; import androidx.annotation.Nullable; +import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.ParserException; import com.google.android.exoplayer2.util.CodecSpecificDataUtil; import com.google.android.exoplayer2.util.NalUnitUtil; import com.google.android.exoplayer2.util.ParsableByteArray; -import com.google.android.exoplayer2.util.ParsableNalUnitBitArray; import java.util.Collections; import java.util.List; @@ -58,6 +58,9 @@ public final class HevcConfig { data.setPosition(csdStartPosition); byte[] buffer = new byte[csdLength]; int bufferPosition = 0; + int width = Format.NO_VALUE; + int height = Format.NO_VALUE; + float pixelWidthAspectRatio = 1; @Nullable String codecs = null; for (int i = 0; i < numberOfArrays; i++) { int nalUnitType = data.readUnsignedByte() & 0x7F; // completeness (1), nal_unit_type (7) @@ -74,12 +77,16 @@ public final class HevcConfig { System.arraycopy( data.getData(), data.getPosition(), buffer, bufferPosition, nalUnitLength); if (nalUnitType == SPS_NAL_UNIT_TYPE && j == 0) { - ParsableNalUnitBitArray bitArray = - new ParsableNalUnitBitArray( - buffer, - /* offset= */ bufferPosition, - /* limit= */ bufferPosition + nalUnitLength); - codecs = CodecSpecificDataUtil.buildHevcCodecStringFromSps(bitArray); + NalUnitUtil.H265SpsData spsData = + NalUnitUtil.parseH265SpsNalUnitPayload( + buffer, bufferPosition + 1, bufferPosition + nalUnitLength); + width = spsData.picWidthInLumaSamples; + height = spsData.picHeightInLumaSamples; + pixelWidthAspectRatio = spsData.pixelWidthHeightRatio; + codecs = CodecSpecificDataUtil.buildHevcCodecString(spsData.generalProfileSpace, + spsData.generalTierFlag, spsData.generalProfileIdc, + spsData.generalProfileCompatibilityFlags, spsData.constraintBytes, + spsData.generalLevelIdc); } bufferPosition += nalUnitLength; data.skipBytes(nalUnitLength); @@ -88,7 +95,13 @@ public final class HevcConfig { @Nullable List initializationData = csdLength == 0 ? null : Collections.singletonList(buffer); - return new HevcConfig(initializationData, lengthSizeMinusOne + 1, codecs); + return new HevcConfig( + initializationData, + lengthSizeMinusOne + 1, + width, + height, + pixelWidthAspectRatio, + codecs); } catch (ArrayIndexOutOfBoundsException e) { throw ParserException.createForMalformedContainer("Error parsing HEVC config", e); } @@ -105,6 +118,9 @@ public final class HevcConfig { @Nullable public final List initializationData; /** The length of the NAL unit length field in the bitstream's container, in bytes. */ public final int nalUnitLengthFieldLength; + public final int width; + public final int height; + public final float pixelWidthHeightRatio; /** * An RFC 6381 codecs string representing the video format, or {@code null} if not known. * @@ -115,9 +131,15 @@ public final class HevcConfig { private HevcConfig( @Nullable List initializationData, int nalUnitLengthFieldLength, + int width, + int height, + float pixelWidthAspectRatio, @Nullable String codecs) { this.initializationData = initializationData; this.nalUnitLengthFieldLength = nalUnitLengthFieldLength; + this.width = width; + this.height = height; + this.pixelWidthHeightRatio = pixelWidthAspectRatio; this.codecs = codecs; } } diff --git a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java index 2f6a3c3c01..5fef4ce231 100644 --- a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java +++ b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java @@ -183,8 +183,8 @@ import com.google.common.collect.ImmutableMap; // Process SPS (Sequence Parameter Set). byte[] spsNalDataWithStartCode = initializationData.get(0); NalUnitUtil.SpsData spsData = - NalUnitUtil.parseSpsNalUnit( - spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length); + NalUnitUtil.parseSpsNalUnitPayload( + spsNalDataWithStartCode, NAL_START_CODE.length + 1, spsNalDataWithStartCode.length); formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthAspectRatio); formatBuilder.setHeight(spsData.height); formatBuilder.setWidth(spsData.width); From dbc708871670dccbf1face1de3c512dda90cbd08 Mon Sep 17 00:00:00 2001 From: Marcus Wichelmann Date: Mon, 13 Sep 2021 12:30:21 +0200 Subject: [PATCH 2/4] Keep the existing parseSpsNalUnit (and similar) methods to avoid breaking changes --- .../android/exoplayer2/util/NalUnitUtil.java | 51 ++++++++++++++++--- .../exoplayer2/util/NalUnitUtilTest.java | 7 ++- .../exoplayer2/extractor/ts/H264Reader.java | 8 +-- .../android/exoplayer2/video/AvcConfig.java | 5 +- .../android/exoplayer2/video/HevcConfig.java | 4 +- .../source/rtsp/RtspMediaTrack.java | 4 +- 6 files changed, 59 insertions(+), 20 deletions(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java index 4b267e0bfa..a911e939e5 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java @@ -292,11 +292,24 @@ public final class NalUnitUtil { } /** - * Parses a SPS NAL unit payload using the syntax defined in ITU-T Recommendation H.264 (2013) + * Parses a SPS NAL unit using the syntax defined in ITU-T Recommendation H.264 (2013) subsection + * 7.3.2.1.1. + * + * @param nalData A buffer containing escaped SPS data. + * @param nalOffset The offset of the NAL unit header in {@code nalData}. + * @param nalLimit The limit of the NAL unit in {@code nalData}. + * @return A parsed representation of the SPS data. + */ + public static SpsData parseSpsNalUnit(byte[] nalData, int nalOffset, int nalLimit) { + return parseSpsNalUnitPayload(nalData, nalOffset + 1, nalLimit); + } + + /** + * Parses a SPS NAL unit payload (excluding the NAL unit header) using the syntax defined in ITU-T Recommendation H.264 (2013) * subsection 7.3.2.1.1. * * @param nalData A buffer containing escaped SPS data. - * @param nalOffset The offset of the NAL unit in {@code nalData}. + * @param nalOffset The offset of the NAL unit payload in {@code nalData}. * @param nalLimit The limit of the NAL unit in {@code nalData}. * @return A parsed representation of the SPS data. */ @@ -426,11 +439,24 @@ public final class NalUnitUtil { } /** - * Parses a H.265 SPS NAL unit payload using the syntax defined in ITU-T Recommendation H.265 (2019) + * Parses a H.265 SPS NAL unit using the syntax defined in ITU-T Recommendation H.265 (2019) * subsection 7.3.2.2.1. * * @param nalData A buffer containing escaped SPS data. - * @param nalOffset The offset of the NAL unit in {@code nalData}. + * @param nalOffset The offset of the NAL unit header in {@code nalData}. + * @param nalLimit The limit of the NAL unit in {@code nalData}. + * @return A parsed representation of the SPS data. + */ + public static H265SpsData parseH265SpsNalUnit(byte[] nalData, int nalOffset, int nalLimit) { + return parseH265SpsNalUnitPayload(nalData, nalOffset + 1, nalLimit); + } + + /** + * Parses a H.265 SPS NAL unit payload (excluding the NAL unit header) using the syntax defined in ITU-T Recommendation H.265 (2019) + * subsection 7.3.2.2.1. + * + * @param nalData A buffer containing escaped SPS data. + * @param nalOffset The offset of the NAL unit payload in {@code nalData}. * @param nalLimit The limit of the NAL unit in {@code nalData}. * @return A parsed representation of the SPS data. */ @@ -576,11 +602,24 @@ public final class NalUnitUtil { } /** - * Parses a PPS NAL unit payload using the syntax defined in ITU-T Recommendation H.264 (2013) + * Parses a PPS NAL unit using the syntax defined in ITU-T Recommendation H.264 (2013) subsection + * 7.3.2.2. + * + * @param nalData A buffer containing escaped PPS data. + * @param nalOffset The offset of the NAL unit header in {@code nalData}. + * @param nalLimit The limit of the NAL unit in {@code nalData}. + * @return A parsed representation of the PPS data. + */ + public static PpsData parsePpsNalUnit(byte[] nalData, int nalOffset, int nalLimit) { + return parsePpsNalUnitPayload(nalData, nalOffset + 1, nalLimit); + } + + /** + * Parses a PPS NAL unit payload (excluding the NAL unit header) using the syntax defined in ITU-T Recommendation H.264 (2013) * subsection 7.3.2.2. * * @param nalData A buffer containing escaped PPS data. - * @param nalOffset The offset of the NAL unit in {@code nalData}. + * @param nalOffset The offset of the NAL unit payload in {@code nalData}. * @param nalLimit The limit of the NAL unit in {@code nalData}. * @return A parsed representation of the PPS data. */ diff --git a/library/common/src/test/java/com/google/android/exoplayer2/util/NalUnitUtilTest.java b/library/common/src/test/java/com/google/android/exoplayer2/util/NalUnitUtilTest.java index 1d202c6eb6..4d9fc89a03 100644 --- a/library/common/src/test/java/com/google/android/exoplayer2/util/NalUnitUtilTest.java +++ b/library/common/src/test/java/com/google/android/exoplayer2/util/NalUnitUtilTest.java @@ -33,7 +33,7 @@ public final class NalUnitUtilTest { createByteArray( 0x00, 0x00, 0x01, 0x67, 0x4D, 0x40, 0x16, 0xEC, 0xA0, 0x50, 0x17, 0xFC, 0xB8, 0x08, 0x80, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x0F, 0x47, 0x8B, 0x16, 0xCB); - private static final int SPS_TEST_DATA_OFFSET = 4; + private static final int SPS_TEST_DATA_OFFSET = 3; @Test public void findNalUnit() { @@ -121,10 +121,9 @@ public final class NalUnitUtilTest { } @Test - public void parseSpsNalUnitPayload() { + public void parseSpsNalUnit() { NalUnitUtil.SpsData data = - NalUnitUtil.parseSpsNalUnitPayload( - SPS_TEST_DATA, SPS_TEST_DATA_OFFSET, SPS_TEST_DATA.length); + NalUnitUtil.parseSpsNalUnit(SPS_TEST_DATA, SPS_TEST_DATA_OFFSET, SPS_TEST_DATA.length); assertThat(data.width).isEqualTo(640); assertThat(data.height).isEqualTo(360); assertThat(data.deltaPicOrderAlwaysZeroFlag).isFalse(); diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java index 8bdc00cc65..6790747ac9 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java @@ -200,8 +200,8 @@ public final class H264Reader implements ElementaryStreamReader { List initializationData = new ArrayList<>(); initializationData.add(Arrays.copyOf(sps.nalData, sps.nalLength)); initializationData.add(Arrays.copyOf(pps.nalData, pps.nalLength)); - NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnitPayload(sps.nalData, 4, sps.nalLength); - NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnitPayload(pps.nalData, 4, pps.nalLength); + NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnit(sps.nalData, 3, sps.nalLength); + NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnit(pps.nalData, 3, pps.nalLength); String codecs = CodecSpecificDataUtil.buildAvcCodecString( spsData.profileIdc, @@ -224,11 +224,11 @@ public final class H264Reader implements ElementaryStreamReader { pps.reset(); } } else if (sps.isCompleted()) { - NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnitPayload(sps.nalData, 4, sps.nalLength); + NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnit(sps.nalData, 3, sps.nalLength); sampleReader.putSps(spsData); sps.reset(); } else if (pps.isCompleted()) { - NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnitPayload(pps.nalData, 4, pps.nalLength); + NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnit(pps.nalData, 3, pps.nalLength); sampleReader.putPps(ppsData); pps.reset(); } diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/video/AvcConfig.java b/library/extractor/src/main/java/com/google/android/exoplayer2/video/AvcConfig.java index 8f46a4afb7..a6c5577749 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/video/AvcConfig.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/video/AvcConfig.java @@ -66,8 +66,9 @@ public final class AvcConfig { @Nullable String codecs = null; if (numSequenceParameterSets > 0) { byte[] sps = initializationData.get(0); - SpsData spsData = NalUnitUtil.parseSpsNalUnitPayload(sps, - nalUnitLengthFieldLength + 1, sps.length); + SpsData spsData = + NalUnitUtil.parseSpsNalUnit( + initializationData.get(0), nalUnitLengthFieldLength, sps.length); width = spsData.width; height = spsData.height; pixelWidthAspectRatio = spsData.pixelWidthAspectRatio; diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java b/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java index a29b17fda1..87ee8fa359 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java @@ -78,8 +78,8 @@ public final class HevcConfig { data.getData(), data.getPosition(), buffer, bufferPosition, nalUnitLength); if (nalUnitType == SPS_NAL_UNIT_TYPE && j == 0) { NalUnitUtil.H265SpsData spsData = - NalUnitUtil.parseH265SpsNalUnitPayload( - buffer, bufferPosition + 1, bufferPosition + nalUnitLength); + NalUnitUtil.parseH265SpsNalUnit( + buffer, bufferPosition, bufferPosition + nalUnitLength); width = spsData.picWidthInLumaSamples; height = spsData.picHeightInLumaSamples; pixelWidthAspectRatio = spsData.pixelWidthHeightRatio; diff --git a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java index 5fef4ce231..2f6a3c3c01 100644 --- a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java +++ b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java @@ -183,8 +183,8 @@ import com.google.common.collect.ImmutableMap; // Process SPS (Sequence Parameter Set). byte[] spsNalDataWithStartCode = initializationData.get(0); NalUnitUtil.SpsData spsData = - NalUnitUtil.parseSpsNalUnitPayload( - spsNalDataWithStartCode, NAL_START_CODE.length + 1, spsNalDataWithStartCode.length); + NalUnitUtil.parseSpsNalUnit( + spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length); formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthAspectRatio); formatBuilder.setHeight(spsData.height); formatBuilder.setWidth(spsData.width); From 45db77dc6ddfb0ca9de191faad184aaf90821012 Mon Sep 17 00:00:00 2001 From: Marcus Wichelmann Date: Mon, 13 Sep 2021 12:34:44 +0200 Subject: [PATCH 3/4] Fixed some spaces --- .../java/com/google/android/exoplayer2/util/NalUnitUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java index a911e939e5..d3d075abf4 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/NalUnitUtil.java @@ -452,7 +452,7 @@ public final class NalUnitUtil { } /** - * Parses a H.265 SPS NAL unit payload (excluding the NAL unit header) using the syntax defined in ITU-T Recommendation H.265 (2019) + * Parses a H.265 SPS NAL unit payload (excluding the NAL unit header) using the syntax defined in ITU-T Recommendation H.265 (2019) * subsection 7.3.2.2.1. * * @param nalData A buffer containing escaped SPS data. @@ -615,7 +615,7 @@ public final class NalUnitUtil { } /** - * Parses a PPS NAL unit payload (excluding the NAL unit header) using the syntax defined in ITU-T Recommendation H.264 (2013) + * Parses a PPS NAL unit payload (excluding the NAL unit header) using the syntax defined in ITU-T Recommendation H.264 (2013) * subsection 7.3.2.2. * * @param nalData A buffer containing escaped PPS data. From 65001cc0e5063121420e698feaf7b1c5aabd0ed0 Mon Sep 17 00:00:00 2001 From: Marcus Wichelmann Date: Mon, 13 Sep 2021 16:48:08 +0200 Subject: [PATCH 4/4] Added javadoc comments to HevcConfig --- .../com/google/android/exoplayer2/video/HevcConfig.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java b/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java index 87ee8fa359..a2b62503c6 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/video/HevcConfig.java @@ -116,11 +116,19 @@ public final class HevcConfig { * @see com.google.android.exoplayer2.Format#initializationData */ @Nullable public final List initializationData; + /** The length of the NAL unit length field in the bitstream's container, in bytes. */ public final int nalUnitLengthFieldLength; + + /** The width of the encoded picture in luma samples. */ public final int width; + + /** The height of the encoded picture in luma samples. */ public final int height; + + /** The pixel aspect ratio. */ public final float pixelWidthHeightRatio; + /** * An RFC 6381 codecs string representing the video format, or {@code null} if not known. *