From 6ae8c8c0417b96e62d1a449f8a06d7300aab97c2 Mon Sep 17 00:00:00 2001 From: peng bin Date: Mon, 24 Apr 2023 11:03:47 +0800 Subject: [PATCH] Use NalUnitUtil.parseH265SpsNalUnitPayload when parsing MPEG-TS files This implicitly fixes a bug by removing the buggy implementation in H265Reader in favour of a working one. This change also adds tests to confirm the parsing bug is fixed. --- .../media3/extractor/ts/H265Reader.java | 222 +----------------- .../media3/extractor/ts/TsExtractorTest.java | 5 +- 2 files changed, 14 insertions(+), 213 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java index 7348f841be..fa1f492eff 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java @@ -246,229 +246,29 @@ public final class H265Reader implements ElementaryStreamReader { System.arraycopy(sps.nalData, 0, csdData, vps.nalLength, sps.nalLength); System.arraycopy(pps.nalData, 0, csdData, vps.nalLength + sps.nalLength, pps.nalLength); - // Parse the SPS NAL unit, as per H.265/HEVC (2014) 7.3.2.2.1. - ParsableNalUnitBitArray bitArray = new ParsableNalUnitBitArray(sps.nalData, 0, sps.nalLength); - bitArray.skipBits(40 + 4); // NAL header, sps_video_parameter_set_id - int maxSubLayersMinus1 = bitArray.readBits(3); - bitArray.skipBit(); // sps_temporal_id_nesting_flag - int generalProfileSpace = bitArray.readBits(2); - boolean generalTierFlag = bitArray.readBit(); - int generalProfileIdc = bitArray.readBits(5); - int generalProfileCompatibilityFlags = 0; - for (int i = 0; i < 32; i++) { - if (bitArray.readBit()) { - generalProfileCompatibilityFlags |= (1 << i); - } - } - int[] constraintBytes = new int[6]; - for (int i = 0; i < constraintBytes.length; ++i) { - constraintBytes[i] = bitArray.readBits(8); - } - int generalLevelIdc = bitArray.readBits(8); - int toSkip = 0; - for (int i = 0; i < maxSubLayersMinus1; i++) { - if (bitArray.readBit()) { // sub_layer_profile_present_flag[i] - toSkip += 89; - } - if (bitArray.readBit()) { // sub_layer_level_present_flag[i] - toSkip += 8; - } - } - bitArray.skipBits(toSkip); - if (maxSubLayersMinus1 > 0) { - bitArray.skipBits(2 * (8 - maxSubLayersMinus1)); - } - - bitArray.readUnsignedExpGolombCodedInt(); // sps_seq_parameter_set_id - int chromaFormatIdc = bitArray.readUnsignedExpGolombCodedInt(); - if (chromaFormatIdc == 3) { - bitArray.skipBit(); // separate_colour_plane_flag - } - int picWidthInLumaSamples = bitArray.readUnsignedExpGolombCodedInt(); - int picHeightInLumaSamples = bitArray.readUnsignedExpGolombCodedInt(); - if (bitArray.readBit()) { // conformance_window_flag - int confWinLeftOffset = bitArray.readUnsignedExpGolombCodedInt(); - int confWinRightOffset = bitArray.readUnsignedExpGolombCodedInt(); - int confWinTopOffset = bitArray.readUnsignedExpGolombCodedInt(); - int confWinBottomOffset = bitArray.readUnsignedExpGolombCodedInt(); - // H.265/HEVC (2014) Table 6-1 - int subWidthC = chromaFormatIdc == 1 || chromaFormatIdc == 2 ? 2 : 1; - int subHeightC = chromaFormatIdc == 1 ? 2 : 1; - picWidthInLumaSamples -= subWidthC * (confWinLeftOffset + confWinRightOffset); - picHeightInLumaSamples -= subHeightC * (confWinTopOffset + confWinBottomOffset); - } - bitArray.readUnsignedExpGolombCodedInt(); // bit_depth_luma_minus8 - bitArray.readUnsignedExpGolombCodedInt(); // bit_depth_chroma_minus8 - int log2MaxPicOrderCntLsbMinus4 = bitArray.readUnsignedExpGolombCodedInt(); - // for (i = sps_sub_layer_ordering_info_present_flag ? 0 : sps_max_sub_layers_minus1; ...) - for (int i = bitArray.readBit() ? 0 : maxSubLayersMinus1; i <= maxSubLayersMinus1; i++) { - bitArray.readUnsignedExpGolombCodedInt(); // sps_max_dec_pic_buffering_minus1[i] - bitArray.readUnsignedExpGolombCodedInt(); // sps_max_num_reorder_pics[i] - bitArray.readUnsignedExpGolombCodedInt(); // sps_max_latency_increase_plus1[i] - } - bitArray.readUnsignedExpGolombCodedInt(); // log2_min_luma_coding_block_size_minus3 - bitArray.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_luma_coding_block_size - bitArray.readUnsignedExpGolombCodedInt(); // log2_min_luma_transform_block_size_minus2 - bitArray.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_luma_transform_block_size - bitArray.readUnsignedExpGolombCodedInt(); // max_transform_hierarchy_depth_inter - bitArray.readUnsignedExpGolombCodedInt(); // max_transform_hierarchy_depth_intra - // if (scaling_list_enabled_flag) { if (sps_scaling_list_data_present_flag) {...}} - boolean scalingListEnabled = bitArray.readBit(); - if (scalingListEnabled && bitArray.readBit()) { - skipScalingList(bitArray); - } - bitArray.skipBits(2); // amp_enabled_flag (1), sample_adaptive_offset_enabled_flag (1) - if (bitArray.readBit()) { // pcm_enabled_flag - // pcm_sample_bit_depth_luma_minus1 (4), pcm_sample_bit_depth_chroma_minus1 (4) - bitArray.skipBits(8); - bitArray.readUnsignedExpGolombCodedInt(); // log2_min_pcm_luma_coding_block_size_minus3 - bitArray.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_pcm_luma_coding_block_size - bitArray.skipBit(); // pcm_loop_filter_disabled_flag - } - // Skips all short term reference picture sets. - skipShortTermRefPicSets(bitArray); - if (bitArray.readBit()) { // long_term_ref_pics_present_flag - // num_long_term_ref_pics_sps - for (int i = 0; i < bitArray.readUnsignedExpGolombCodedInt(); i++) { - int ltRefPicPocLsbSpsLength = log2MaxPicOrderCntLsbMinus4 + 4; - // lt_ref_pic_poc_lsb_sps[i], used_by_curr_pic_lt_sps_flag[i] - bitArray.skipBits(ltRefPicPocLsbSpsLength + 1); - } - } - bitArray.skipBits(2); // sps_temporal_mvp_enabled_flag, strong_intra_smoothing_enabled_flag - float pixelWidthHeightRatio = 1; - if (bitArray.readBit()) { // vui_parameters_present_flag - if (bitArray.readBit()) { // aspect_ratio_info_present_flag - int aspectRatioIdc = bitArray.readBits(8); - if (aspectRatioIdc == NalUnitUtil.EXTENDED_SAR) { - int sarWidth = bitArray.readBits(16); - int sarHeight = bitArray.readBits(16); - if (sarWidth != 0 && sarHeight != 0) { - pixelWidthHeightRatio = (float) sarWidth / sarHeight; - } - } else if (aspectRatioIdc < NalUnitUtil.ASPECT_RATIO_IDC_VALUES.length) { - pixelWidthHeightRatio = NalUnitUtil.ASPECT_RATIO_IDC_VALUES[aspectRatioIdc]; - } else { - Log.w(TAG, "Unexpected aspect_ratio_idc value: " + aspectRatioIdc); - } - } - if (bitArray.readBit()) { // overscan_info_present_flag - bitArray.skipBit(); // overscan_appropriate_flag - } - if (bitArray.readBit()) { // video_signal_type_present_flag - bitArray.skipBits(4); // video_format, video_full_range_flag - if (bitArray.readBit()) { // colour_description_present_flag - // colour_primaries, transfer_characteristics, matrix_coeffs - bitArray.skipBits(24); - } - } - if (bitArray.readBit()) { // chroma_loc_info_present_flag - bitArray.readUnsignedExpGolombCodedInt(); // chroma_sample_loc_type_top_field - bitArray.readUnsignedExpGolombCodedInt(); // chroma_sample_loc_type_bottom_field - } - bitArray.skipBit(); // neutral_chroma_indication_flag - if (bitArray.readBit()) { // field_seq_flag - // field_seq_flag equal to 1 indicates that the coded video sequence conveys pictures that - // represent fields, which means that frame height is double the picture height. - picHeightInLumaSamples *= 2; - } - } + NalUnitUtil.H265SpsData spsData = + NalUnitUtil.parseH265SpsNalUnitPayload(sps.nalData, 40/8, sps.nalLength); String codecs = CodecSpecificDataUtil.buildHevcCodecString( - generalProfileSpace, - generalTierFlag, - generalProfileIdc, - generalProfileCompatibilityFlags, - constraintBytes, - generalLevelIdc); + spsData.generalProfileSpace, + spsData.generalTierFlag, + spsData.generalProfileIdc, + spsData.generalProfileCompatibilityFlags, + spsData.constraintBytes, + spsData.generalLevelIdc); return new Format.Builder() .setId(formatId) .setSampleMimeType(MimeTypes.VIDEO_H265) .setCodecs(codecs) - .setWidth(picWidthInLumaSamples) - .setHeight(picHeightInLumaSamples) - .setPixelWidthHeightRatio(pixelWidthHeightRatio) + .setWidth(spsData.width) + .setHeight(spsData.height) + .setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio) .setInitializationData(Collections.singletonList(csdData)) .build(); } - /** Skips scaling_list_data(). See H.265/HEVC (2014) 7.3.4. */ - private static void skipScalingList(ParsableNalUnitBitArray bitArray) { - for (int sizeId = 0; sizeId < 4; sizeId++) { - for (int matrixId = 0; matrixId < 6; matrixId += sizeId == 3 ? 3 : 1) { - if (!bitArray.readBit()) { // scaling_list_pred_mode_flag[sizeId][matrixId] - // scaling_list_pred_matrix_id_delta[sizeId][matrixId] - bitArray.readUnsignedExpGolombCodedInt(); - } else { - int coefNum = min(64, 1 << (4 + (sizeId << 1))); - if (sizeId > 1) { - // scaling_list_dc_coef_minus8[sizeId - 2][matrixId] - bitArray.readSignedExpGolombCodedInt(); - } - for (int i = 0; i < coefNum; i++) { - bitArray.readSignedExpGolombCodedInt(); // scaling_list_delta_coef - } - } - } - } - } - - /** - * Reads the number of short term reference picture sets in a SPS as ue(v), then skips all of - * them. See H.265/HEVC (2014) 7.3.7 and 7.4.8. - */ - private static void skipShortTermRefPicSets(ParsableNalUnitBitArray bitArray) { - int numShortTermRefPicSets = bitArray.readUnsignedExpGolombCodedInt(); - boolean interRefPicSetPredictionFlag = false; - int numNegativePics; - int numPositivePics; - // As this method applies in a SPS, the only element of NumDeltaPocs accessed is the previous - // one, so we just keep track of that rather than storing the whole array. - // RefRpsIdx = stRpsIdx - (delta_idx_minus1 + 1) and delta_idx_minus1 is always zero in SPS. - int previousNumDeltaPocs = 0; - for (int stRpsIdx = 0; stRpsIdx < numShortTermRefPicSets; stRpsIdx++) { - if (stRpsIdx != 0) { - interRefPicSetPredictionFlag = bitArray.readBit(); - } - if (interRefPicSetPredictionFlag) { - boolean deltaRpsSign = bitArray.readBit(); // delta_rps_sign - bitArray.readUnsignedExpGolombCodedInt(); // abs_delta_rps_minus1 - numNegativePics = 0; - numPositivePics = 0; - for (int j = 0; j <= previousNumDeltaPocs; j++) { - if (!bitArray.readBit()) { // used_by_curr_pic_flag[j] - if (!bitArray.readBit()) { // use_delta_flag[j] - continue; // if not use_delta_flag, skip increase numDeltaPocs - } - } - if (deltaRpsSign) { - // See H.265/HEVC (2014) section 7.4.8 equation 7-61 - numNegativePics++; - } else { - // See H.265/HEVC (2014) section 7.4.8 equation 7-62 - numPositivePics++; - } - } - - } else { - numNegativePics = bitArray.readUnsignedExpGolombCodedInt(); - numPositivePics = bitArray.readUnsignedExpGolombCodedInt(); - for (int i = 0; i < numNegativePics; i++) { - bitArray.readUnsignedExpGolombCodedInt(); // delta_poc_s0_minus1[i] - bitArray.skipBit(); // used_by_curr_pic_s0_flag[i] - } - for (int i = 0; i < numPositivePics; i++) { - bitArray.readUnsignedExpGolombCodedInt(); // delta_poc_s1_minus1[i] - bitArray.skipBit(); // used_by_curr_pic_s1_flag[i] - } - } - // See H.265/HEVC (2014) section 7.4.8 equation 7-71 - previousNumDeltaPocs = numNegativePics + numPositivePics; - } - } - @EnsuresNonNull({"output", "sampleReader"}) private void assertTracksCreated() { Assertions.checkStateNotNull(output); diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/TsExtractorTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/TsExtractorTest.java index 48c29c8e39..0cd2e9cd30 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/TsExtractorTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/TsExtractorTest.java @@ -91,10 +91,11 @@ public final class TsExtractorTest { public void sampleWithH265() throws Exception { ExtractorAsserts.assertBehavior(TsExtractor::new, "media/ts/sample_h265.ts", simulationConfig); } - + @Test public void sampleWithH265RpsPred() throws Exception { - ExtractorAsserts.assertBehavior(TsExtractor::new, "media/ts/sample_h265_rps_pred.ts", simulationConfig); + ExtractorAsserts.assertBehavior( + TsExtractor::new, "media/ts/sample_h265_rps_pred.ts", simulationConfig); } @Test