loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use NalUnitUtil.parseH265SpsNalUnitPayload when parsing MPEG-TS files

Browse Source 
This implicitly fixes a bug by removing the buggy implementation in H265Reader in favour of a working one. This change also adds tests to confirm the parsing bug is fixed.
This commit is contained in:
peng bin 2023-04-24 11:03:47 +08:00 committed by Ian Baker
parent 114020c2d8
commit 6ae8c8c041
2 changed files with 14 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

222
libraries/extractor/src/main/java/androidx/media3/extractor/ts/H265Reader.java
View File

@ -246,229 +246,29 @@ public final class H265Reader implements ElementaryStreamReader {
System.arraycopy(sps.nalData, 0, csdData, vps.nalLength, sps.nalLength); System.arraycopy(sps.nalData, 0, csdData, vps.nalLength, sps.nalLength);
System.arraycopy(pps.nalData, 0, csdData, vps.nalLength + sps.nalLength, pps.nalLength); System.arraycopy(pps.nalData, 0, csdData, vps.nalLength + sps.nalLength, pps.nalLength);
// Parse the SPS NAL unit, as per H.265/HEVC (2014) 7.3.2.2.1. NalUnitUtil.H265SpsData spsData =
ParsableNalUnitBitArray bitArray = new ParsableNalUnitBitArray(sps.nalData, 0, sps.nalLength); NalUnitUtil.parseH265SpsNalUnitPayload(sps.nalData, 40/8, sps.nalLength);
bitArray.skipBits(40 + 4); // NAL header, sps_video_parameter_set_id
int maxSubLayersMinus1 = bitArray.readBits(3);
bitArray.skipBit(); // sps_temporal_id_nesting_flag
int generalProfileSpace = bitArray.readBits(2);
boolean generalTierFlag = bitArray.readBit();
int generalProfileIdc = bitArray.readBits(5);
int generalProfileCompatibilityFlags = 0;
for (int i = 0; i < 32; i++) {
if (bitArray.readBit()) {
generalProfileCompatibilityFlags |= (1 << i);
}
}
int[] constraintBytes = new int[6];
for (int i = 0; i < constraintBytes.length; ++i) {
constraintBytes[i] = bitArray.readBits(8);
}
int generalLevelIdc = bitArray.readBits(8);
int toSkip = 0;
for (int i = 0; i < maxSubLayersMinus1; i++) {
if (bitArray.readBit()) { // sub_layer_profile_present_flag[i]
toSkip += 89;
}
if (bitArray.readBit()) { // sub_layer_level_present_flag[i]
toSkip += 8;
}
}
bitArray.skipBits(toSkip);
if (maxSubLayersMinus1 > 0) {
bitArray.skipBits(2 * (8 - maxSubLayersMinus1));
}
bitArray.readUnsignedExpGolombCodedInt(); // sps_seq_parameter_set_id
int chromaFormatIdc = bitArray.readUnsignedExpGolombCodedInt();
if (chromaFormatIdc == 3) {
bitArray.skipBit(); // separate_colour_plane_flag
}
int picWidthInLumaSamples = bitArray.readUnsignedExpGolombCodedInt();
int picHeightInLumaSamples = bitArray.readUnsignedExpGolombCodedInt();
if (bitArray.readBit()) { // conformance_window_flag
int confWinLeftOffset = bitArray.readUnsignedExpGolombCodedInt();
int confWinRightOffset = bitArray.readUnsignedExpGolombCodedInt();
int confWinTopOffset = bitArray.readUnsignedExpGolombCodedInt();
int confWinBottomOffset = bitArray.readUnsignedExpGolombCodedInt();
// H.265/HEVC (2014) Table 6-1
int subWidthC = chromaFormatIdc == 1 || chromaFormatIdc == 2 ? 2 : 1;
int subHeightC = chromaFormatIdc == 1 ? 2 : 1;
picWidthInLumaSamples -= subWidthC * (confWinLeftOffset + confWinRightOffset);
picHeightInLumaSamples -= subHeightC * (confWinTopOffset + confWinBottomOffset);
}
bitArray.readUnsignedExpGolombCodedInt(); // bit_depth_luma_minus8
bitArray.readUnsignedExpGolombCodedInt(); // bit_depth_chroma_minus8
int log2MaxPicOrderCntLsbMinus4 = bitArray.readUnsignedExpGolombCodedInt();
// for (i = sps_sub_layer_ordering_info_present_flag ? 0 : sps_max_sub_layers_minus1; ...)
for (int i = bitArray.readBit() ? 0 : maxSubLayersMinus1; i <= maxSubLayersMinus1; i++) {
bitArray.readUnsignedExpGolombCodedInt(); // sps_max_dec_pic_buffering_minus1[i]
bitArray.readUnsignedExpGolombCodedInt(); // sps_max_num_reorder_pics[i]
bitArray.readUnsignedExpGolombCodedInt(); // sps_max_latency_increase_plus1[i]
}
bitArray.readUnsignedExpGolombCodedInt(); // log2_min_luma_coding_block_size_minus3
bitArray.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_luma_coding_block_size
bitArray.readUnsignedExpGolombCodedInt(); // log2_min_luma_transform_block_size_minus2
bitArray.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_luma_transform_block_size
bitArray.readUnsignedExpGolombCodedInt(); // max_transform_hierarchy_depth_inter
bitArray.readUnsignedExpGolombCodedInt(); // max_transform_hierarchy_depth_intra
// if (scaling_list_enabled_flag) { if (sps_scaling_list_data_present_flag) {...}}
boolean scalingListEnabled = bitArray.readBit();
if (scalingListEnabled && bitArray.readBit()) {
skipScalingList(bitArray);
}
bitArray.skipBits(2); // amp_enabled_flag (1), sample_adaptive_offset_enabled_flag (1)
if (bitArray.readBit()) { // pcm_enabled_flag
// pcm_sample_bit_depth_luma_minus1 (4), pcm_sample_bit_depth_chroma_minus1 (4)
bitArray.skipBits(8);
bitArray.readUnsignedExpGolombCodedInt(); // log2_min_pcm_luma_coding_block_size_minus3
bitArray.readUnsignedExpGolombCodedInt(); // log2_diff_max_min_pcm_luma_coding_block_size
bitArray.skipBit(); // pcm_loop_filter_disabled_flag
}
// Skips all short term reference picture sets.
skipShortTermRefPicSets(bitArray);
if (bitArray.readBit()) { // long_term_ref_pics_present_flag
// num_long_term_ref_pics_sps
for (int i = 0; i < bitArray.readUnsignedExpGolombCodedInt(); i++) {
int ltRefPicPocLsbSpsLength = log2MaxPicOrderCntLsbMinus4 + 4;
// lt_ref_pic_poc_lsb_sps[i], used_by_curr_pic_lt_sps_flag[i]
bitArray.skipBits(ltRefPicPocLsbSpsLength + 1);
}
}
bitArray.skipBits(2); // sps_temporal_mvp_enabled_flag, strong_intra_smoothing_enabled_flag
float pixelWidthHeightRatio = 1;
if (bitArray.readBit()) { // vui_parameters_present_flag
if (bitArray.readBit()) { // aspect_ratio_info_present_flag
int aspectRatioIdc = bitArray.readBits(8);
if (aspectRatioIdc == NalUnitUtil.EXTENDED_SAR) {
int sarWidth = bitArray.readBits(16);
int sarHeight = bitArray.readBits(16);
if (sarWidth != 0 && sarHeight != 0) {
pixelWidthHeightRatio = (float) sarWidth / sarHeight;
}
} else if (aspectRatioIdc < NalUnitUtil.ASPECT_RATIO_IDC_VALUES.length) {
pixelWidthHeightRatio = NalUnitUtil.ASPECT_RATIO_IDC_VALUES[aspectRatioIdc];
} else {
Log.w(TAG, "Unexpected aspect_ratio_idc value: " + aspectRatioIdc);
}
}
if (bitArray.readBit()) { // overscan_info_present_flag
bitArray.skipBit(); // overscan_appropriate_flag
}
if (bitArray.readBit()) { // video_signal_type_present_flag
bitArray.skipBits(4); // video_format, video_full_range_flag
if (bitArray.readBit()) { // colour_description_present_flag
// colour_primaries, transfer_characteristics, matrix_coeffs
bitArray.skipBits(24);
}
}
if (bitArray.readBit()) { // chroma_loc_info_present_flag
bitArray.readUnsignedExpGolombCodedInt(); // chroma_sample_loc_type_top_field
bitArray.readUnsignedExpGolombCodedInt(); // chroma_sample_loc_type_bottom_field
}
bitArray.skipBit(); // neutral_chroma_indication_flag
if (bitArray.readBit()) { // field_seq_flag
// field_seq_flag equal to 1 indicates that the coded video sequence conveys pictures that
// represent fields, which means that frame height is double the picture height.
picHeightInLumaSamples *= 2;
}
}
String codecs = String codecs =
CodecSpecificDataUtil.buildHevcCodecString( CodecSpecificDataUtil.buildHevcCodecString(
generalProfileSpace, spsData.generalProfileSpace,
generalTierFlag, spsData.generalTierFlag,
generalProfileIdc, spsData.generalProfileIdc,
generalProfileCompatibilityFlags, spsData.generalProfileCompatibilityFlags,
constraintBytes, spsData.constraintBytes,
generalLevelIdc); spsData.generalLevelIdc);
return new Format.Builder() return new Format.Builder()
.setId(formatId) .setId(formatId)
.setSampleMimeType(MimeTypes.VIDEO_H265) .setSampleMimeType(MimeTypes.VIDEO_H265)
.setCodecs(codecs) .setCodecs(codecs)
.setWidth(picWidthInLumaSamples) .setWidth(spsData.width)
.setHeight(picHeightInLumaSamples) .setHeight(spsData.height)
.setPixelWidthHeightRatio(pixelWidthHeightRatio) .setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio)
.setInitializationData(Collections.singletonList(csdData)) .setInitializationData(Collections.singletonList(csdData))
.build(); .build();
} }
/** Skips scaling_list_data(). See H.265/HEVC (2014) 7.3.4. */
private static void skipScalingList(ParsableNalUnitBitArray bitArray) {
for (int sizeId = 0; sizeId < 4; sizeId++) {
for (int matrixId = 0; matrixId < 6; matrixId += sizeId == 3 ? 3 : 1) {
if (!bitArray.readBit()) { // scaling_list_pred_mode_flag[sizeId][matrixId]
// scaling_list_pred_matrix_id_delta[sizeId][matrixId]
bitArray.readUnsignedExpGolombCodedInt();
} else {
int coefNum = min(64, 1 << (4 + (sizeId << 1)));
if (sizeId > 1) {
// scaling_list_dc_coef_minus8[sizeId - 2][matrixId]
bitArray.readSignedExpGolombCodedInt();
}
for (int i = 0; i < coefNum; i++) {
bitArray.readSignedExpGolombCodedInt(); // scaling_list_delta_coef
}
}
}
}
}
/**
* Reads the number of short term reference picture sets in a SPS as ue(v), then skips all of
* them. See H.265/HEVC (2014) 7.3.7 and 7.4.8.
*/
private static void skipShortTermRefPicSets(ParsableNalUnitBitArray bitArray) {
int numShortTermRefPicSets = bitArray.readUnsignedExpGolombCodedInt();
boolean interRefPicSetPredictionFlag = false;
int numNegativePics;
int numPositivePics;
// As this method applies in a SPS, the only element of NumDeltaPocs accessed is the previous
// one, so we just keep track of that rather than storing the whole array.
// RefRpsIdx = stRpsIdx - (delta_idx_minus1 + 1) and delta_idx_minus1 is always zero in SPS.
int previousNumDeltaPocs = 0;
for (int stRpsIdx = 0; stRpsIdx < numShortTermRefPicSets; stRpsIdx++) {
if (stRpsIdx != 0) {
interRefPicSetPredictionFlag = bitArray.readBit();
}
if (interRefPicSetPredictionFlag) {
boolean deltaRpsSign = bitArray.readBit(); // delta_rps_sign
bitArray.readUnsignedExpGolombCodedInt(); // abs_delta_rps_minus1
numNegativePics = 0;
numPositivePics = 0;
for (int j = 0; j <= previousNumDeltaPocs; j++) {
if (!bitArray.readBit()) { // used_by_curr_pic_flag[j]
if (!bitArray.readBit()) { // use_delta_flag[j]
continue; // if not use_delta_flag, skip increase numDeltaPocs
}
}
if (deltaRpsSign) {
// See H.265/HEVC (2014) section 7.4.8 equation 7-61
numNegativePics++;
} else {
// See H.265/HEVC (2014) section 7.4.8 equation 7-62
numPositivePics++;
}
}
} else {
numNegativePics = bitArray.readUnsignedExpGolombCodedInt();
numPositivePics = bitArray.readUnsignedExpGolombCodedInt();
for (int i = 0; i < numNegativePics; i++) {
bitArray.readUnsignedExpGolombCodedInt(); // delta_poc_s0_minus1[i]
bitArray.skipBit(); // used_by_curr_pic_s0_flag[i]
}
for (int i = 0; i < numPositivePics; i++) {
bitArray.readUnsignedExpGolombCodedInt(); // delta_poc_s1_minus1[i]
bitArray.skipBit(); // used_by_curr_pic_s1_flag[i]
}
}
// See H.265/HEVC (2014) section 7.4.8 equation 7-71
previousNumDeltaPocs = numNegativePics + numPositivePics;
}
}
@EnsuresNonNull({"output", "sampleReader"}) @EnsuresNonNull({"output", "sampleReader"})
private void assertTracksCreated() { private void assertTracksCreated() {
Assertions.checkStateNotNull(output); Assertions.checkStateNotNull(output);

5
libraries/extractor/src/test/java/androidx/media3/extractor/ts/TsExtractorTest.java
View File

@ -91,10 +91,11 @@ public final class TsExtractorTest {
public void sampleWithH265() throws Exception { public void sampleWithH265() throws Exception {
ExtractorAsserts.assertBehavior(TsExtractor::new, "media/ts/sample_h265.ts", simulationConfig); ExtractorAsserts.assertBehavior(TsExtractor::new, "media/ts/sample_h265.ts", simulationConfig);
} }
@Test @Test
public void sampleWithH265RpsPred() throws Exception { public void sampleWithH265RpsPred() throws Exception {
ExtractorAsserts.assertBehavior(TsExtractor::new, "media/ts/sample_h265_rps_pred.ts", simulationConfig); ExtractorAsserts.assertBehavior(
TsExtractor::new, "media/ts/sample_h265_rps_pred.ts", simulationConfig);
} }
@Test @Test