Add support for parsing LHEVCConfigurationBox.

Parse LHEVCDecoderConfigurationRecord with the ‘lhvC’ type and set the corresponding sample mime type to video/mv-hevc.  With no MV-HEVC decoder available, fallback to single-layer HEVC decoding.

PiperOrigin-RevId: 649119173
This commit is contained in:
Googler 2024-07-03 10:21:23 -07:00 committed by Copybara-Service
parent 8632c3add6
commit 0d4a785b61
9 changed files with 670 additions and 8 deletions

View File

@ -62,6 +62,7 @@ public final class MimeTypes {
public static final String VIDEO_MJPEG = BASE_TYPE_VIDEO + "/mjpeg";
public static final String VIDEO_MP42 = BASE_TYPE_VIDEO + "/mp42";
public static final String VIDEO_MP43 = BASE_TYPE_VIDEO + "/mp43";
@UnstableApi public static final String VIDEO_MV_HEVC = BASE_TYPE_VIDEO + "/mv-hevc";
@UnstableApi public static final String VIDEO_RAW = BASE_TYPE_VIDEO + "/raw";
@UnstableApi public static final String VIDEO_UNKNOWN = BASE_TYPE_VIDEO + "/x-unknown";

View File

@ -25,6 +25,7 @@ import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Assertions;
import androidx.media3.common.util.CodecSpecificDataUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
@ -1628,6 +1629,76 @@ public final class NalUnitUtil {
prefixFlags[2] = false;
}
/**
* Returns a new RFC 6381 codecs description string specifically for the single-layer HEVC case.
* When falling back to single-layer HEVC from L-HEVC, both profile and level should be adjusted
* for the base layer case and the codecs description string should represent that. For the
* single-layer HEVC case, the string is derived from the SPS of the base layer.
*
* @param csdBuffers The CSD buffers that include the SPS of the base layer.
* @return A RFC 6381 codecs string derived from the SPS of the base layer if such information is
* available, or null otherwise.
*/
@Nullable
public static String getH265BaseLayerCodecsString(List<byte[]> csdBuffers) {
for (int i = 0; i < csdBuffers.size(); i++) {
byte[] buffer = csdBuffers.get(i);
int limit = buffer.length;
if (limit > 3) {
ImmutableList<Integer> nalUnitPositions = findNalUnitPositions(buffer);
for (int j = 0; j < nalUnitPositions.size(); j++) {
// Start code prefix of 3 bytes is included in the nalUnitPositions.
if (nalUnitPositions.get(j) + 3 < limit) {
// Use the base layer (layerId == 0) SPS to derive new codecs string.
ParsableNalUnitBitArray data =
new ParsableNalUnitBitArray(buffer, nalUnitPositions.get(j) + 3, limit);
H265NalHeader nalHeader = parseH265NalHeader(data);
if (nalHeader.nalUnitType == H265_NAL_UNIT_TYPE_SPS && nalHeader.layerId == 0) {
return createCodecStringFromH265SpsPalyoad(data);
}
}
}
}
}
return null;
}
/** Finds all NAL unit positions from a given bitstream buffer. */
private static ImmutableList<Integer> findNalUnitPositions(byte[] data) {
int offset = 0;
boolean[] prefixFlags = new boolean[3];
ImmutableList.Builder<Integer> nalUnitPositions = ImmutableList.builder();
while (offset < data.length) {
int nalUnitOffset = findNalUnit(data, offset, data.length, prefixFlags);
if (nalUnitOffset != data.length) {
nalUnitPositions.add(nalUnitOffset);
}
offset = nalUnitOffset + 3;
}
return nalUnitPositions.build();
}
/** Creates a RFC 6381 HEVC codec string from a given SPS NAL unit payload. */
@Nullable
private static String createCodecStringFromH265SpsPalyoad(ParsableNalUnitBitArray data) {
data.skipBits(4); // sps_video_parameter_set_id
int maxSubLayersMinus1 = data.readBits(3);
data.skipBit(); // sps_temporal_id_nesting_flag
H265ProfileTierLevel profileTierLevel =
parseH265ProfileTierLevel(
data,
/* profilePresentFlag= */ true,
maxSubLayersMinus1,
/* prevProfileTierLevel= */ null);
return CodecSpecificDataUtil.buildHevcCodecString(
profileTierLevel.generalProfileSpace,
profileTierLevel.generalTierFlag,
profileTierLevel.generalProfileIdc,
profileTierLevel.generalProfileCompatibilityFlags,
profileTierLevel.constraintBytes,
profileTierLevel.generalLevelIdc);
}
private static int findNextUnescapeIndex(byte[] bytes, int offset, int limit) {
for (int i = offset; i < limit - 2; i++) {
if (bytes[i] == 0x00 && bytes[i + 1] == 0x00 && bytes[i + 2] == 0x03) {

View File

@ -296,6 +296,12 @@ public final class MediaCodecInfo {
private boolean isCodecProfileAndLevelSupported(
Format format, boolean checkPerformanceCapabilities) {
Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
if (format.sampleMimeType != null
&& format.sampleMimeType.equals(MimeTypes.VIDEO_MV_HEVC)
&& codecMimeType.equals(MimeTypes.VIDEO_H265)) {
// Falling back to single-layer HEVC from MV-HEVC. Get base layer profile and level.
codecProfileAndLevel = MediaCodecUtil.getHevcBaseLayerCodecProfileAndLevel(format);
}
if (codecProfileAndLevel == null) {
// If we don't know any better, we assume that the profile and level are supported.
return true;

View File

@ -35,6 +35,7 @@ import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.container.NalUnitUtil;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
@ -334,6 +335,24 @@ public final class MediaCodecUtil {
}
}
/**
* Returns profile and level (as defined by {@link CodecProfileLevel}) corresponding to the base
* layer (for the case of falling back to single-layer HEVC from L-HEVC).
*
* @param format Media format with codec specific initialization data.
* @return A pair (profile constant, level constant) if the initializationData of the {@code
* format} is well-formed and recognized, or null otherwise.
*/
@Nullable
public static Pair<Integer, Integer> getHevcBaseLayerCodecProfileAndLevel(Format format) {
String codecs = NalUnitUtil.getH265BaseLayerCodecsString(format.initializationData);
if (codecs == null) {
return null;
}
String[] parts = Util.split(codecs.trim(), "\\.");
return getHevcProfileAndLevel(codecs, parts, format.colorInfo);
}
/**
* Returns an alternative codec MIME type (besides the default {@link Format#sampleMimeType}) that
* can be used to decode samples of the provided {@link Format}.
@ -367,6 +386,10 @@ public final class MediaCodecUtil {
}
}
}
if (MimeTypes.VIDEO_MV_HEVC.equals(format.sampleMimeType)) {
// Single-layer HEVC decoders can decode the base layer of MV-HEVC streams.
return MimeTypes.VIDEO_H265;
}
return null;
}
@ -793,6 +816,9 @@ public final class MediaCodecUtil {
// Android versions, but we still map to Main10 for backwards compatibility.
profile = CodecProfileLevel.HEVCProfileMain10;
}
} else if ("6".equals(profileString)) {
// Framework does not have profileLevel.HEVCProfileMultiviewMain defined.
profile = 6;
} else {
Log.w(TAG, "Unknown HEVC profile string: " + profileString);
return null;

View File

@ -25,6 +25,7 @@ import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -32,6 +33,177 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public final class MediaCodecUtilTest {
private static final byte[] CSD0 =
new byte[] {
// Start code
0,
0,
0,
1,
// VPS
64,
1,
12,
17,
-1,
-1,
1,
96,
0,
0,
3,
0,
-80,
0,
0,
3,
0,
0,
3,
0,
120,
21,
-63,
91,
0,
32,
0,
40,
36,
-63,
-105,
6,
2,
0,
0,
3,
0,
-65,
-128,
0,
0,
3,
0,
0,
120,
-115,
7,
-128,
4,
64,
-96,
30,
92,
82,
-65,
72,
// Start code
0,
0,
0,
1,
// SPS for layer 0
66,
1,
1,
1,
96,
0,
0,
3,
0,
-80,
0,
0,
3,
0,
0,
3,
0,
120,
-96,
3,
-64,
-128,
17,
7,
-53,
-120,
21,
-18,
69,
-107,
77,
64,
64,
64,
64,
32,
// Start code
0,
0,
0,
1,
// PPS for layer 0
68,
1,
-64,
44,
-68,
20,
-55,
// Start code
0,
0,
0,
1,
// SEI
78,
1,
-80,
4,
4,
10,
-128,
32,
-128
};
private static final byte[] CSD1 =
new byte[] {
// Start code
0,
0,
0,
1,
// SPS for layer 1
66,
9,
14,
-126,
46,
69,
-118,
-96,
5,
1,
// Start code
0,
0,
0,
1,
// PPS for layer 1
68,
9,
72,
2,
-53,
-63,
77,
-88,
5
};
@Test
public void getCodecProfileAndLevel_handlesVp9Profile1CodecString() {
assertCodecProfileAndLevelForCodecsString(
@ -159,6 +331,39 @@ public final class MediaCodecUtilTest {
assertThat(MediaCodecUtil.getCodecProfileAndLevel(format)).isNull();
}
@Test
public void getCodecProfileAndLevel_handlesMvHevcCodecString() {
assertCodecProfileAndLevelForCodecsString(
MimeTypes.VIDEO_MV_HEVC,
"hvc1.6.40.L120.BF.80",
/* profile= */ 6,
MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel4);
}
@Test
public void getHevcBaseLayerCodecProfileAndLevel_handlesFallbackFromMvHevc() {
Format format =
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_MV_HEVC)
.setCodecs("hvc1.6.40.L120.BF.80")
.setInitializationData(ImmutableList.of(CSD0, CSD1))
.build();
assertHevcBaseLayerCodecProfileAndLevelForFormat(
format,
MediaCodecInfo.CodecProfileLevel.HEVCProfileMain,
MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel4);
}
@Test
public void getHevcBaseLayerCodecProfileAndLevel_rejectsFormatWithNoInitializationData() {
Format format =
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_MV_HEVC)
.setCodecs("hvc1.6.40.L120.BF.80")
.build();
assertThat(MediaCodecUtil.getHevcBaseLayerCodecProfileAndLevel(format)).isNull();
}
private static void assertCodecProfileAndLevelForCodecsString(
String sampleMimeType, String codecs, int profile, int level) {
Format format =
@ -173,4 +378,14 @@ public final class MediaCodecUtilTest {
assertThat(codecProfileAndLevel.first).isEqualTo(profile);
assertThat(codecProfileAndLevel.second).isEqualTo(level);
}
private static void assertHevcBaseLayerCodecProfileAndLevelForFormat(
Format format, int profile, int level) {
@Nullable
Pair<Integer, Integer> codecProfileAndLevel =
MediaCodecUtil.getHevcBaseLayerCodecProfileAndLevel(format);
assertThat(codecProfileAndLevel).isNotNull();
assertThat(codecProfileAndLevel.first).isEqualTo(profile);
assertThat(codecProfileAndLevel.second).isEqualTo(level);
}
}

View File

@ -39,8 +39,43 @@ public final class HevcConfig {
* @throws ParserException If an error occurred parsing the data.
*/
public static HevcConfig parse(ParsableByteArray data) throws ParserException {
return parseImpl(data, /* layered= */ false, /* vpsData= */ null);
}
/**
* Parses L-HEVC configuration data.
*
* @param data A {@link ParsableByteArray}, whose position is set to the start of the L-HEVC
* configuration data to parse.
* @param vpsData A parsed representation of VPS data.
* @return A parsed representation of the L-HEVC configuration data.
* @throws ParserException If an error occurred parsing the data.
*/
public static HevcConfig parseLayered(ParsableByteArray data, NalUnitUtil.H265VpsData vpsData)
throws ParserException {
return parseImpl(data, /* layered= */ true, vpsData);
}
/**
* Parses HEVC or L-HEVC configuration data.
*
* @param data A {@link ParsableByteArray}, whose position is set to the start of the HEVC/L-HEVC
* configuration data to parse.
* @param layered A flag indicating whether layered HEVC (L-HEVC) is being parsed or not.
* @param vpsData A parsed representation of VPS data or {@code null} if not available.
* @return A parsed representation of the HEVC/L-HEVC configuration data.
* @throws ParserException If an error occurred parsing the data.
*/
private static HevcConfig parseImpl(
ParsableByteArray data, boolean layered, @Nullable NalUnitUtil.H265VpsData vpsData)
throws ParserException {
try {
data.skipBytes(21); // Skip to the NAL unit length size field.
// Skip to the NAL unit length size field.
if (layered) {
data.skipBytes(4);
} else {
data.skipBytes(21);
}
int lengthSizeMinusOne = data.readUnsignedByte() & 0x03;
// Calculate the combined size of all VPS/SPS/PPS bitstreams.
@ -71,6 +106,7 @@ public final class HevcConfig {
float pixelWidthHeightRatio = 1;
int maxNumReorderPics = Format.NO_VALUE;
@Nullable String codecs = null;
@Nullable NalUnitUtil.H265VpsData currentVpsData = vpsData;
for (int i = 0; i < numberOfArrays; i++) {
int nalUnitType =
data.readUnsignedByte() & 0x3F; // completeness (1), reserved (1), nal_unit_type (6)
@ -86,10 +122,14 @@ public final class HevcConfig {
bufferPosition += NalUnitUtil.NAL_START_CODE.length;
System.arraycopy(
data.getData(), data.getPosition(), buffer, bufferPosition, nalUnitLength);
if (nalUnitType == SPS_NAL_UNIT_TYPE && j == 0) {
if (nalUnitType == NalUnitUtil.H265_NAL_UNIT_TYPE_VPS && j == 0) {
currentVpsData =
NalUnitUtil.parseH265VpsNalUnit(
buffer, bufferPosition, bufferPosition + nalUnitLength);
} else if (nalUnitType == NalUnitUtil.H265_NAL_UNIT_TYPE_SPS && j == 0) {
NalUnitUtil.H265SpsData spsData =
NalUnitUtil.parseH265SpsNalUnit(
buffer, bufferPosition, bufferPosition + nalUnitLength, null);
buffer, bufferPosition, bufferPosition + nalUnitLength, currentVpsData);
width = spsData.width;
height = spsData.height;
bitdepthLuma = spsData.bitDepthLumaMinus8 + 8;
@ -130,14 +170,14 @@ public final class HevcConfig {
colorTransfer,
pixelWidthHeightRatio,
maxNumReorderPics,
codecs);
codecs,
currentVpsData);
} catch (ArrayIndexOutOfBoundsException e) {
throw ParserException.createForMalformedContainer("Error parsing HEVC config", e);
throw ParserException.createForMalformedContainer(
"Error parsing" + (layered ? "L-HEVC config" : "HEVC config"), e);
}
}
private static final int SPS_NAL_UNIT_TYPE = 33;
/**
* List of buffers containing the codec-specific data to be provided to the decoder.
*
@ -195,6 +235,9 @@ public final class HevcConfig {
*/
@Nullable public final String codecs;
/** The parsed representation of VPS data or {@code null} if not available. */
@Nullable public final NalUnitUtil.H265VpsData vpsData;
private HevcConfig(
List<byte[]> initializationData,
int nalUnitLengthFieldLength,
@ -207,7 +250,8 @@ public final class HevcConfig {
@C.ColorTransfer int colorTransfer,
float pixelWidthHeightRatio,
int maxNumReorderPics,
@Nullable String codecs) {
@Nullable String codecs,
@Nullable NalUnitUtil.H265VpsData vpsData) {
this.initializationData = initializationData;
this.nalUnitLengthFieldLength = nalUnitLengthFieldLength;
this.width = width;
@ -220,5 +264,6 @@ public final class HevcConfig {
this.pixelWidthHeightRatio = pixelWidthHeightRatio;
this.maxNumReorderPics = maxNumReorderPics;
this.codecs = codecs;
this.vpsData = vpsData;
}
}

View File

@ -60,6 +60,9 @@ import java.util.List;
@SuppressWarnings("ConstantCaseForConstants")
public static final int TYPE_hvcC = 0x68766343;
@SuppressWarnings("ConstantCaseForConstants")
public static final int TYPE_lhvC = 0x6C687643;
@SuppressWarnings("ConstantCaseForConstants")
public static final int TYPE_vp08 = 0x76703038;

View File

@ -37,6 +37,7 @@ import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.Util;
import androidx.media3.container.Mp4LocationData;
import androidx.media3.container.Mp4TimestampData;
import androidx.media3.container.NalUnitUtil;
import androidx.media3.extractor.AacUtil;
import androidx.media3.extractor.Ac3Util;
import androidx.media3.extractor.Ac4Util;
@ -1154,6 +1155,7 @@ import java.util.Objects;
@C.StereoMode int stereoMode = Format.NO_VALUE;
@Nullable EsdsData esdsData = null;
int maxNumReorderSamples = Format.NO_VALUE;
@Nullable NalUnitUtil.H265VpsData vpsData = null;
// HDR related metadata.
@C.ColorSpace int colorSpace = Format.NO_VALUE;
@ -1206,6 +1208,54 @@ import java.util.Objects;
colorTransfer = hevcConfig.colorTransfer;
bitdepthLuma = hevcConfig.bitdepthLuma;
bitdepthChroma = hevcConfig.bitdepthChroma;
vpsData = hevcConfig.vpsData;
} else if (childAtomType == Atom.TYPE_lhvC) {
// The lhvC atom must follow the hvcC atom; so the media type must be already set.
ExtractorUtil.checkContainerInput(
MimeTypes.VIDEO_H265.equals(mimeType), "lhvC must follow hvcC atom");
ExtractorUtil.checkContainerInput(
vpsData != null && vpsData.layerInfos.size() >= 2, "must have at least two layers");
parent.setPosition(childStartPosition + Atom.HEADER_SIZE);
HevcConfig lhevcConfig = HevcConfig.parseLayered(parent, checkNotNull(vpsData));
ExtractorUtil.checkContainerInput(
out.nalUnitLengthFieldLength == lhevcConfig.nalUnitLengthFieldLength,
"nalUnitLengthFieldLength must be same for both hvcC and lhvC atoms");
// Only stereo MV-HEVC is currently supported, for which both views must have the same below
// configuration values.
if (lhevcConfig.colorSpace != Format.NO_VALUE) {
ExtractorUtil.checkContainerInput(
colorSpace == lhevcConfig.colorSpace, "colorSpace must be the same for both views");
}
if (lhevcConfig.colorRange != Format.NO_VALUE) {
ExtractorUtil.checkContainerInput(
colorRange == lhevcConfig.colorRange, "colorRange must be the same for both views");
}
if (lhevcConfig.colorTransfer != Format.NO_VALUE) {
ExtractorUtil.checkContainerInput(
colorTransfer == lhevcConfig.colorTransfer,
"colorTransfer must be the same for both views");
}
ExtractorUtil.checkContainerInput(
bitdepthLuma == lhevcConfig.bitdepthLuma,
"bitdepthLuma must be the same for both views");
ExtractorUtil.checkContainerInput(
bitdepthChroma == lhevcConfig.bitdepthChroma,
"bitdepthChroma must be the same for both views");
mimeType = MimeTypes.VIDEO_MV_HEVC;
if (initializationData != null) {
initializationData =
ImmutableList.<byte[]>builder()
.addAll(initializationData)
.addAll(lhevcConfig.initializationData)
.build();
} else {
ExtractorUtil.checkContainerInput(
false, "initializationData must be already set from hvcC atom");
}
codecs = lhevcConfig.codecs;
} else if (childAtomType == Atom.TYPE_dvcC || childAtomType == Atom.TYPE_dvvC) {
@Nullable DolbyVisionConfig dolbyVisionConfig = DolbyVisionConfig.parse(parent);
if (dolbyVisionConfig != null) {

View File

@ -242,6 +242,236 @@ public final class HevcConfigTest {
37
};
private static final byte[] HVCC_BOX_PAYLOAD_MV_HEVC =
new byte[] {
// Header
1,
1,
96,
0,
0,
0,
-80,
0,
0,
0,
0,
0,
120,
-16,
0,
-4,
-3,
-8,
-8,
0,
0,
11,
// Number of arrays
4,
// NAL unit type = VPS
-96,
// Number of NAL units
0,
1,
// NAL unit length
0,
56,
// NAL unit
64,
1,
12,
17,
-1,
-1,
1,
96,
0,
0,
3,
0,
-80,
0,
0,
3,
0,
0,
3,
0,
120,
21,
-63,
91,
0,
32,
0,
40,
36,
-63,
-105,
6,
2,
0,
0,
3,
0,
-65,
-128,
0,
0,
3,
0,
0,
120,
-115,
7,
-128,
4,
64,
-96,
30,
92,
82,
-65,
72,
// NAL unit type = SPS
-95,
// Number of NAL units
0,
1,
// NAL unit length
0,
36,
// NAL unit
66,
1,
1,
1,
96,
0,
0,
3,
0,
-80,
0,
0,
3,
0,
0,
3,
0,
120,
-96,
3,
-64,
-128,
17,
7,
-53,
-120,
21,
-18,
69,
-107,
77,
64,
64,
64,
64,
32,
// NAL unit type = PPS
-94,
// Number of NAL units
0,
1,
// NAL unit length
0,
7,
// NAL unit
68,
1,
-64,
44,
-68,
20,
-55,
// NAL unit type = SEI
-89,
// Number of NAL units
0,
1,
// NAL unit length
0,
9,
// NAL unit
78,
1,
-80,
4,
4,
10,
-128,
32,
-128
};
private static final byte[] LHVC_BOX_PAYLOAD_MV_HEVC =
new byte[] {
// Header
1,
-16,
0,
-4,
-53,
// Number of arrays
2,
// NAL unit type = SPS
-95,
// Number of NAL units
0,
1,
// NAL unit length
0,
10,
// NAL unit
66,
9,
14,
-126,
46,
69,
-118,
-96,
5,
1,
// NAL unit type = PPS
-94,
// Number of NAL units
0,
1,
// NAL unit length
0,
9,
// NAL unit
68,
9,
72,
2,
-53,
-63,
77,
-88,
5,
};
@Test
public void parseHevcDecoderConfigurationRecord() throws Exception {
ParsableByteArray data = new ParsableByteArray(HVCC_BOX_PAYLOAD);
@ -260,4 +490,19 @@ public final class HevcConfigTest {
assertThat(hevcConfig.codecs).isEqualTo("hvc1.1.6.L153.B0");
assertThat(hevcConfig.nalUnitLengthFieldLength).isEqualTo(4);
}
@Test
public void parseLhevcDecoderConfigurationRecord() throws Exception {
ParsableByteArray hevcData = new ParsableByteArray(HVCC_BOX_PAYLOAD_MV_HEVC);
HevcConfig hevcConfig = HevcConfig.parse(hevcData);
assertThat(hevcConfig.codecs).isEqualTo("hvc1.1.6.L120.B0");
assertThat(hevcConfig.nalUnitLengthFieldLength).isEqualTo(4);
ParsableByteArray lhevcData = new ParsableByteArray(LHVC_BOX_PAYLOAD_MV_HEVC);
HevcConfig lhevcConfig = HevcConfig.parseLayered(lhevcData, hevcConfig.vpsData);
assertThat(lhevcConfig.codecs).isEqualTo("hvc1.6.40.L120.BF.80");
assertThat(lhevcConfig.nalUnitLengthFieldLength).isEqualTo(4);
}
}