From dc9d023e854ac0690bda9e161664cdb2199a54ee Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 21 Apr 2025 05:08:17 -0700 Subject: [PATCH] Dolby-Vision: Add dolby-vision codec support in Mp4Muxer Add dolby vision with hevc and avc codec in Mp4Muxer according to Dolby ISO media format standard. As initialization data is required for creation of dovi box, CSD is populated in BoxParser. PiperOrigin-RevId: 749765993 --- .../common/util/CodecSpecificDataUtil.java | 128 ++ .../media3/common/util/MediaFormatUtil.java | 15 +- .../androidx/media3/muxer/AnnexBUtils.java | 14 +- .../java/androidx/media3/muxer/Boxes.java | 92 ++ .../media3/muxer/FragmentedMp4Muxer.java | 4 +- .../media3/muxer/FragmentedMp4Writer.java | 2 +- .../java/androidx/media3/muxer/Mp4Muxer.java | 4 +- .../java/androidx/media3/muxer/Mp4Writer.java | 2 +- .../muxer/FragmentedMp4MuxerEndToEndTest.java | 4 + .../Mp4MuxerEndToEndParameterizedTest.java | 5 + .../video_dovi_1920x1080_60fps_dvav_09.mp4 | Bin 0 -> 3326794 bytes .../video_dovi_3840x2160_30fps_dav1_10.mp4 | Bin 0 -> 1460455 bytes .../muxerdumps/sample_edit_list.mp4.dump | 1208 +++++++++++++++++ .../sample_edit_list.mp4_fragmented.dump | 1192 ++++++++++++++++ ...ideo_dovi_1920x1080_60fps_dvav_09.mp4.dump | 672 +++++++++ ...920x1080_60fps_dvav_09.mp4_fragmented.dump | 659 +++++++++ .../transmuxed_with_inappmuxer.dump | 5 +- .../transformer/TransformerEndToEndTest.java | 20 +- 18 files changed, 4005 insertions(+), 21 deletions(-) create mode 100644 libraries/test_data/src/test/assets/media/mp4/video_dovi_1920x1080_60fps_dvav_09.mp4 create mode 100644 libraries/test_data/src/test/assets/media/mp4/video_dovi_3840x2160_30fps_dav1_10.mp4 create mode 100644 libraries/test_data/src/test/assets/muxerdumps/sample_edit_list.mp4.dump create mode 100644 libraries/test_data/src/test/assets/muxerdumps/sample_edit_list.mp4_fragmented.dump create mode 100644 libraries/test_data/src/test/assets/muxerdumps/video_dovi_1920x1080_60fps_dvav_09.mp4.dump create mode 100644 libraries/test_data/src/test/assets/muxerdumps/video_dovi_1920x1080_60fps_dvav_09.mp4_fragmented.dump diff --git a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java index 44c42ff85e..f9a9881d84 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java @@ -137,6 +137,40 @@ public final class CodecSpecificDataUtil { }); } + /** + * Returns initialization data for Dolby Vision according to Dolby + * Vision ISO MediaFormat (section 2.2) specification. + * + * @param profile The Dolby Vision codec profile. + * @param level The Dolby Vision codec level. + */ + public static byte[] buildDolbyVisionInitializationData(int profile, int level) { + byte[] dolbyVisionCsd = new byte[24]; + byte blCompatibilityId = 0x00; + // MD compression is not permitted for profile 7 and earlier. Only some devices + // support it from profile 8 + byte mdCompression = 0x00; + if (profile == 8) { + blCompatibilityId = 0x04; + } else if (profile == 9) { + blCompatibilityId = 0x02; + mdCompression = 0x01; + } + + dolbyVisionCsd[0] = 0x01; // dv_version_major + dolbyVisionCsd[1] = 0x00; // dv_version_minor + dolbyVisionCsd[2] = (byte) ((profile & 0x7f) << 1); // dv_profile + dolbyVisionCsd[2] = (byte) ((dolbyVisionCsd[2] | ((level >> 5) & 0x1)) & 0xff); + dolbyVisionCsd[3] = (byte) ((level & 0x1f) << 3); // dv_level + dolbyVisionCsd[3] = (byte) (dolbyVisionCsd[3] | (1 << 2)); // rpu_present_flag + dolbyVisionCsd[3] = (byte) (dolbyVisionCsd[3] | (0 << 1)); // el_present_flag + dolbyVisionCsd[3] = (byte) (dolbyVisionCsd[3] | 1); // bl_present_flag + dolbyVisionCsd[4] = (byte) (blCompatibilityId << 4); // dv_bl_signal_compatibility_id + dolbyVisionCsd[4] = (byte) (dolbyVisionCsd[4] | (mdCompression << 2)); // dv_md_compression + return dolbyVisionCsd; + } + /** * Parses an MPEG-4 Visual configuration information, as defined in ISO/IEC14496-2. * @@ -266,6 +300,23 @@ public final class CodecSpecificDataUtil { return Util.formatInvariant("s263.%d.%d", profile, level); } + /** + * Builds a Dolby Vision codec string using profile and level. + * + *

Reference: + * href="https://professionalsupport.dolby.com/s/article/What-is-Dolby-Vision-Profile?language=en_US"> + * Dolby Vision Profile and Level (section 2.3) + */ + public static String buildDolbyVisionCodecString(int profile, int level) { + if (profile > 9) { + return Util.formatInvariant("dvh1.%02d.%02d", profile, level); + } else if (profile > 8) { + return Util.formatInvariant("dvav.%02d.%02d", profile, level); + } else { + return Util.formatInvariant("dvhe.%02d.%02d", profile, level); + } + } + /** * Returns profile and level (as defined by {@link MediaCodecInfo.CodecProfileLevel}) * corresponding to the codec description string (as defined by RFC 6381) of the given format. @@ -407,6 +458,83 @@ public final class CodecSpecificDataUtil { return split; } + /** + * Returns Dolby Vision level number corresponding to the level constant. + * + * @param levelConstant The Dolby Vision level constant. + * @return The Dolby Vision level number. + * @throws IllegalArgumentException if the level constant is not recognized. + */ + public static int dolbyVisionConstantToLevelNumber(int levelConstant) { + switch (levelConstant) { + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelHd24: + return 1; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelHd30: + return 2; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelFhd24: + return 3; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelFhd30: + return 4; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelFhd60: + return 5; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelUhd24: + return 6; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelUhd30: + return 7; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelUhd48: + return 8; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelUhd60: + return 9; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevelUhd120: + return 10; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevel8k30: + return 11; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionLevel8k60: + return 12; + // TODO: b/179261323 - use framework constant for level 13. + case 0x1000: + return 13; + default: + throw new IllegalArgumentException("Unknown Dolby Vision level: " + levelConstant); + } + } + + /** + * Returns Dolby Vision profile number corresponding to the profile constant. + * + * @param profileConstant The Dolby Vision profile constant. + * @return The Dolby Vision profile number. + * @throws IllegalArgumentException if the profile constant is not recognized. + */ + public static int dolbyVisionConstantToProfileNumber(int profileConstant) { + switch (profileConstant) { + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvavPer: + return 0; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvavPen: + return 1; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvheDer: + return 2; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvheDen: + return 3; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvheDtr: + return 4; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvheStn: + return 5; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvheDth: + return 6; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvheDtb: + return 7; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvheSt: + return 8; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvavSe: + return 9; + case MediaCodecInfo.CodecProfileLevel.DolbyVisionProfileDvav110: + return 10; + default: + throw new IllegalArgumentException("Unknown Dolby Vision profile: " + profileConstant); + } + } + /** * Finds the next occurrence of the NAL start code from a given index. * diff --git a/libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java index 5f91180945..a040a2cb49 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java @@ -337,8 +337,9 @@ public final class MediaFormatUtil { } /** - * Returns a {@code Codecs string} of {@link MediaFormat}. In case of an H263 codec string, builds - * and returns an RFC 6381 H263 codec string using profile and level. + * Returns a {@code Codecs string} of {@link MediaFormat}. + * + *

For H263 and Dolby Vision formats, builds a codec string using profile and level. */ @Nullable @SuppressLint("InlinedApi") // Inlined MediaFormat keys. @@ -350,6 +351,16 @@ public final class MediaFormatUtil { return CodecSpecificDataUtil.buildH263CodecString( mediaFormat.getInteger(MediaFormat.KEY_PROFILE), mediaFormat.getInteger(MediaFormat.KEY_LEVEL)); + } else if (Objects.equals( + mediaFormat.getString(MediaFormat.KEY_MIME), MimeTypes.VIDEO_DOLBY_VISION) + && mediaFormat.containsKey(MediaFormat.KEY_PROFILE) + && mediaFormat.containsKey(MediaFormat.KEY_LEVEL)) { + // Add Dolby Vision profile and level to codec string as per Dolby Vision ISO media format. + return CodecSpecificDataUtil.buildDolbyVisionCodecString( + CodecSpecificDataUtil.dolbyVisionConstantToProfileNumber( + mediaFormat.getInteger(MediaFormat.KEY_PROFILE)), + CodecSpecificDataUtil.dolbyVisionConstantToLevelNumber( + mediaFormat.getInteger(MediaFormat.KEY_LEVEL))); } else { return getString(mediaFormat, MediaFormat.KEY_CODECS_STRING, /* defaultValue= */ null); } diff --git a/libraries/muxer/src/main/java/androidx/media3/muxer/AnnexBUtils.java b/libraries/muxer/src/main/java/androidx/media3/muxer/AnnexBUtils.java index 153366e454..9f30a8bd4e 100644 --- a/libraries/muxer/src/main/java/androidx/media3/muxer/AnnexBUtils.java +++ b/libraries/muxer/src/main/java/androidx/media3/muxer/AnnexBUtils.java @@ -15,8 +15,11 @@ */ package androidx.media3.muxer; +import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.muxer.Boxes.getDolbyVisionProfileAndLevel; +import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; @@ -105,7 +108,16 @@ import java.nio.ByteOrder; * Returns whether the sample of the given MIME type will contain NAL units in Annex-B format * (ISO/IEC 14496-10 Annex B, which uses start codes to delineate NAL units). */ - public static boolean doesSampleContainAnnexBNalUnits(String sampleMimeType) { + public static boolean doesSampleContainAnnexBNalUnits(Format format) { + String sampleMimeType = format.sampleMimeType; + checkNotNull(sampleMimeType); + if (sampleMimeType.equals(MimeTypes.VIDEO_DOLBY_VISION)) { + // Dolby vision with AV1 profile does not contain Nal units. + int profile = checkNotNull(getDolbyVisionProfileAndLevel(format)).first; + // Dolby vision with Profile 10 is equivalent to DolbyVisionProfileDvav110 of framework + // media codec constants. + return profile != 10; + } return sampleMimeType.equals(MimeTypes.VIDEO_H264) || sampleMimeType.equals(MimeTypes.VIDEO_H265); } diff --git a/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java b/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java index 12cb4c79bc..a654518e34 100644 --- a/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java +++ b/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java @@ -32,11 +32,15 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.CodecSpecificDataUtil; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.Util; +import androidx.media3.container.DolbyVisionConfig; import androidx.media3.container.MdtaMetadataEntry; import androidx.media3.container.Mp4LocationData; import androidx.media3.container.NalUnitUtil; import androidx.media3.muxer.FragmentedMp4Writer.SampleMetadata; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -91,6 +95,8 @@ import org.checkerframework.checker.nullness.qual.PolyNull; */ private static final int TRUN_BOX_NON_SYNC_SAMPLE_FLAGS = 0b00000001_00000001_00000000_00000000; + private static final String TAG = "Boxes"; + private Boxes() {} public static final ImmutableList XMP_UUID = @@ -722,6 +728,8 @@ import org.checkerframework.checker.nullness.qual.PolyNull; return esdsBox(format); case MimeTypes.VIDEO_VP9: return vpcCBox(format); + case MimeTypes.VIDEO_DOLBY_VISION: + return doviSpecificBox(format); default: throw new IllegalArgumentException("Unsupported format: " + mimeType); } @@ -1547,6 +1555,34 @@ import org.checkerframework.checker.nullness.qual.PolyNull; return BoxUtils.wrapIntoBox("av1C", ByteBuffer.wrap(csd0)); } + /** Returns a dvcC/dvwC/dvvC vision box which will be included in dolby vision box. */ + private static ByteBuffer doviBox(int profile, byte[] csd) { + checkArgument(csd.length > 0, "csd is empty for dovi box."); + if (profile <= 7) { + return BoxUtils.wrapIntoBox("dvcC", ByteBuffer.wrap(csd)); + } else if (profile <= 10) { + return BoxUtils.wrapIntoBox("dvvC", ByteBuffer.wrap(csd)); + } else if (profile <= 19) { + return BoxUtils.wrapIntoBox("dvwC", ByteBuffer.wrap(csd)); + } else if (profile == 20) { + return BoxUtils.wrapIntoBox("dvcC", ByteBuffer.wrap(csd)); + } else { + return BoxUtils.wrapIntoBox("dvwC", ByteBuffer.wrap(csd)); + } + } + + /** Returns a dolby vision box as per Dolby Vision ISO media format. */ + private static ByteBuffer doviSpecificBox(Format format) { + checkArgument( + !format.initializationData.isEmpty(), "csd is not found in the format for dolby vision"); + byte[] dolbyVisionCsd = Iterables.getLast(format.initializationData); + DolbyVisionConfig dolbyVisionConfig = getDolbyVisionConfig(format); + checkNotNull(dolbyVisionConfig, "Dolby vision codec is not supported."); + ByteBuffer avcHevcBox = dolbyVisionConfig.profile <= 8 ? hvcCBox(format) : avcCBox(format); + ByteBuffer dolbyBox = doviBox(dolbyVisionConfig.profile, dolbyVisionCsd); + return BoxUtils.concatenateBuffers(avcHevcBox, dolbyBox); + } + /** Returns the vpcC box as per VP Codec ISO Media File Format Binding v1.0. */ private static ByteBuffer vpcCBox(Format format) { // For VP9, the CodecPrivate or vpcCBox data is packed into csd-0. @@ -1690,6 +1726,45 @@ import org.checkerframework.checker.nullness.qual.PolyNull; return BoxUtils.wrapIntoBox("colr", contents); } + @Nullable + private static DolbyVisionConfig getDolbyVisionConfig(Format format) { + @Nullable + DolbyVisionConfig dolbyVisionConfig = + DolbyVisionConfig.parse( + new ParsableByteArray(Iterables.getLast(format.initializationData))); + if (dolbyVisionConfig == null && format.codecs != null) { + Pair profileAndLevel = getDolbyVisionProfileAndLevel(format); + checkNotNull(profileAndLevel, "Dolby Vision profile and level is not found."); + byte[] dolbyVisionCsd = + CodecSpecificDataUtil.buildDolbyVisionInitializationData( + /* profile= */ profileAndLevel.first, /* level= */ profileAndLevel.second); + dolbyVisionConfig = DolbyVisionConfig.parse(new ParsableByteArray(dolbyVisionCsd)); + } + return dolbyVisionConfig; + } + + /** Returns codec specific fourcc for Dolby vision. */ + private static String getDoviFourcc(Format format) { + @Nullable DolbyVisionConfig dolbyVisionConfig = getDolbyVisionConfig(format); + checkNotNull( + dolbyVisionConfig, + "Dolby Vision Initialization data is not found for format: %s" + format.sampleMimeType); + switch (dolbyVisionConfig.profile) { + case 5: + return "dvh1"; + case 8: + return "hvc1"; + case 9: + return "avc1"; + default: + throw new IllegalArgumentException( + "Unsupported profile " + + dolbyVisionConfig.profile + + " for format: " + + format.sampleMimeType); + } + } + /** Returns codec specific fourcc. */ private static String codecSpecificFourcc(Format format) { String mimeType = checkNotNull(format.sampleMimeType); @@ -1725,6 +1800,8 @@ import org.checkerframework.checker.nullness.qual.PolyNull; return "mp4v-es"; case MimeTypes.VIDEO_VP9: return "vp09"; + case MimeTypes.VIDEO_DOLBY_VISION: + return getDoviFourcc(format); default: throw new IllegalArgumentException("Unsupported format: " + mimeType); } @@ -1943,4 +2020,19 @@ import org.checkerframework.checker.nullness.qual.PolyNull; } return minInputPtsUs != Long.MAX_VALUE ? minInputPtsUs : C.TIME_UNSET; } + + /** Returns profile and level of dolby vision */ + @Nullable + /* package */ static Pair getDolbyVisionProfileAndLevel(Format format) { + checkNotNull(format.codecs, "Codec string is null for Dolby Vision format."); + List parts = Splitter.on('.').splitToList(format.codecs); + if (parts.size() < 3) { + // The codec has fewer parts than required by the Dolby Vision codec string format. + Log.w(TAG, "Invalid Dolby Vision codec string: " + format.codecs); + return null; + } + int profile = Integer.parseInt(parts.get(1)); + int level = Integer.parseInt(parts.get(2)); + return Pair.create(profile, level); + } } diff --git a/libraries/muxer/src/main/java/androidx/media3/muxer/FragmentedMp4Muxer.java b/libraries/muxer/src/main/java/androidx/media3/muxer/FragmentedMp4Muxer.java index e7fed6c030..c64490531b 100644 --- a/libraries/muxer/src/main/java/androidx/media3/muxer/FragmentedMp4Muxer.java +++ b/libraries/muxer/src/main/java/androidx/media3/muxer/FragmentedMp4Muxer.java @@ -49,6 +49,7 @@ import java.nio.ByteBuffer; *

  • H.265 (HEVC) *
  • VP9 *
  • APV + *
  • Dolby Vision * *
  • Audio Codecs: * *
  • Audio Codecs: *