From 31ae260db49a405811ecef8c8ba29a67231f0549 Mon Sep 17 00:00:00 2001 From: dancho Date: Wed, 6 Nov 2024 03:55:06 -0800 Subject: [PATCH] Support alternative sample MIME types in MuxerWrapper Enable transformer to transmux into alternative sample MIME types. For example, some Dolby Vision profiles have a backwards-compatible AVC or HEVC layer. MV-HEVC is backwards compatible with HEVC. This change enables Transformer to transmux into the backwards compatible format to improve compatibility with legacy APIs such as MediaMetadataRetriever. PiperOrigin-RevId: 693667597 --- RELEASENOTES.md | 2 + .../transmuxed_with_inappmuxer.dump | 96 +++++++++++++++++++ .../transformer/TransformerEndToEndTest.java | 55 +++++++++++ .../media3/transformer/SampleExporter.java | 8 ++ .../media3/transformer/TransformerUtil.java | 17 +++- ...thInAppMuxerEndToEndParameterizedTest.java | 4 +- 6 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 libraries/test_data/src/test/assets/transformerdumps/mp4/dolbyVision-hdr.MOV/transmuxed_with_inappmuxer.dump diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 287a53cb69..77053a9feb 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -26,6 +26,8 @@ * Transformer: * Update parameters of `VideoFrameProcessor.registerInputStream` and `VideoFrameProcessor.Listener.onInputStreamRegistered` to use `Format`. + * Add support for transmuxing into alternative backwards compatible + formats. * Extractors: * DataSource: * `DataSourceContractTest`: Assert that `DataSource.getUri()` and diff --git a/libraries/test_data/src/test/assets/transformerdumps/mp4/dolbyVision-hdr.MOV/transmuxed_with_inappmuxer.dump b/libraries/test_data/src/test/assets/transformerdumps/mp4/dolbyVision-hdr.MOV/transmuxed_with_inappmuxer.dump new file mode 100644 index 0000000000..8c600a6ebf --- /dev/null +++ b/libraries/test_data/src/test/assets/transformerdumps/mp4/dolbyVision-hdr.MOV/transmuxed_with_inappmuxer.dump @@ -0,0 +1,96 @@ +seekMap: + isSeekable = true + duration = 166700 + getPosition(0) = [[timeUs=0, position=400052]] + getPosition(1) = [[timeUs=0, position=400052]] + getPosition(83350) = [[timeUs=0, position=400052]] + getPosition(166700) = [[timeUs=0, position=400052]] +numberOfTracks = 2 +track 0: + total output bytes = 56045 + sample count = 5 + track duration = 166700 + format 0: + id = 1 + sampleMimeType = video/hevc + codecs = hvc1.2.4.L93.B0 + maxInputSize = 30755 + maxNumReorderSamples = 2 + width = 1280 + height = 720 + frameRate = 30.00 + rotationDegrees = 90 + colorInfo: + colorSpace = 6 + colorRange = 2 + colorTransfer = 7 + lumaBitdepth = 10 + chromaBitdepth = 10 + metadata = entries=[mdta: key=com.apple.quicktime.location.accuracy.horizontal, value=35.000000, mdta: key=com.apple.quicktime.location.ISO6709, value=+51.5334-000.1255+026.756/, mdta: key=com.apple.quicktime.software, value=16.4.1, mdta: key=com.apple.quicktime.creationdate, value=2023-05-09T12:41:46+0100, mdta: key=com.apple.quicktime.make, value=Apple, mdta: key=com.apple.quicktime.model, value=iPhone 14 Pro, Mp4Timestamp: creation time=3000000000, modification time=4000000000, timescale=10000] + initializationData: + data = length 97, hash 71D67547 + sample 0: + time = 0 + flags = 1 + data = length 30725, hash A9F08085 + sample 1: + time = 133333 + flags = 0 + data = length 11543, hash D0A79439 + sample 2: + time = 66666 + flags = 0 + data = length 6139, hash 3FCEAFBE + sample 3: + time = 33333 + flags = 0 + data = length 3660, hash A80DE201 + sample 4: + time = 100000 + flags = 536870912 + data = length 3978, hash 59521A2C +track 1: + total output bytes = 2674 + sample count = 7 + track duration = 162600 + format 0: + peakBitrate = 192000 + id = 2 + sampleMimeType = audio/mp4a-latm + codecs = mp4a.40.2 + maxInputSize = 616 + channelCount = 2 + sampleRate = 44100 + language = und + metadata = entries=[mdta: key=com.apple.quicktime.location.accuracy.horizontal, value=35.000000, mdta: key=com.apple.quicktime.location.ISO6709, value=+51.5334-000.1255+026.756/, mdta: key=com.apple.quicktime.software, value=16.4.1, mdta: key=com.apple.quicktime.creationdate, value=2023-05-09T12:41:46+0100, mdta: key=com.apple.quicktime.make, value=Apple, mdta: key=com.apple.quicktime.model, value=iPhone 14 Pro, Mp4Timestamp: creation time=3000000000, modification time=4000000000, timescale=10000] + initializationData: + data = length 2, hash 5FF + sample 0: + time = 0 + flags = 1 + data = length 6, hash 6EFB71F1 + sample 1: + time = 23229 + flags = 1 + data = length 260, hash EE23C4F4 + sample 2: + time = 46458 + flags = 1 + data = length 458, hash C54D6937 + sample 3: + time = 69687 + flags = 1 + data = length 448, hash 319412F3 + sample 4: + time = 92916 + flags = 1 + data = length 422, hash 1E293713 + sample 5: + time = 116145 + flags = 1 + data = length 586, hash C4CD7E3D + sample 6: + time = 139375 + flags = 536870913 + data = length 494, hash A7AFD1B8 +tracksEnded = true diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 4a5cb08839..2919e798ff 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -25,6 +25,7 @@ import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET; import static androidx.media3.transformer.AndroidTestUtil.JPG_PIXEL_MOTION_PHOTO_ASSET; import static androidx.media3.transformer.AndroidTestUtil.MP3_ASSET; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S; @@ -1773,6 +1774,60 @@ public class TransformerEndToEndTest { assertThat(new File(result.filePath).length()).isGreaterThan(0); } + @Test + public void dolbyVisionVideo_noEffects_transmuxesToHevc() throws Exception { + assumeTrue("This test requires B-frame support", Util.SDK_INT > 24); + assumeTrue( + new DefaultMuxer.Factory() + .getSupportedSampleMimeTypes(C.TRACK_TYPE_VIDEO) + .contains(MimeTypes.VIDEO_H265)); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_DOLBY_VISION_HDR.uri))) + .setRemoveAudio(true) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder( + context, + new Transformer.Builder(context).setVideoMimeType(MimeTypes.VIDEO_H265).build()) + .build() + .run(testId, editedMediaItem); + + MediaExtractorCompat mediaExtractor = new MediaExtractorCompat(context); + mediaExtractor.setDataSource(Uri.parse(result.filePath), /* offset= */ 0); + checkState(mediaExtractor.getTrackCount() == 1); + MediaFormat mediaFormat = mediaExtractor.getTrackFormat(/* trackIndex= */ 0); + Format format = createFormatFromMediaFormat(mediaFormat); + assertThat(format.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H265); + assertThat(result.exportResult.videoConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED); + } + + @Test + public void dolbyVisionVideo_noEffects_withInAppMuxer_transmuxesToHevc() throws Exception { + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_DOLBY_VISION_HDR.uri))) + .setRemoveAudio(true) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder( + context, + new Transformer.Builder(context) + .setVideoMimeType(MimeTypes.VIDEO_H265) + .setMuxerFactory(new InAppMuxer.Factory.Builder().build()) + .build()) + .build() + .run(testId, editedMediaItem); + + MediaExtractorCompat mediaExtractor = new MediaExtractorCompat(context); + mediaExtractor.setDataSource(Uri.parse(result.filePath), /* offset= */ 0); + checkState(mediaExtractor.getTrackCount() == 1); + MediaFormat mediaFormat = mediaExtractor.getTrackFormat(/* trackIndex= */ 0); + Format format = createFormatFromMediaFormat(mediaFormat); + assertThat(format.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H265); + assertThat(result.exportResult.videoConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED); + } + @Test public void audioComposition_compositionEffects_transcodes() throws Exception { EditedMediaItem editedMediaItem = diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java index 98aa296560..33be1e3309 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java @@ -19,6 +19,7 @@ package androidx.media3.transformer; import static androidx.media3.common.ColorInfo.isTransferHdr; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.exoplayer.mediacodec.MediaCodecUtil.getAlternativeCodecMimeType; import static androidx.media3.transformer.EncoderUtil.getSupportedEncoders; import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing; import static androidx.media3.transformer.TransformerUtil.getProcessedTrackType; @@ -109,6 +110,13 @@ import java.util.List; if (metadata != null) { inputFormat = inputFormat.buildUpon().setMetadata(metadata).build(); } + if (!muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { + String alternativeSampleMimeType = getAlternativeCodecMimeType(inputFormat); + if (muxerWrapper.supportsSampleMimeType(alternativeSampleMimeType)) { + inputFormat = + inputFormat.buildUpon().setSampleMimeType(alternativeSampleMimeType).build(); + } + } try { muxerWrapper.addTrackFormat(inputFormat); } catch (MuxerException e) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java index 143cced7af..01e1d7bdae 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java @@ -19,6 +19,7 @@ package androidx.media3.transformer; import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED; import static androidx.media3.common.ColorInfo.isTransferHdr; import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.exoplayer.mediacodec.MediaCodecUtil.getAlternativeCodecMimeType; import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR; import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL; import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing; @@ -153,12 +154,18 @@ public final class TransformerUtil { if (transformationRequest.hdrMode != HDR_MODE_KEEP_HDR) { return true; } - if (transformationRequest.videoMimeType != null - && !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) { - return true; + @Nullable String requestedMimeType = transformationRequest.videoMimeType; + if (requestedMimeType != null) { + boolean requestedMimeTypeEqualsPrimaryOrAlternativeMimeType = + requestedMimeType.equals(inputFormat.sampleMimeType) + || requestedMimeType.equals(getAlternativeCodecMimeType(inputFormat)); + if (!requestedMimeTypeEqualsPrimaryOrAlternativeMimeType) { + return true; + } } - if (transformationRequest.videoMimeType == null - && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { + if (requestedMimeType == null + && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType) + && !muxerWrapper.supportsSampleMimeType(getAlternativeCodecMimeType(inputFormat))) { return true; } if (inputFormat.pixelWidthHeightRatio != 1f) { diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerWithInAppMuxerEndToEndParameterizedTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerWithInAppMuxerEndToEndParameterizedTest.java index 464c3739c6..1688948ba9 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerWithInAppMuxerEndToEndParameterizedTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerWithInAppMuxerEndToEndParameterizedTest.java @@ -53,6 +53,7 @@ public class TransformerWithInAppMuxerEndToEndParameterizedTest { private static final String AMR_WB_3GP = "mp4/bbb_mono_16kHz_23.05kbps_amrwb.3gp"; private static final String OPUS_OGG = "mp4/bbb_6ch_8kHz_opus.ogg"; private static final String VORBIS_OGG = "mp4/bbb_1ch_16kHz_q10_vorbis.ogg"; + private static final String DOLBY_VISION_MOV = "mp4/dolbyVision-hdr.MOV"; @Parameters(name = "{0}") public static ImmutableList mediaFiles() { @@ -65,7 +66,8 @@ public class TransformerWithInAppMuxerEndToEndParameterizedTest { AMR_NB_3GP, AMR_WB_3GP, OPUS_OGG, - VORBIS_OGG); + VORBIS_OGG, + DOLBY_VISION_MOV); } @Parameter public @MonotonicNonNull String inputFile;