loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Support alternative sample MIME types in MuxerWrapper

Browse Source 
Enable transformer to transmux into alternative sample MIME types.
For example, some Dolby Vision profiles have a backwards-compatible
AVC or HEVC layer. MV-HEVC is backwards compatible with HEVC.

This change enables Transformer to transmux into the backwards compatible
format to improve compatibility with legacy APIs such as
MediaMetadataRetriever.

PiperOrigin-RevId: 693667597
This commit is contained in:
dancho 2024-11-06 03:55:06 -08:00 committed by Copybara-Service
parent f9fd8badec
commit 31ae260db4
6 changed files with 176 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -26,6 +26,8 @@
* Transformer:
* Update parameters of `VideoFrameProcessor.registerInputStream` and
`VideoFrameProcessor.Listener.onInputStreamRegistered` to use `Format`.
* Add support for transmuxing into alternative backwards compatible
formats.
* Extractors:
* DataSource:
* `DataSourceContractTest`: Assert that `DataSource.getUri()` and

96
libraries/test_data/src/test/assets/transformerdumps/mp4/dolbyVision-hdr.MOV/transmuxed_with_inappmuxer.dump Normal file
View File

@ -0,0 +1,96 @@
seekMap:
isSeekable = true
duration = 166700
getPosition(0) = [[timeUs=0, position=400052]]
getPosition(1) = [[timeUs=0, position=400052]]
getPosition(83350) = [[timeUs=0, position=400052]]
getPosition(166700) = [[timeUs=0, position=400052]]
numberOfTracks = 2
track 0:
total output bytes = 56045
sample count = 5
track duration = 166700
format 0:
id = 1
sampleMimeType = video/hevc
codecs = hvc1.2.4.L93.B0
maxInputSize = 30755
maxNumReorderSamples = 2
width = 1280
height = 720
frameRate = 30.00
rotationDegrees = 90
colorInfo:
colorSpace = 6
colorRange = 2
colorTransfer = 7
lumaBitdepth = 10
chromaBitdepth = 10
metadata = entries=[mdta: key=com.apple.quicktime.location.accuracy.horizontal, value=35.000000, mdta: key=com.apple.quicktime.location.ISO6709, value=+51.5334-000.1255+026.756/, mdta: key=com.apple.quicktime.software, value=16.4.1, mdta: key=com.apple.quicktime.creationdate, value=2023-05-09T12:41:46+0100, mdta: key=com.apple.quicktime.make, value=Apple, mdta: key=com.apple.quicktime.model, value=iPhone 14 Pro, Mp4Timestamp: creation time=3000000000, modification time=4000000000, timescale=10000]
initializationData:
data = length 97, hash 71D67547
sample 0:
time = 0
flags = 1
data = length 30725, hash A9F08085
sample 1:
time = 133333
flags = 0
data = length 11543, hash D0A79439
sample 2:
time = 66666
flags = 0
data = length 6139, hash 3FCEAFBE
sample 3:
time = 33333
flags = 0
data = length 3660, hash A80DE201
sample 4:
time = 100000
flags = 536870912
data = length 3978, hash 59521A2C
track 1:
total output bytes = 2674
sample count = 7
track duration = 162600
format 0:
peakBitrate = 192000
id = 2
sampleMimeType = audio/mp4a-latm
codecs = mp4a.40.2
maxInputSize = 616
channelCount = 2
sampleRate = 44100
language = und
metadata = entries=[mdta: key=com.apple.quicktime.location.accuracy.horizontal, value=35.000000, mdta: key=com.apple.quicktime.location.ISO6709, value=+51.5334-000.1255+026.756/, mdta: key=com.apple.quicktime.software, value=16.4.1, mdta: key=com.apple.quicktime.creationdate, value=2023-05-09T12:41:46+0100, mdta: key=com.apple.quicktime.make, value=Apple, mdta: key=com.apple.quicktime.model, value=iPhone 14 Pro, Mp4Timestamp: creation time=3000000000, modification time=4000000000, timescale=10000]
initializationData:
data = length 2, hash 5FF
sample 0:
time = 0
flags = 1
data = length 6, hash 6EFB71F1
sample 1:
time = 23229
flags = 1
data = length 260, hash EE23C4F4
sample 2:
time = 46458
flags = 1
data = length 458, hash C54D6937
sample 3:
time = 69687
flags = 1
data = length 448, hash 319412F3
sample 4:
time = 92916
flags = 1
data = length 422, hash 1E293713
sample 5:
time = 116145
flags = 1
data = length 586, hash C4CD7E3D
sample 6:
time = 139375
flags = 536870913
data = length 494, hash A7AFD1B8
tracksEnded = true

55
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -25,6 +25,7 @@ import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.JPG_PIXEL_MOTION_PHOTO_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP3_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S;
@ -1773,6 +1774,60 @@ public class TransformerEndToEndTest {
assertThat(new File(result.filePath).length()).isGreaterThan(0);
}
@Test
public void dolbyVisionVideo_noEffects_transmuxesToHevc() throws Exception {
assumeTrue("This test requires B-frame support", Util.SDK_INT > 24);
assumeTrue(
new DefaultMuxer.Factory()
.getSupportedSampleMimeTypes(C.TRACK_TYPE_VIDEO)
.contains(MimeTypes.VIDEO_H265));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_DOLBY_VISION_HDR.uri)))
.setRemoveAudio(true)
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(
context,
new Transformer.Builder(context).setVideoMimeType(MimeTypes.VIDEO_H265).build())
.build()
.run(testId, editedMediaItem);
MediaExtractorCompat mediaExtractor = new MediaExtractorCompat(context);
mediaExtractor.setDataSource(Uri.parse(result.filePath), /* offset= */ 0);
checkState(mediaExtractor.getTrackCount() == 1);
MediaFormat mediaFormat = mediaExtractor.getTrackFormat(/* trackIndex= */ 0);
Format format = createFormatFromMediaFormat(mediaFormat);
assertThat(format.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H265);
assertThat(result.exportResult.videoConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED);
}
@Test
public void dolbyVisionVideo_noEffects_withInAppMuxer_transmuxesToHevc() throws Exception {
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_DOLBY_VISION_HDR.uri)))
.setRemoveAudio(true)
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(
context,
new Transformer.Builder(context)
.setVideoMimeType(MimeTypes.VIDEO_H265)
.setMuxerFactory(new InAppMuxer.Factory.Builder().build())
.build())
.build()
.run(testId, editedMediaItem);
MediaExtractorCompat mediaExtractor = new MediaExtractorCompat(context);
mediaExtractor.setDataSource(Uri.parse(result.filePath), /* offset= */ 0);
checkState(mediaExtractor.getTrackCount() == 1);
MediaFormat mediaFormat = mediaExtractor.getTrackFormat(/* trackIndex= */ 0);
Format format = createFormatFromMediaFormat(mediaFormat);
assertThat(format.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H265);
assertThat(result.exportResult.videoConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED);
}
@Test
public void audioComposition_compositionEffects_transcodes() throws Exception {
EditedMediaItem editedMediaItem =

8
libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java
View File

@ -19,6 +19,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.ColorInfo.isTransferHdr;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.exoplayer.mediacodec.MediaCodecUtil.getAlternativeCodecMimeType;
import static androidx.media3.transformer.EncoderUtil.getSupportedEncoders;
import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing;
import static androidx.media3.transformer.TransformerUtil.getProcessedTrackType;
@ -109,6 +110,13 @@ import java.util.List;
if (metadata != null) {
inputFormat = inputFormat.buildUpon().setMetadata(metadata).build();
}
if (!muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
String alternativeSampleMimeType = getAlternativeCodecMimeType(inputFormat);
if (muxerWrapper.supportsSampleMimeType(alternativeSampleMimeType)) {
inputFormat =
inputFormat.buildUpon().setSampleMimeType(alternativeSampleMimeType).build();
}
}
try {
muxerWrapper.addTrackFormat(inputFormat);
} catch (MuxerException e) {

15
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java
View File

@ -19,6 +19,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED;
import static androidx.media3.common.ColorInfo.isTransferHdr;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.exoplayer.mediacodec.MediaCodecUtil.getAlternativeCodecMimeType;
import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR;
import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL;
import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing;
@ -153,12 +154,18 @@ public final class TransformerUtil {
if (transformationRequest.hdrMode != HDR_MODE_KEEP_HDR) {
return true;
}
if (transformationRequest.videoMimeType != null
&& !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) {
@Nullable String requestedMimeType = transformationRequest.videoMimeType;
if (requestedMimeType != null) {
boolean requestedMimeTypeEqualsPrimaryOrAlternativeMimeType =
requestedMimeType.equals(inputFormat.sampleMimeType)
|| requestedMimeType.equals(getAlternativeCodecMimeType(inputFormat));
if (!requestedMimeTypeEqualsPrimaryOrAlternativeMimeType) {
return true;
}
if (transformationRequest.videoMimeType == null
&& !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
}
if (requestedMimeType == null
&& !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)
&& !muxerWrapper.supportsSampleMimeType(getAlternativeCodecMimeType(inputFormat))) {
return true;
}
if (inputFormat.pixelWidthHeightRatio != 1f) {

4
libraries/transformer/src/test/java/androidx/media3/transformer/TransformerWithInAppMuxerEndToEndParameterizedTest.java
View File

@ -53,6 +53,7 @@ public class TransformerWithInAppMuxerEndToEndParameterizedTest {
private static final String AMR_WB_3GP = "mp4/bbb_mono_16kHz_23.05kbps_amrwb.3gp";
private static final String OPUS_OGG = "mp4/bbb_6ch_8kHz_opus.ogg";
private static final String VORBIS_OGG = "mp4/bbb_1ch_16kHz_q10_vorbis.ogg";
private static final String DOLBY_VISION_MOV = "mp4/dolbyVision-hdr.MOV";
@Parameters(name = "{0}")
public static ImmutableList<String> mediaFiles() {
@ -65,7 +66,8 @@ public class TransformerWithInAppMuxerEndToEndParameterizedTest {
AMR_NB_3GP,
AMR_WB_3GP,
OPUS_OGG,
VORBIS_OGG);
VORBIS_OGG,
DOLBY_VISION_MOV);
}
@Parameter public @MonotonicNonNull String inputFile;