From 71fb4f9a5cc5a1e62ad68cf64d9446d9b44edc66 Mon Sep 17 00:00:00 2001 From: kimvde Date: Fri, 3 Mar 2023 14:12:51 +0000 Subject: [PATCH] Make sure that tracks are transcoded/transmuxed consistently All audio tracks should either all be transcoded or all be transmuxed. Same for video tracks. To achieve this, simplify the behaviour of transmuxAudio/Video. PiperOrigin-RevId: 513809287 --- .../androidx/media3/transformer/Composition.java | 14 ++++++++------ .../media3/transformer/TransformerInternal.java | 9 ++++----- .../transformer/TransformerEndToEndTest.java | 15 ++++++--------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java index c2164b6915..5d45bea420 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java @@ -122,9 +122,10 @@ public final class Composition { * audio track will only be transcoded if necessary. * *

If the input {@link Composition} contains multiple {@linkplain MediaItem media items}, all - * the audio tracks are transmuxed if {@code transmuxAudio} is {@code true} and exporting the - * first {@link MediaItem} doesn't require audio transcoding. Otherwise, they are all - * transcoded. Transmuxed tracks must be compatible and must not overlap in time. + * the audio tracks are transcoded by default. They are all transmuxed if {@code transmuxAudio} + * is {@code true}. Transmuxed tracks must be compatible (typically, all the {@link MediaItem} + * instances containing the track to transmux are concatenated in a single {@link + * EditedMediaItemSequence} and have the same sample format for that track). * *

Requesting audio transmuxing and {@linkplain #experimentalSetForceAudioTrack(boolean) * forcing an audio track} are not allowed together because generating silence requires @@ -148,9 +149,10 @@ public final class Composition { * video track will only be transcoded if necessary. * *

If the input {@link Composition} contains multiple {@linkplain MediaItem media items}, all - * the video tracks are transmuxed if {@code transmuxVideo} is {@code true} and exporting the - * first {@link MediaItem} doesn't require video transcoding. Otherwise, they are all - * transcoded. Transmuxed tracks must be compatible and must not overlap in time. + * the video tracks are transcoded by default. They are all transmuxed if {@code transmuxVideo} + * is {@code true}. Transmuxed tracks must be compatible (typically, all the {@link MediaItem} + * instances containing the track to transmux are concatenated in a single {@link + * EditedMediaItemSequence} and have the same sample format for that track). * * @param transmuxVideo Whether to transmux the video tracks. * @return This builder. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index eb92c8d064..1944233d64 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -582,8 +582,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private boolean shouldTranscodeAudio(Format inputFormat) { - if (editedMediaItems.size() > 1 && !composition.transmuxAudio) { - return true; + if (composition.sequences.size() > 1 || editedMediaItems.size() > 1) { + return !composition.transmuxAudio; } if (encoderFactory.audioNeedsEncoding()) { return true; @@ -603,14 +603,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; if (!firstEditedMediaItem.effects.audioProcessors.isEmpty()) { return true; } - return false; } private boolean shouldTranscodeVideo( Format inputFormat, long streamStartPositionUs, long streamOffsetUs) { - if (editedMediaItems.size() > 1 && !composition.transmuxVideo) { - return true; + if (composition.sequences.size() > 1 || editedMediaItems.size() > 1) { + return !composition.transmuxVideo; } EditedMediaItem firstEditedMediaItem = editedMediaItems.get(0); if ((streamStartPositionUs - streamOffsetUs) != 0 diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java index 9dada9986a..055074360d 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -50,6 +50,7 @@ import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.audio.SonicAudioProcessor; import androidx.media3.common.util.Util; import androidx.media3.effect.Presentation; +import androidx.media3.effect.RgbFilter; import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.exoplayer.source.MediaSource; @@ -516,15 +517,16 @@ public final class TransformerEndToEndTest { } @Test - public void start_multipleMediaItemsWithEffectsAndTransmux_ignoresTransmux() throws Exception { + public void start_multipleMediaItemsAndTransmux_transmux() throws Exception { Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build(); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); sonicAudioProcessor.setPitch(2f); + Effect videoEffect = RgbFilter.createGrayscaleFilter(); Effects effects = - new Effects(ImmutableList.of(sonicAudioProcessor), /* videoEffects= */ ImmutableList.of()); + new Effects(ImmutableList.of(sonicAudioProcessor), ImmutableList.of(videoEffect)); EditedMediaItem editedMediaItem = - new EditedMediaItem.Builder(mediaItem).setEffects(effects).setRemoveVideo(true).build(); + new EditedMediaItem.Builder(mediaItem).setEffects(effects).build(); EditedMediaItemSequence editedMediaItemSequence = new EditedMediaItemSequence(ImmutableList.of(editedMediaItem, editedMediaItem)); Composition composition = @@ -536,13 +538,8 @@ public final class TransformerEndToEndTest { transformer.start(composition, outputPath); TransformerTestRunner.runLooper(transformer); - // The inputs should be transcoded even though transmuxing has been requested. This is because - // audio effects have been added to the first MediaItem in the sequence, so the transcoding - // audio sample pipeline should be picked to apply these effects. DumpFileAsserts.assertOutput( - context, - testMuxer, - getDumpFileName(FILE_AUDIO_VIDEO + ".concatenated_with_high_pitch_and_no_video")); + context, testMuxer, getDumpFileName(FILE_AUDIO_VIDEO + ".concatenated")); } @Test