From 07ff48165a8042dbdadc7ada6778a395e5a98452 Mon Sep 17 00:00:00 2001 From: kimvde Date: Wed, 8 Feb 2023 17:13:20 +0000 Subject: [PATCH] Handle when some MediaItems require transcoding but others don't - For single-asset, the behavior stays the same. Transcode if and only if it's necessary, - For constrained multi-asset, always transcode, except if the setter to transmux is set. This is to avoid failing if a MediaItem that doesn't require transcoding is followed by a MediaItem that does require transcoding. PiperOrigin-RevId: 508097798 --- .../exoplayer2/transformer/Transformer.java | 29 ++++++++++- .../transformer/TransformerInternal.java | 24 +++++++--- .../transformer/TransformerEndToEndTest.java | 48 ++++++++++++++++++- 3 files changed, 93 insertions(+), 8 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 2947d9f160..0526d002db 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -83,6 +83,7 @@ public final class Transformer { private boolean removeAudio; private boolean removeVideo; private boolean flattenForSlowMotion; + private boolean transmux; private boolean generateSilentAudio; private ListenerSet listeners; private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory; @@ -204,6 +205,28 @@ public final class Transformer { return this; } + /** + * Sets whether to transmux the {@linkplain MediaItem media items} in the input {@link + * Composition}. + * + *

The default value is {@code false}. + * + *

If the input {@link Composition} contains one {@link MediaItem}, the value set is ignored. + * The {@link MediaItem} will only be transcoded if necessary. + * + *

If the input {@link Composition} contains multiple {@linkplain MediaItem media items}, + * they are all transmuxed if {@code transmux} is {@code true} and exporting the first {@link + * MediaItem} doesn't require transcoding. Otherwise, they are all transcoded. + * + * @param transmux Whether to transmux. + * @return This builder. + */ + @CanIgnoreReturnValue + public Builder setTransmux(boolean transmux) { + this.transmux = transmux; + return this; + } + /** * @deprecated Use {@link #addListener(Listener)}, {@link #removeListener(Listener)} or {@link * #removeAllListeners()} instead. @@ -421,6 +444,7 @@ public final class Transformer { removeAudio, removeVideo, flattenForSlowMotion, + transmux, generateSilentAudio, listeners, assetLoaderFactory, @@ -578,6 +602,7 @@ public final class Transformer { private final boolean removeAudio; private final boolean removeVideo; private final boolean flattenForSlowMotion; + private final boolean transmux; private final boolean generateSilentAudio; private final ListenerSet listeners; private final AssetLoader.Factory assetLoaderFactory; @@ -598,6 +623,7 @@ public final class Transformer { boolean removeAudio, boolean removeVideo, boolean flattenForSlowMotion, + boolean transmux, boolean generateSilentAudio, ListenerSet listeners, AssetLoader.Factory assetLoaderFactory, @@ -615,6 +641,7 @@ public final class Transformer { this.removeAudio = removeAudio; this.removeVideo = removeVideo; this.flattenForSlowMotion = flattenForSlowMotion; + this.transmux = transmux; this.generateSilentAudio = generateSilentAudio; this.listeners = listeners; this.assetLoaderFactory = assetLoaderFactory; @@ -692,7 +719,6 @@ public final class Transformer { * flattening}). *

  • have identical {@link EditedMediaItem#effects Effects} applied. *
  • not represent an image. - *
  • be such that either all or none requires transcoding. * * * @@ -735,6 +761,7 @@ public final class Transformer { composition, path, transformationRequest, + transmux, generateSilentAudio, assetLoaderFactory, encoderFactory, diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java index 254161fa6c..1709b72ff7 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java @@ -112,6 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Composition composition, String outputPath, TransformationRequest transformationRequest, + boolean transmux, boolean generateSilentAudio, AssetLoader.Factory assetLoaderFactory, Codec.EncoderFactory encoderFactory, @@ -133,7 +134,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; internalHandlerThread.start(); Looper internalLooper = internalHandlerThread.getLooper(); EditedMediaItemSequence sequence = composition.sequences.get(0); - ComponentListener componentListener = new ComponentListener(sequence, fallbackListener); + ComponentListener componentListener = + new ComponentListener(sequence, transmux, fallbackListener); compositeAssetLoader = new CompositeAssetLoader( sequence, assetLoaderFactory, internalLooper, componentListener, clock); @@ -314,6 +316,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // The first EditedMediaItem in the sequence determines which SamplePipeline to use. private final EditedMediaItem firstEditedMediaItem; + private final int mediaItemCount; + private final boolean transmux; private final FallbackListener fallbackListener; private final AtomicInteger trackCount; @@ -321,8 +325,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private volatile long durationUs; - public ComponentListener(EditedMediaItemSequence sequence, FallbackListener fallbackListener) { + public ComponentListener( + EditedMediaItemSequence sequence, boolean transmux, FallbackListener fallbackListener) { firstEditedMediaItem = sequence.editedMediaItems.get(0); + mediaItemCount = sequence.editedMediaItems.size(); + this.transmux = transmux; this.fallbackListener = fallbackListener; trackCount = new AtomicInteger(); durationUs = C.TIME_UNSET; @@ -457,10 +464,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throws TransformationException { checkState(supportedOutputTypes != 0); boolean isAudio = MimeTypes.isAudio(firstInputFormat.sampleMimeType); - boolean shouldTranscode = - isAudio - ? shouldTranscodeAudio(firstInputFormat) - : shouldTranscodeVideo(firstInputFormat, streamStartPositionUs, streamOffsetUs); + boolean shouldTranscode; + if (mediaItemCount > 1 && !transmux) { + shouldTranscode = true; + } else { + shouldTranscode = + isAudio + ? shouldTranscodeAudio(firstInputFormat) + : shouldTranscodeVideo(firstInputFormat, streamStartPositionUs, streamOffsetUs); + } boolean assetLoaderNeverDecodes = (supportedOutputTypes & SUPPORTED_OUTPUT_TYPE_DECODED) == 0; checkState(!shouldTranscode || !assetLoaderNeverDecodes); boolean assetLoaderAlwaysDecodes = diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java index 3ca9614e65..4c0fc36c56 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java @@ -379,7 +379,8 @@ public final class TransformerEndToEndTest { @Test public void startTransformation_concatenateMediaItemsWithSameFormat_completesSuccessfully() throws Exception { - Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build(); + Transformer transformer = + createTransformerBuilder(/* enableFallback= */ false).setTransmux(true).build(); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).setEffects(Effects.EMPTY).build(); @@ -423,6 +424,51 @@ public final class TransformerEndToEndTest { context, testMuxer, getDumpFileName(FILE_AUDIO_VIDEO + ".silence_skipped_concatenated")); } + @Test + public void startTransformation_singleMediaItemAndTransmux_ignoresTransmux() throws Exception { + SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); + sonicAudioProcessor.setOutputSampleRateHz(48000); + Transformer transformer = + createTransformerBuilder(/* enableFallback= */ false).setTransmux(true).build(); + MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); + ImmutableList audioProcessors = ImmutableList.of(sonicAudioProcessor); + Effects effects = new Effects(audioProcessors, /* videoEffects= */ ImmutableList.of()); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(mediaItem).setEffects(effects).build(); + + transformer.start(editedMediaItem, outputPath); + TransformerTestRunner.runLooper(transformer); + + DumpFileAsserts.assertOutput( + context, testMuxer, getDumpFileName(FILE_AUDIO_VIDEO + ".48000hz")); + } + + @Test + public void startTransformation_multipleMediaItemsWithEffectsAndTransmux_ignoresTransmux() + throws Exception { + Transformer transformer = + createTransformerBuilder(/* enableFallback= */ false).setTransmux(true).build(); + MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); + AudioProcessor audioProcessor = new SilenceSkippingAudioProcessor(); + Effects effects = + new Effects(ImmutableList.of(audioProcessor), /* videoEffects= */ ImmutableList.of()); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(mediaItem).setEffects(effects).setRemoveVideo(true).build(); + EditedMediaItemSequence editedMediaItemSequence = + new EditedMediaItemSequence(ImmutableList.of(editedMediaItem, editedMediaItem)); + Composition composition = + new Composition(ImmutableList.of(editedMediaItemSequence), Effects.EMPTY); + + transformer.start(composition, outputPath); + TransformerTestRunner.runLooper(transformer); + + // The inputs should be transcoded even though transmuxing has been requested. This is because + // audio effects have been added to the first MediaItem in the sequence, so the transcoding + // audio sample pipeline should be picked to apply these effects. + DumpFileAsserts.assertOutput( + context, testMuxer, getDumpFileName(FILE_AUDIO_VIDEO + ".silence_skipped_concatenated")); + } + @Test public void startTransformation_withMultipleListeners_callsEachOnCompletion() throws Exception { Transformer.Listener mockListener1 = mock(Transformer.Listener.class);