loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split transmux into transmuxAudio/Video

Browse Source 
- Split the transmux setting into transmuxAudio and transmuxVideo. This
  is more flexible for apps and will also be useful for unit testing
  (particularly as we can't test video transcoding on Robolectric at the
  moment).
- Move these settings to Composition. It makes sense for these settings
  to be next to forceAudioTrack. Apps may also want to set these
  settings based on the current Composition's MediaItems.
- Add a Composition.Builder because Composition now contains a few
  optional fields.

PiperOrigin-RevId: 511708618
This commit is contained in:
kimvde 2023-02-23 08:21:44 +00:00 committed by tonihei
parent 2baa70206c
commit 31c44c7061
8 changed files with 222 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -378,7 +378,9 @@ public final class TransformerActivity extends AppCompatActivity {
editedMediaItems.add(editedMediaItemBuilder.build());
List<EditedMediaItemSequence> sequences = new ArrayList<>();
sequences.add(new EditedMediaItemSequence(editedMediaItems));
return new Composition(sequences, Effects.EMPTY, forceAudioTrack);
return new Composition.Builder(sequences)
.experimentalSetForceAudioTrack(forceAudioTrack)
.build();
}
private ImmutableList<AudioProcessor> createAudioProcessorsFromBundle(Bundle bundle) {

172
libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java
View File

@ -21,28 +21,61 @@ import androidx.media3.common.MediaItem;
import androidx.media3.common.audio.AudioProcessor;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.util.List;
/**
* A composition of {@link MediaItem} instances, with transformations to apply to them.
*
* <p>The {@linkplain MediaItem} instances can be concatenated or mixed. {@link Effects} can be
* applied to individual {@linkplain MediaItem} instances, as well as to the composition.
* <p>The {@link MediaItem} instances can be concatenated or mixed. {@link Effects} can be applied
* to individual {@link MediaItem} instances, as well as to the composition.
*/
@UnstableApi
public final class Composition {
/** A builder for {@link Composition} instances. */
public static final class Builder {
private final ImmutableList<EditedMediaItemSequence> sequences;
private Effects effects;
private boolean forceAudioTrack;
private boolean transmuxAudio;
private boolean transmuxVideo;
/**
* The {@link EditedMediaItemSequence} instances to compose. {@link MediaItem} instances from
* different sequences that are overlapping in time will be mixed in the output.
* Creates an instance.
*
* <p>This list must not be empty.
* @param sequences The {@link EditedMediaItemSequence} instances to compose. {@link MediaItem}
* instances from different sequences that are overlapping in time will be mixed in the
* output. This list must not be empty.
*/
public final ImmutableList<EditedMediaItemSequence> sequences;
/** The {@link Effects} to apply to the composition. */
public final Effects effects;
public Builder(List<EditedMediaItemSequence> sequences) {
checkArgument(
!sequences.isEmpty(),
"The composition must contain at least one EditedMediaItemSequence.");
this.sequences = ImmutableList.copyOf(sequences);
effects = Effects.EMPTY;
}
/**
* Whether the output file should always contain an audio track.
* Sets the {@link Effects} to apply to the {@link Composition}.
*
* <p>The default value is {@link Effects#EMPTY}.
*
* @param effects The {@link Composition} {@link Effects}.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setEffects(Effects effects) {
this.effects = effects;
return this;
}
/**
* Sets whether the output file should always contain an audio track.
*
* <p>The default value is {@code false}.
*
* <ul>
* <li>If {@code false}:
@ -60,38 +93,117 @@ public final class Composition {
* the {@link Composition} export doesn't produce any audio.
*
* <p>The MIME type of the output's audio track can be set using {@link
* TransformationRequest.Builder#setAudioMimeType(String)}. The sample rate and channel count can
* be set by passing relevant {@link AudioProcessor} instances to the {@link Composition}.
* TransformationRequest.Builder#setAudioMimeType(String)}. The sample rate and channel count
* can be set by passing relevant {@link AudioProcessor} instances to the {@link Composition}.
*
* <p>This parameter is experimental and may be removed or changed without warning.
*/
public final boolean experimentalForceAudioTrack;
/**
* Creates an instance.
* <p>Forcing an audio track and {@linkplain #setTransmuxAudio(boolean) requesting audio
* transmuxing} are not allowed together because generating silence requires transcoding.
*
* <p>This is equivalent to calling {@link Composition#Composition(List, Effects, boolean)} with
* {@link #experimentalForceAudioTrack} set to {@code false}.
* <p>This method is experimental and may be removed or changed without warning.
*
* @param forceAudioTrack Whether to force an audio track in the output.
* @return This builder.
*/
public Composition(List<EditedMediaItemSequence> sequences, Effects effects) {
this(sequences, effects, /* experimentalForceAudioTrack= */ false);
@CanIgnoreReturnValue
public Builder experimentalSetForceAudioTrack(boolean forceAudioTrack) {
this.forceAudioTrack = forceAudioTrack;
return this;
}
/**
* Creates an instance.
* Sets whether to transmux the {@linkplain MediaItem media items'} audio tracks.
*
* @param sequences The {@link #sequences}.
* @param effects The {@link #effects}.
* @param experimentalForceAudioTrack Whether to {@linkplain #experimentalForceAudioTrack always
* add an audio track in the output}.
* <p>The default value is {@code false}.
*
* <p>If the {@link Composition} contains one {@link MediaItem}, the value set is ignored. The
* audio track will only be transcoded if necessary.
*
* <p>If the input {@link Composition} contains multiple {@linkplain MediaItem media items}, all
* the audio tracks are transmuxed if {@code transmuxAudio} is {@code true} and exporting the
* first {@link MediaItem} doesn't require audio transcoding. Otherwise, they are all
* transcoded. Transmuxed tracks must be compatible and must not overlap in time.
*
* <p>Requesting audio transmuxing and {@linkplain #experimentalSetForceAudioTrack(boolean)
* forcing an audio track} are not allowed together because generating silence requires
* transcoding.
*
* @param transmuxAudio Whether to transmux the audio tracks.
* @return This builder.
*/
public Composition(
@CanIgnoreReturnValue
public Builder setTransmuxAudio(boolean transmuxAudio) {
this.transmuxAudio = transmuxAudio;
return this;
}
/**
* Sets whether to transmux the {@linkplain MediaItem media items'} video tracks.
*
* <p>The default value is {@code false}.
*
* <p>If the {@link Composition} contains one {@link MediaItem}, the value set is ignored. The
* video track will only be transcoded if necessary.
*
* <p>If the input {@link Composition} contains multiple {@linkplain MediaItem media items}, all
* the video tracks are transmuxed if {@code transmuxVideo} is {@code true} and exporting the
* first {@link MediaItem} doesn't require video transcoding. Otherwise, they are all
* transcoded. Transmuxed tracks must be compatible and must not overlap in time.
*
* @param transmuxVideo Whether to transmux the video tracks.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setTransmuxVideo(boolean transmuxVideo) {
this.transmuxVideo = transmuxVideo;
return this;
}
/** Builds a {@link Composition} instance. */
public Composition build() {
return new Composition(sequences, effects, forceAudioTrack, transmuxAudio, transmuxVideo);
}
}
/**
* The {@link EditedMediaItemSequence} instances to compose.
*
* <p>For more information, see {@link Builder#Builder(List)}.
*/
public final ImmutableList<EditedMediaItemSequence> sequences;
/** The {@link Effects} to apply to the composition. */
public final Effects effects;
/**
* Whether the output file should always contain an audio track.
*
* <p>For more information, see {@link Builder#experimentalSetForceAudioTrack(boolean)}.
*/
public final boolean forceAudioTrack;
/**
* Whether to transmux the {@linkplain MediaItem media items'} audio tracks.
*
* <p>For more information, see {@link Builder#setTransmuxAudio(boolean)}.
*/
public final boolean transmuxAudio;
/**
* Whether to transmux the {@linkplain MediaItem media items'} video tracks.
*
* <p>For more information, see {@link Builder#setTransmuxVideo(boolean)}.
*/
public final boolean transmuxVideo;
private Composition(
List<EditedMediaItemSequence> sequences,
Effects effects,
boolean experimentalForceAudioTrack) {
checkArgument(!sequences.isEmpty());
boolean forceAudioTrack,
boolean transmuxAudio,
boolean transmuxVideo) {
checkArgument(
!transmuxAudio || !forceAudioTrack,
"Audio transmuxing and audio track forcing are not allowed together.");
this.sequences = ImmutableList.copyOf(sequences);
this.effects = effects;
this.experimentalForceAudioTrack = experimentalForceAudioTrack;
this.transmuxAudio = transmuxAudio;
this.transmuxVideo = transmuxVideo;
this.forceAudioTrack = forceAudioTrack;
}
}

2
libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java
View File

@ -24,7 +24,7 @@ import androidx.media3.effect.DefaultVideoFrameProcessor;
import com.google.common.collect.ImmutableList;
import java.util.List;
/** Effects to apply to a {@link MediaItem}. */
/** Effects to apply to a {@link MediaItem} or to a {@link Composition}. */
@UnstableApi
public final class Effects {

30
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -84,7 +84,6 @@ public final class Transformer {
private boolean removeAudio;
private boolean removeVideo;
private boolean flattenForSlowMotion;
private boolean transmux;
private ListenerSet<Transformer.Listener> listeners;
private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory;
private VideoFrameProcessor.Factory videoFrameProcessorFactory;
@ -204,28 +203,6 @@ public final class Transformer {
return this;
}
/**
* Sets whether to transmux the {@linkplain MediaItem media items} in the input {@link
* Composition}.
*
* <p>The default value is {@code false}.
*
* <p>If the input {@link Composition} contains one {@link MediaItem}, the value set is ignored.
* The {@link MediaItem} will only be transcoded if necessary.
*
* <p>If the input {@link Composition} contains multiple {@linkplain MediaItem media items},
* they are all transmuxed if {@code transmux} is {@code true} and exporting the first {@link
* MediaItem} doesn't require transcoding. Otherwise, they are all transcoded.
*
* @param transmux Whether to transmux.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setTransmux(boolean transmux) {
this.transmux = transmux;
return this;
}
/**
* @deprecated Use {@link #addListener(Listener)}, {@link #removeListener(Listener)} or {@link
* #removeAllListeners()} instead.
@ -416,7 +393,6 @@ public final class Transformer {
removeAudio,
removeVideo,
flattenForSlowMotion,
transmux,
listeners,
assetLoaderFactory,
videoFrameProcessorFactory,
@ -576,7 +552,6 @@ public final class Transformer {
private final boolean removeAudio;
private final boolean removeVideo;
private final boolean flattenForSlowMotion;
private final boolean transmux;
private final ListenerSet<Transformer.Listener> listeners;
private final AssetLoader.Factory assetLoaderFactory;
private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
@ -596,7 +571,6 @@ public final class Transformer {
boolean removeAudio,
boolean removeVideo,
boolean flattenForSlowMotion,
boolean transmux,
ListenerSet<Listener> listeners,
AssetLoader.Factory assetLoaderFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
@ -613,7 +587,6 @@ public final class Transformer {
this.removeAudio = removeAudio;
this.removeVideo = removeVideo;
this.flattenForSlowMotion = flattenForSlowMotion;
this.transmux = transmux;
this.listeners = listeners;
this.assetLoaderFactory = assetLoaderFactory;
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
@ -732,7 +705,6 @@ public final class Transformer {
composition,
path,
transformationRequest,
transmux,
assetLoaderFactory,
encoderFactory,
muxerFactory,
@ -772,7 +744,7 @@ public final class Transformer {
public void start(EditedMediaItem editedMediaItem, String path) {
EditedMediaItemSequence sequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
start(new Composition(ImmutableList.of(sequence), Effects.EMPTY), path);
start(new Composition.Builder(ImmutableList.of(sequence)).build(), path);
}
/**

26
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -112,7 +112,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Composition composition,
String outputPath,
TransformationRequest transformationRequest,
boolean transmux,
AssetLoader.Factory assetLoaderFactory,
Codec.EncoderFactory encoderFactory,
Muxer.Factory muxerFactory,
@ -133,11 +132,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Looper internalLooper = internalHandlerThread.getLooper();
EditedMediaItemSequence sequence = composition.sequences.get(0);
ComponentListener componentListener =
new ComponentListener(sequence, transmux, fallbackListener);
new ComponentListener(
sequence, composition.transmuxAudio, composition.transmuxVideo, fallbackListener);
compositeAssetLoader =
new CompositeAssetLoader(
sequence,
composition.experimentalForceAudioTrack,
composition.forceAudioTrack,
assetLoaderFactory,
internalLooper,
componentListener,
@ -314,17 +314,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// The first EditedMediaItem in the sequence determines which SamplePipeline to use.
private final EditedMediaItem firstEditedMediaItem;
private final int mediaItemCount;
private final boolean transmux;
private final boolean transmuxAudio;
private final boolean transmuxVideo;
private final FallbackListener fallbackListener;
private final AtomicInteger trackCount;
private boolean trackAdded;
public ComponentListener(
EditedMediaItemSequence sequence, boolean transmux, FallbackListener fallbackListener) {
EditedMediaItemSequence sequence,
boolean transmuxAudio,
boolean transmuxVideo,
FallbackListener fallbackListener) {
firstEditedMediaItem = sequence.editedMediaItems.get(0);
mediaItemCount = sequence.editedMediaItems.size();
this.transmux = transmux;
this.transmuxAudio = transmuxAudio;
this.transmuxVideo = transmuxVideo;
this.fallbackListener = fallbackListener;
trackCount = new AtomicInteger();
}
@ -479,12 +484,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
boolean shouldTranscode = false;
if (!assetLoaderCanOutputEncoded) {
shouldTranscode = true;
} else if (mediaItemCount > 1 && !transmux) {
shouldTranscode = true;
} else if (MimeTypes.isAudio(inputFormat.sampleMimeType)) {
shouldTranscode = shouldTranscodeAudio(inputFormat);
shouldTranscode =
(mediaItemCount > 1 && !transmuxAudio) || shouldTranscodeAudio(inputFormat);
} else if (MimeTypes.isVideo(inputFormat.sampleMimeType)) {
shouldTranscode = shouldTranscodeVideo(inputFormat, streamStartPositionUs, streamOffsetUs);
shouldTranscode =
(mediaItemCount > 1 && !transmuxVideo)
|| shouldTranscodeVideo(inputFormat, streamStartPositionUs, streamOffsetUs);
}
checkState(!shouldTranscode || assetLoaderCanOutputDecoded);

6
libraries/transformer/src/test/java/androidx/media3/transformer/FallbackListenerTest.java
View File

@ -39,12 +39,12 @@ import org.robolectric.shadows.ShadowLooper;
public class FallbackListenerTest {
private static final Composition PLACEHOLDER_COMPOSITION =
new Composition(
new Composition.Builder(
ImmutableList.of(
new EditedMediaItemSequence(
ImmutableList.of(
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.EMPTY)).build()))),
Effects.EMPTY);
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.EMPTY)).build()))))
.build();
@Test
public void onTransformationRequestFinalized_withoutTrackCountSet_throwsException() {

54
libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -286,8 +286,9 @@ public final class TransformerEndToEndTest {
EditedMediaItemSequence sequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
Composition composition =
new Composition(
ImmutableList.of(sequence), Effects.EMPTY, /* experimentalForceAudioTrack= */ true);
new Composition.Builder(ImmutableList.of(sequence))
.experimentalSetForceAudioTrack(true)
.build();
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);
@ -304,8 +305,9 @@ public final class TransformerEndToEndTest {
EditedMediaItemSequence sequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
Composition composition =
new Composition(
ImmutableList.of(sequence), Effects.EMPTY, /* experimentalForceAudioTrack= */ true);
new Composition.Builder(ImmutableList.of(sequence))
.experimentalSetForceAudioTrack(true)
.build();
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);
@ -323,8 +325,9 @@ public final class TransformerEndToEndTest {
EditedMediaItemSequence sequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
Composition composition =
new Composition(
ImmutableList.of(sequence), Effects.EMPTY, /* experimentalForceAudioTrack= */ true);
new Composition.Builder(ImmutableList.of(sequence))
.experimentalSetForceAudioTrack(true)
.build();
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);
@ -343,8 +346,9 @@ public final class TransformerEndToEndTest {
EditedMediaItemSequence sequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
Composition composition =
new Composition(
ImmutableList.of(sequence), Effects.EMPTY, /* experimentalForceAudioTrack= */ true);
new Composition.Builder(ImmutableList.of(sequence))
.experimentalSetForceAudioTrack(true)
.build();
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);
@ -360,8 +364,9 @@ public final class TransformerEndToEndTest {
EditedMediaItemSequence sequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
Composition composition =
new Composition(
ImmutableList.of(sequence), Effects.EMPTY, /* experimentalForceAudioTrack= */ true);
new Composition.Builder(ImmutableList.of(sequence))
.experimentalSetForceAudioTrack(true)
.build();
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);
@ -390,15 +395,17 @@ public final class TransformerEndToEndTest {
@Test
public void start_concatenateMediaItemsWithSameFormat_completesSuccessfully() throws Exception {
Transformer transformer =
createTransformerBuilder(/* enableFallback= */ false).setTransmux(true).build();
Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(Effects.EMPTY).build();
EditedMediaItemSequence editedMediaItemSequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem, editedMediaItem));
Composition composition =
new Composition(ImmutableList.of(editedMediaItemSequence), Effects.EMPTY);
new Composition.Builder(ImmutableList.of(editedMediaItemSequence))
.setTransmuxAudio(true)
.setTransmuxVideo(true)
.build();
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);
@ -425,7 +432,7 @@ public final class TransformerEndToEndTest {
EditedMediaItemSequence editedMediaItemSequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem, editedMediaItem));
Composition composition =
new Composition(ImmutableList.of(editedMediaItemSequence), Effects.EMPTY);
new Composition.Builder(ImmutableList.of(editedMediaItemSequence)).build();
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);
@ -438,15 +445,20 @@ public final class TransformerEndToEndTest {
public void start_singleMediaItemAndTransmux_ignoresTransmux() throws Exception {
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48000);
Transformer transformer =
createTransformerBuilder(/* enableFallback= */ false).setTransmux(true).build();
Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
ImmutableList<AudioProcessor> audioProcessors = ImmutableList.of(sonicAudioProcessor);
Effects effects = new Effects(audioProcessors, /* videoEffects= */ ImmutableList.of());
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
EditedMediaItemSequence editedMediaItemSequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
Composition composition =
new Composition.Builder(ImmutableList.of(editedMediaItemSequence))
.setTransmuxAudio(true)
.build();
transformer.start(editedMediaItem, outputPath);
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);
DumpFileAsserts.assertOutput(
@ -455,8 +467,7 @@ public final class TransformerEndToEndTest {
@Test
public void start_multipleMediaItemsWithEffectsAndTransmux_ignoresTransmux() throws Exception {
Transformer transformer =
createTransformerBuilder(/* enableFallback= */ false).setTransmux(true).build();
Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
AudioProcessor audioProcessor = new SilenceSkippingAudioProcessor();
Effects effects =
@ -466,7 +477,10 @@ public final class TransformerEndToEndTest {
EditedMediaItemSequence editedMediaItemSequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem, editedMediaItem));
Composition composition =
new Composition(ImmutableList.of(editedMediaItemSequence), Effects.EMPTY);
new Composition.Builder(ImmutableList.of(editedMediaItemSequence))
.setTransmuxAudio(true)
.setTransmuxVideo(true)
.build();
transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer);

6
libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java
View File

@ -44,12 +44,12 @@ import org.robolectric.shadows.ShadowMediaCodecList;
@RunWith(AndroidJUnit4.class)
public final class VideoEncoderWrapperTest {
private static final Composition FAKE_COMPOSITION =
new Composition(
new Composition.Builder(
ImmutableList.of(
new EditedMediaItemSequence(
ImmutableList.of(
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.EMPTY)).build()))),
Effects.EMPTY);
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.EMPTY)).build()))))
.build();
private final TransformationRequest emptyTransformationRequest =
new TransformationRequest.Builder().build();