From c740b58efa2ef787b983b87a09ffe4d6fc84262c Mon Sep 17 00:00:00 2001 From: samrobinson Date: Fri, 11 Aug 2023 11:01:57 +0000 Subject: [PATCH] Add API for injecting AudioMixer.Factory. Rename AudioMixerImpl to DefaultAudioMixer. Removes the AudioMixerImpl specific getOutputAudioFormat, as the caller defines and sets this. PiperOrigin-RevId: 555887722 --- .../media3/transformer/AudioGraph.java | 15 ++++--- .../media3/transformer/AudioMixer.java | 15 +++++-- .../transformer/AudioSampleExporter.java | 3 +- ...oMixerImpl.java => DefaultAudioMixer.java} | 44 ++++++++++++++----- .../media3/transformer/Transformer.java | 28 +++++++++++- .../transformer/TransformerInternal.java | 6 +++ ...plTest.java => DefaultAudioMixerTest.java} | 6 +-- 7 files changed, 89 insertions(+), 28 deletions(-) rename libraries/transformer/src/main/java/androidx/media3/transformer/{AudioMixerImpl.java => DefaultAudioMixer.java} (91%) rename libraries/transformer/src/test/java/androidx/media3/transformer/{AudioMixerImplTest.java => DefaultAudioMixerTest.java} (99%) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java index 6b8a45007e..a2c17bfab4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java @@ -28,17 +28,19 @@ import java.nio.ByteBuffer; /** Processes raw audio samples. */ /* package */ final class AudioGraph { - private final AudioMixerImpl mixer; + private final AudioMixer mixer; private final SparseArray inputs; + private AudioFormat outputAudioFormat; private int finishedInputs; private ByteBuffer currentOutput; /** Creates an instance. */ - public AudioGraph() { - mixer = new AudioMixerImpl(/* outputSilenceWithNoSources= */ false); + public AudioGraph(AudioMixer.Factory mixerFactory) { + mixer = mixerFactory.create(); inputs = new SparseArray<>(); currentOutput = EMPTY_BUFFER; + outputAudioFormat = AudioFormat.NOT_SET; } /** Returns a new {@link AudioGraphInput} instance. */ @@ -48,10 +50,9 @@ import java.nio.ByteBuffer; AudioGraphInput audioGraphInput = new AudioGraphInput(item, format); if (inputs.size() == 0) { + outputAudioFormat = audioGraphInput.getOutputAudioFormat(); mixer.configure( - audioGraphInput.getOutputAudioFormat(), - /* bufferSizeMs= */ C.LENGTH_UNSET, - /* startTimeUs= */ 0); + outputAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0); } int sourceId = mixer.addSource(audioGraphInput.getOutputAudioFormat(), /* startTimeUs= */ 0); @@ -69,7 +70,7 @@ import java.nio.ByteBuffer; * registered}. */ public AudioFormat getOutputAudioFormat() { - return mixer.getOutputAudioFormat(); + return outputAudioFormat; } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixer.java index 5968c96c46..c0a979db10 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixer.java @@ -53,9 +53,18 @@ import java.nio.ByteBuffer; */ @UnstableApi public interface AudioMixer { - /** Creates an unconfigured instance. */ - public static AudioMixer create() { - return new AudioMixerImpl(/* outputSilenceWithNoSources= */ true); + + /** A factory for {@link AudioMixer} instances. */ + interface Factory { + AudioMixer create(); + } + + /** + * @deprecated Use {@link DefaultAudioMixer.Factory#create()}. + */ + @Deprecated + static AudioMixer create() { + return new DefaultAudioMixer.Factory(/* outputSilenceWithNoSources= */ true).create(); } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java index 7630e2611f..d80d888dcd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java @@ -52,12 +52,13 @@ import org.checkerframework.dataflow.qual.Pure; Format firstInputFormat, TransformationRequest transformationRequest, EditedMediaItem firstEditedMediaItem, + AudioMixer.Factory mixerFactory, Codec.EncoderFactory encoderFactory, MuxerWrapper muxerWrapper, FallbackListener fallbackListener) throws ExportException { super(firstAssetLoaderTrackFormat, muxerWrapper); - audioGraph = new AudioGraph(); + audioGraph = new AudioGraph(mixerFactory); this.firstInputFormat = firstInputFormat; firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat); encoderInputAudioFormat = audioGraph.getOutputAudioFormat(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAudioMixer.java similarity index 91% rename from libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java rename to libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAudioMixer.java index bd93db2a0b..6c19fe96ad 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAudioMixer.java @@ -28,12 +28,42 @@ import androidx.media3.common.audio.AudioMixingUtil; import androidx.media3.common.audio.AudioProcessor.AudioFormat; import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; import androidx.media3.common.audio.ChannelMixingMatrix; +import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import java.nio.ByteBuffer; import java.nio.ByteOrder; /** An {@link AudioMixer} that incrementally mixes source audio into a fixed size mixing buffer. */ -/* package */ final class AudioMixerImpl implements AudioMixer { +@UnstableApi +public final class DefaultAudioMixer implements AudioMixer { + + /** An {@link AudioMixer.Factory} implementation for {@link DefaultAudioMixer} instances. */ + public static final class Factory implements AudioMixer.Factory { + private final boolean outputSilenceWithNoSources; + + /** + * Creates an instance that does not {@linkplain #getOutput() output} silence when there are no + * {@linkplain #addSource sources}. + */ + public Factory() { + this(/* outputSilenceWithNoSources= */ false); + } + + /** + * Creates an instance. + * + * @param outputSilenceWithNoSources Whether to {@linkplain #getOutput() output} silence when + * there are no {@linkplain #addSource sources}. + */ + public Factory(boolean outputSilenceWithNoSources) { + this.outputSilenceWithNoSources = outputSilenceWithNoSources; + } + + @Override + public DefaultAudioMixer create() { + return new DefaultAudioMixer(outputSilenceWithNoSources); + } + } // TODO(b/290002438, b/276734854): Improve buffer management & determine best default size. private static final int DEFAULT_BUFFER_SIZE_MS = 500; @@ -60,13 +90,7 @@ import java.nio.ByteOrder; */ private long maxPositionOfRemovedSources; - /** - * Creates an instance. - * - * @param outputSilenceWithNoSources Whether {@link #getOutput()} should output buffers of silence - * when there are no {@link #addSource sources}. - */ - public AudioMixerImpl(boolean outputSilenceWithNoSources) { + private DefaultAudioMixer(boolean outputSilenceWithNoSources) { sources = new SparseArray<>(); outputAudioFormat = AudioFormat.NOT_SET; bufferSizeFrames = C.LENGTH_UNSET; @@ -79,10 +103,6 @@ import java.nio.ByteOrder; } } - public AudioFormat getOutputAudioFormat() { - return outputAudioFormat; - } - @Override public void configure(AudioFormat outputAudioFormat, int bufferSizeMs, long startTimeUs) throws UnhandledAudioFormatException { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 006f2772ea..e2d045915d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -94,6 +94,7 @@ public final class Transformer { private boolean flattenForSlowMotion; private ListenerSet listeners; private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory; + private AudioMixer.Factory audioMixerFactory; private VideoFrameProcessor.Factory videoFrameProcessorFactory; private Codec.EncoderFactory encoderFactory; private Muxer.Factory muxerFactory; @@ -110,6 +111,7 @@ public final class Transformer { this.context = context.getApplicationContext(); audioProcessors = ImmutableList.of(); videoEffects = ImmutableList.of(); + audioMixerFactory = new DefaultAudioMixer.Factory(); videoFrameProcessorFactory = new DefaultVideoFrameProcessor.Factory.Builder().build(); encoderFactory = new DefaultEncoderFactory.Builder(this.context).build(); muxerFactory = new DefaultMuxer.Factory(); @@ -131,6 +133,7 @@ public final class Transformer { this.removeVideo = transformer.removeVideo; this.listeners = transformer.listeners; this.assetLoaderFactory = transformer.assetLoaderFactory; + this.audioMixerFactory = transformer.audioMixerFactory; this.videoFrameProcessorFactory = transformer.videoFrameProcessorFactory; this.encoderFactory = transformer.encoderFactory; this.muxerFactory = transformer.muxerFactory; @@ -339,7 +342,23 @@ public final class Transformer { } /** - * Sets the factory to be used to create {@link VideoFrameProcessor} instances. + * Sets the {@link AudioMixer.Factory} to be used when {@linkplain AudioMixer audio mixing} is + * needed. + * + *

The default value is a {@link DefaultAudioMixer.Factory} with default values. + * + * @param audioMixerFactory A {@link AudioMixer.Factory}. + * @return This builder. + */ + @CanIgnoreReturnValue + public Builder setAudioMixerFactory(AudioMixer.Factory audioMixerFactory) { + this.audioMixerFactory = audioMixerFactory; + return this; + } + + /** + * Sets the {@link VideoFrameProcessor.Factory} to be used to create {@link VideoFrameProcessor} + * instances. * *

The default value is a {@link DefaultVideoFrameProcessor.Factory} built with default * values. @@ -369,7 +388,7 @@ public final class Transformer { } /** - * Sets the factory for muxers that write the media container. + * Sets the {@link Muxer.Factory} for muxers that write the media container. * *

The default value is a {@link DefaultMuxer.Factory}. * @@ -468,6 +487,7 @@ public final class Transformer { flattenForSlowMotion, listeners, assetLoaderFactory, + audioMixerFactory, videoFrameProcessorFactory, encoderFactory, muxerFactory, @@ -637,6 +657,7 @@ public final class Transformer { private final boolean flattenForSlowMotion; private final ListenerSet listeners; @Nullable private final AssetLoader.Factory assetLoaderFactory; + private final AudioMixer.Factory audioMixerFactory; private final VideoFrameProcessor.Factory videoFrameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Muxer.Factory muxerFactory; @@ -656,6 +677,7 @@ public final class Transformer { boolean flattenForSlowMotion, ListenerSet listeners, @Nullable AssetLoader.Factory assetLoaderFactory, + AudioMixer.Factory audioMixerFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory, Codec.EncoderFactory encoderFactory, Muxer.Factory muxerFactory, @@ -672,6 +694,7 @@ public final class Transformer { this.flattenForSlowMotion = flattenForSlowMotion; this.listeners = listeners; this.assetLoaderFactory = assetLoaderFactory; + this.audioMixerFactory = audioMixerFactory; this.videoFrameProcessorFactory = videoFrameProcessorFactory; this.encoderFactory = encoderFactory; this.muxerFactory = muxerFactory; @@ -839,6 +862,7 @@ public final class Transformer { path, transformationRequest, assetLoaderFactory, + audioMixerFactory, videoFrameProcessorFactory, encoderFactory, muxerFactory, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 696686a51a..7c37f52208 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -136,6 +136,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; String outputPath, TransformationRequest transformationRequest, AssetLoader.Factory assetLoaderFactory, + AudioMixer.Factory audioMixerFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory, Codec.EncoderFactory encoderFactory, Muxer.Factory muxerFactory, @@ -162,6 +163,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* sequenceIndex= */ i, composition, transformationRequest, + audioMixerFactory, videoFrameProcessorFactory, fallbackListener, debugViewProvider); @@ -449,6 +451,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final ImmutableList editedMediaItems; private final Composition composition; private final TransformationRequest transformationRequest; + private final AudioMixer.Factory audioMixerFactory; private final VideoFrameProcessor.Factory videoFrameProcessorFactory; private final FallbackListener fallbackListener; private final DebugViewProvider debugViewProvider; @@ -458,6 +461,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int sequenceIndex, Composition composition, TransformationRequest transformationRequest, + AudioMixer.Factory audioMixerFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory, FallbackListener fallbackListener, DebugViewProvider debugViewProvider) { @@ -465,6 +469,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.editedMediaItems = composition.sequences.get(sequenceIndex).editedMediaItems; this.composition = composition; this.transformationRequest = transformationRequest; + this.audioMixerFactory = audioMixerFactory; this.videoFrameProcessorFactory = videoFrameProcessorFactory; this.fallbackListener = fallbackListener; this.debugViewProvider = debugViewProvider; @@ -579,6 +584,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* firstInputFormat= */ assetLoaderOutputFormat, transformationRequest, editedMediaItems.get(0), + audioMixerFactory, encoderFactory, muxerWrapper, fallbackListener)); diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/AudioMixerImplTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultAudioMixerTest.java similarity index 99% rename from libraries/transformer/src/test/java/androidx/media3/transformer/AudioMixerImplTest.java rename to libraries/transformer/src/test/java/androidx/media3/transformer/DefaultAudioMixerTest.java index c49cf5303a..959b5cca82 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/AudioMixerImplTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultAudioMixerTest.java @@ -33,7 +33,7 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameter; import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; /** - * Unit tests for {@link AudioMixerImpl}. + * Unit tests for {@link DefaultAudioMixer}. * *

The duration of a given buffer can be calculated with the {@link AudioFormat}: * @@ -48,7 +48,7 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; */ // TODO(b/290002720): Expand and generalize parameterized test cases. @RunWith(ParameterizedRobolectricTestRunner.class) -public final class AudioMixerImplTest { +public final class DefaultAudioMixerTest { @Parameters(name = "outputSilenceWithNoSources={0}") public static ImmutableList parameters() { @@ -67,7 +67,7 @@ public final class AudioMixerImplTest { @Before public void setup() { - mixer = new AudioMixerImpl(outputSilenceWithNoSources); + mixer = new DefaultAudioMixer.Factory(outputSilenceWithNoSources).create(); } @Test