Add API for injecting AudioMixer.Factory.

Rename AudioMixerImpl to DefaultAudioMixer.

Removes the AudioMixerImpl specific getOutputAudioFormat, as the caller
defines and sets this.

PiperOrigin-RevId: 555887722
This commit is contained in:
samrobinson 2023-08-11 11:01:57 +00:00 committed by Tianyi Feng
parent a9f9537e5f
commit c740b58efa
7 changed files with 89 additions and 28 deletions

View File

@ -28,17 +28,19 @@ import java.nio.ByteBuffer;
/** Processes raw audio samples. */ /** Processes raw audio samples. */
/* package */ final class AudioGraph { /* package */ final class AudioGraph {
private final AudioMixerImpl mixer; private final AudioMixer mixer;
private final SparseArray<AudioGraphInput> inputs; private final SparseArray<AudioGraphInput> inputs;
private AudioFormat outputAudioFormat;
private int finishedInputs; private int finishedInputs;
private ByteBuffer currentOutput; private ByteBuffer currentOutput;
/** Creates an instance. */ /** Creates an instance. */
public AudioGraph() { public AudioGraph(AudioMixer.Factory mixerFactory) {
mixer = new AudioMixerImpl(/* outputSilenceWithNoSources= */ false); mixer = mixerFactory.create();
inputs = new SparseArray<>(); inputs = new SparseArray<>();
currentOutput = EMPTY_BUFFER; currentOutput = EMPTY_BUFFER;
outputAudioFormat = AudioFormat.NOT_SET;
} }
/** Returns a new {@link AudioGraphInput} instance. */ /** Returns a new {@link AudioGraphInput} instance. */
@ -48,10 +50,9 @@ import java.nio.ByteBuffer;
AudioGraphInput audioGraphInput = new AudioGraphInput(item, format); AudioGraphInput audioGraphInput = new AudioGraphInput(item, format);
if (inputs.size() == 0) { if (inputs.size() == 0) {
outputAudioFormat = audioGraphInput.getOutputAudioFormat();
mixer.configure( mixer.configure(
audioGraphInput.getOutputAudioFormat(), outputAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0);
/* bufferSizeMs= */ C.LENGTH_UNSET,
/* startTimeUs= */ 0);
} }
int sourceId = mixer.addSource(audioGraphInput.getOutputAudioFormat(), /* startTimeUs= */ 0); int sourceId = mixer.addSource(audioGraphInput.getOutputAudioFormat(), /* startTimeUs= */ 0);
@ -69,7 +70,7 @@ import java.nio.ByteBuffer;
* registered}. * registered}.
*/ */
public AudioFormat getOutputAudioFormat() { public AudioFormat getOutputAudioFormat() {
return mixer.getOutputAudioFormat(); return outputAudioFormat;
} }
/** /**

View File

@ -53,9 +53,18 @@ import java.nio.ByteBuffer;
*/ */
@UnstableApi @UnstableApi
public interface AudioMixer { public interface AudioMixer {
/** Creates an unconfigured instance. */
public static AudioMixer create() { /** A factory for {@link AudioMixer} instances. */
return new AudioMixerImpl(/* outputSilenceWithNoSources= */ true); interface Factory {
AudioMixer create();
}
/**
* @deprecated Use {@link DefaultAudioMixer.Factory#create()}.
*/
@Deprecated
static AudioMixer create() {
return new DefaultAudioMixer.Factory(/* outputSilenceWithNoSources= */ true).create();
} }
/** /**

View File

@ -52,12 +52,13 @@ import org.checkerframework.dataflow.qual.Pure;
Format firstInputFormat, Format firstInputFormat,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
EditedMediaItem firstEditedMediaItem, EditedMediaItem firstEditedMediaItem,
AudioMixer.Factory mixerFactory,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
MuxerWrapper muxerWrapper, MuxerWrapper muxerWrapper,
FallbackListener fallbackListener) FallbackListener fallbackListener)
throws ExportException { throws ExportException {
super(firstAssetLoaderTrackFormat, muxerWrapper); super(firstAssetLoaderTrackFormat, muxerWrapper);
audioGraph = new AudioGraph(); audioGraph = new AudioGraph(mixerFactory);
this.firstInputFormat = firstInputFormat; this.firstInputFormat = firstInputFormat;
firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat); firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat);
encoderInputAudioFormat = audioGraph.getOutputAudioFormat(); encoderInputAudioFormat = audioGraph.getOutputAudioFormat();

View File

@ -28,12 +28,42 @@ import androidx.media3.common.audio.AudioMixingUtil;
import androidx.media3.common.audio.AudioProcessor.AudioFormat; import androidx.media3.common.audio.AudioProcessor.AudioFormat;
import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException;
import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.media3.common.audio.ChannelMixingMatrix;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.ByteOrder; import java.nio.ByteOrder;
/** An {@link AudioMixer} that incrementally mixes source audio into a fixed size mixing buffer. */ /** An {@link AudioMixer} that incrementally mixes source audio into a fixed size mixing buffer. */
/* package */ final class AudioMixerImpl implements AudioMixer { @UnstableApi
public final class DefaultAudioMixer implements AudioMixer {
/** An {@link AudioMixer.Factory} implementation for {@link DefaultAudioMixer} instances. */
public static final class Factory implements AudioMixer.Factory {
private final boolean outputSilenceWithNoSources;
/**
* Creates an instance that does not {@linkplain #getOutput() output} silence when there are no
* {@linkplain #addSource sources}.
*/
public Factory() {
this(/* outputSilenceWithNoSources= */ false);
}
/**
* Creates an instance.
*
* @param outputSilenceWithNoSources Whether to {@linkplain #getOutput() output} silence when
* there are no {@linkplain #addSource sources}.
*/
public Factory(boolean outputSilenceWithNoSources) {
this.outputSilenceWithNoSources = outputSilenceWithNoSources;
}
@Override
public DefaultAudioMixer create() {
return new DefaultAudioMixer(outputSilenceWithNoSources);
}
}
// TODO(b/290002438, b/276734854): Improve buffer management & determine best default size. // TODO(b/290002438, b/276734854): Improve buffer management & determine best default size.
private static final int DEFAULT_BUFFER_SIZE_MS = 500; private static final int DEFAULT_BUFFER_SIZE_MS = 500;
@ -60,13 +90,7 @@ import java.nio.ByteOrder;
*/ */
private long maxPositionOfRemovedSources; private long maxPositionOfRemovedSources;
/** private DefaultAudioMixer(boolean outputSilenceWithNoSources) {
* Creates an instance.
*
* @param outputSilenceWithNoSources Whether {@link #getOutput()} should output buffers of silence
* when there are no {@link #addSource sources}.
*/
public AudioMixerImpl(boolean outputSilenceWithNoSources) {
sources = new SparseArray<>(); sources = new SparseArray<>();
outputAudioFormat = AudioFormat.NOT_SET; outputAudioFormat = AudioFormat.NOT_SET;
bufferSizeFrames = C.LENGTH_UNSET; bufferSizeFrames = C.LENGTH_UNSET;
@ -79,10 +103,6 @@ import java.nio.ByteOrder;
} }
} }
public AudioFormat getOutputAudioFormat() {
return outputAudioFormat;
}
@Override @Override
public void configure(AudioFormat outputAudioFormat, int bufferSizeMs, long startTimeUs) public void configure(AudioFormat outputAudioFormat, int bufferSizeMs, long startTimeUs)
throws UnhandledAudioFormatException { throws UnhandledAudioFormatException {

View File

@ -94,6 +94,7 @@ public final class Transformer {
private boolean flattenForSlowMotion; private boolean flattenForSlowMotion;
private ListenerSet<Transformer.Listener> listeners; private ListenerSet<Transformer.Listener> listeners;
private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory; private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory;
private AudioMixer.Factory audioMixerFactory;
private VideoFrameProcessor.Factory videoFrameProcessorFactory; private VideoFrameProcessor.Factory videoFrameProcessorFactory;
private Codec.EncoderFactory encoderFactory; private Codec.EncoderFactory encoderFactory;
private Muxer.Factory muxerFactory; private Muxer.Factory muxerFactory;
@ -110,6 +111,7 @@ public final class Transformer {
this.context = context.getApplicationContext(); this.context = context.getApplicationContext();
audioProcessors = ImmutableList.of(); audioProcessors = ImmutableList.of();
videoEffects = ImmutableList.of(); videoEffects = ImmutableList.of();
audioMixerFactory = new DefaultAudioMixer.Factory();
videoFrameProcessorFactory = new DefaultVideoFrameProcessor.Factory.Builder().build(); videoFrameProcessorFactory = new DefaultVideoFrameProcessor.Factory.Builder().build();
encoderFactory = new DefaultEncoderFactory.Builder(this.context).build(); encoderFactory = new DefaultEncoderFactory.Builder(this.context).build();
muxerFactory = new DefaultMuxer.Factory(); muxerFactory = new DefaultMuxer.Factory();
@ -131,6 +133,7 @@ public final class Transformer {
this.removeVideo = transformer.removeVideo; this.removeVideo = transformer.removeVideo;
this.listeners = transformer.listeners; this.listeners = transformer.listeners;
this.assetLoaderFactory = transformer.assetLoaderFactory; this.assetLoaderFactory = transformer.assetLoaderFactory;
this.audioMixerFactory = transformer.audioMixerFactory;
this.videoFrameProcessorFactory = transformer.videoFrameProcessorFactory; this.videoFrameProcessorFactory = transformer.videoFrameProcessorFactory;
this.encoderFactory = transformer.encoderFactory; this.encoderFactory = transformer.encoderFactory;
this.muxerFactory = transformer.muxerFactory; this.muxerFactory = transformer.muxerFactory;
@ -339,7 +342,23 @@ public final class Transformer {
} }
/** /**
* Sets the factory to be used to create {@link VideoFrameProcessor} instances. * Sets the {@link AudioMixer.Factory} to be used when {@linkplain AudioMixer audio mixing} is
* needed.
*
* <p>The default value is a {@link DefaultAudioMixer.Factory} with default values.
*
* @param audioMixerFactory A {@link AudioMixer.Factory}.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setAudioMixerFactory(AudioMixer.Factory audioMixerFactory) {
this.audioMixerFactory = audioMixerFactory;
return this;
}
/**
* Sets the {@link VideoFrameProcessor.Factory} to be used to create {@link VideoFrameProcessor}
* instances.
* *
* <p>The default value is a {@link DefaultVideoFrameProcessor.Factory} built with default * <p>The default value is a {@link DefaultVideoFrameProcessor.Factory} built with default
* values. * values.
@ -369,7 +388,7 @@ public final class Transformer {
} }
/** /**
* Sets the factory for muxers that write the media container. * Sets the {@link Muxer.Factory} for muxers that write the media container.
* *
* <p>The default value is a {@link DefaultMuxer.Factory}. * <p>The default value is a {@link DefaultMuxer.Factory}.
* *
@ -468,6 +487,7 @@ public final class Transformer {
flattenForSlowMotion, flattenForSlowMotion,
listeners, listeners,
assetLoaderFactory, assetLoaderFactory,
audioMixerFactory,
videoFrameProcessorFactory, videoFrameProcessorFactory,
encoderFactory, encoderFactory,
muxerFactory, muxerFactory,
@ -637,6 +657,7 @@ public final class Transformer {
private final boolean flattenForSlowMotion; private final boolean flattenForSlowMotion;
private final ListenerSet<Transformer.Listener> listeners; private final ListenerSet<Transformer.Listener> listeners;
@Nullable private final AssetLoader.Factory assetLoaderFactory; @Nullable private final AssetLoader.Factory assetLoaderFactory;
private final AudioMixer.Factory audioMixerFactory;
private final VideoFrameProcessor.Factory videoFrameProcessorFactory; private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Muxer.Factory muxerFactory; private final Muxer.Factory muxerFactory;
@ -656,6 +677,7 @@ public final class Transformer {
boolean flattenForSlowMotion, boolean flattenForSlowMotion,
ListenerSet<Listener> listeners, ListenerSet<Listener> listeners,
@Nullable AssetLoader.Factory assetLoaderFactory, @Nullable AssetLoader.Factory assetLoaderFactory,
AudioMixer.Factory audioMixerFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Muxer.Factory muxerFactory, Muxer.Factory muxerFactory,
@ -672,6 +694,7 @@ public final class Transformer {
this.flattenForSlowMotion = flattenForSlowMotion; this.flattenForSlowMotion = flattenForSlowMotion;
this.listeners = listeners; this.listeners = listeners;
this.assetLoaderFactory = assetLoaderFactory; this.assetLoaderFactory = assetLoaderFactory;
this.audioMixerFactory = audioMixerFactory;
this.videoFrameProcessorFactory = videoFrameProcessorFactory; this.videoFrameProcessorFactory = videoFrameProcessorFactory;
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.muxerFactory = muxerFactory; this.muxerFactory = muxerFactory;
@ -839,6 +862,7 @@ public final class Transformer {
path, path,
transformationRequest, transformationRequest,
assetLoaderFactory, assetLoaderFactory,
audioMixerFactory,
videoFrameProcessorFactory, videoFrameProcessorFactory,
encoderFactory, encoderFactory,
muxerFactory, muxerFactory,

View File

@ -136,6 +136,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
String outputPath, String outputPath,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
AssetLoader.Factory assetLoaderFactory, AssetLoader.Factory assetLoaderFactory,
AudioMixer.Factory audioMixerFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Muxer.Factory muxerFactory, Muxer.Factory muxerFactory,
@ -162,6 +163,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* sequenceIndex= */ i, /* sequenceIndex= */ i,
composition, composition,
transformationRequest, transformationRequest,
audioMixerFactory,
videoFrameProcessorFactory, videoFrameProcessorFactory,
fallbackListener, fallbackListener,
debugViewProvider); debugViewProvider);
@ -449,6 +451,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final ImmutableList<EditedMediaItem> editedMediaItems; private final ImmutableList<EditedMediaItem> editedMediaItems;
private final Composition composition; private final Composition composition;
private final TransformationRequest transformationRequest; private final TransformationRequest transformationRequest;
private final AudioMixer.Factory audioMixerFactory;
private final VideoFrameProcessor.Factory videoFrameProcessorFactory; private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
private final FallbackListener fallbackListener; private final FallbackListener fallbackListener;
private final DebugViewProvider debugViewProvider; private final DebugViewProvider debugViewProvider;
@ -458,6 +461,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
int sequenceIndex, int sequenceIndex,
Composition composition, Composition composition,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
AudioMixer.Factory audioMixerFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory,
FallbackListener fallbackListener, FallbackListener fallbackListener,
DebugViewProvider debugViewProvider) { DebugViewProvider debugViewProvider) {
@ -465,6 +469,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.editedMediaItems = composition.sequences.get(sequenceIndex).editedMediaItems; this.editedMediaItems = composition.sequences.get(sequenceIndex).editedMediaItems;
this.composition = composition; this.composition = composition;
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.audioMixerFactory = audioMixerFactory;
this.videoFrameProcessorFactory = videoFrameProcessorFactory; this.videoFrameProcessorFactory = videoFrameProcessorFactory;
this.fallbackListener = fallbackListener; this.fallbackListener = fallbackListener;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
@ -579,6 +584,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* firstInputFormat= */ assetLoaderOutputFormat, /* firstInputFormat= */ assetLoaderOutputFormat,
transformationRequest, transformationRequest,
editedMediaItems.get(0), editedMediaItems.get(0),
audioMixerFactory,
encoderFactory, encoderFactory,
muxerWrapper, muxerWrapper,
fallbackListener)); fallbackListener));

View File

@ -33,7 +33,7 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameter;
import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
/** /**
* Unit tests for {@link AudioMixerImpl}. * Unit tests for {@link DefaultAudioMixer}.
* *
* <p>The duration of a given buffer can be calculated with the {@link AudioFormat}: * <p>The duration of a given buffer can be calculated with the {@link AudioFormat}:
* *
@ -48,7 +48,7 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
*/ */
// TODO(b/290002720): Expand and generalize parameterized test cases. // TODO(b/290002720): Expand and generalize parameterized test cases.
@RunWith(ParameterizedRobolectricTestRunner.class) @RunWith(ParameterizedRobolectricTestRunner.class)
public final class AudioMixerImplTest { public final class DefaultAudioMixerTest {
@Parameters(name = "outputSilenceWithNoSources={0}") @Parameters(name = "outputSilenceWithNoSources={0}")
public static ImmutableList<Boolean> parameters() { public static ImmutableList<Boolean> parameters() {
@ -67,7 +67,7 @@ public final class AudioMixerImplTest {
@Before @Before
public void setup() { public void setup() {
mixer = new AudioMixerImpl(outputSilenceWithNoSources); mixer = new DefaultAudioMixer.Factory(outputSilenceWithNoSources).create();
} }
@Test @Test