Add API for injecting AudioMixer.Factory.

Rename AudioMixerImpl to DefaultAudioMixer.

Removes the AudioMixerImpl specific getOutputAudioFormat, as the caller
defines and sets this.

PiperOrigin-RevId: 555887722
This commit is contained in:
samrobinson 2023-08-11 11:01:57 +00:00 committed by Tianyi Feng
parent a9f9537e5f
commit c740b58efa
7 changed files with 89 additions and 28 deletions

View File

@ -28,17 +28,19 @@ import java.nio.ByteBuffer;
/** Processes raw audio samples. */
/* package */ final class AudioGraph {
private final AudioMixerImpl mixer;
private final AudioMixer mixer;
private final SparseArray<AudioGraphInput> inputs;
private AudioFormat outputAudioFormat;
private int finishedInputs;
private ByteBuffer currentOutput;
/** Creates an instance. */
public AudioGraph() {
mixer = new AudioMixerImpl(/* outputSilenceWithNoSources= */ false);
public AudioGraph(AudioMixer.Factory mixerFactory) {
mixer = mixerFactory.create();
inputs = new SparseArray<>();
currentOutput = EMPTY_BUFFER;
outputAudioFormat = AudioFormat.NOT_SET;
}
/** Returns a new {@link AudioGraphInput} instance. */
@ -48,10 +50,9 @@ import java.nio.ByteBuffer;
AudioGraphInput audioGraphInput = new AudioGraphInput(item, format);
if (inputs.size() == 0) {
outputAudioFormat = audioGraphInput.getOutputAudioFormat();
mixer.configure(
audioGraphInput.getOutputAudioFormat(),
/* bufferSizeMs= */ C.LENGTH_UNSET,
/* startTimeUs= */ 0);
outputAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0);
}
int sourceId = mixer.addSource(audioGraphInput.getOutputAudioFormat(), /* startTimeUs= */ 0);
@ -69,7 +70,7 @@ import java.nio.ByteBuffer;
* registered}.
*/
public AudioFormat getOutputAudioFormat() {
return mixer.getOutputAudioFormat();
return outputAudioFormat;
}
/**

View File

@ -53,9 +53,18 @@ import java.nio.ByteBuffer;
*/
@UnstableApi
public interface AudioMixer {
/** Creates an unconfigured instance. */
public static AudioMixer create() {
return new AudioMixerImpl(/* outputSilenceWithNoSources= */ true);
/** A factory for {@link AudioMixer} instances. */
interface Factory {
AudioMixer create();
}
/**
* @deprecated Use {@link DefaultAudioMixer.Factory#create()}.
*/
@Deprecated
static AudioMixer create() {
return new DefaultAudioMixer.Factory(/* outputSilenceWithNoSources= */ true).create();
}
/**

View File

@ -52,12 +52,13 @@ import org.checkerframework.dataflow.qual.Pure;
Format firstInputFormat,
TransformationRequest transformationRequest,
EditedMediaItem firstEditedMediaItem,
AudioMixer.Factory mixerFactory,
Codec.EncoderFactory encoderFactory,
MuxerWrapper muxerWrapper,
FallbackListener fallbackListener)
throws ExportException {
super(firstAssetLoaderTrackFormat, muxerWrapper);
audioGraph = new AudioGraph();
audioGraph = new AudioGraph(mixerFactory);
this.firstInputFormat = firstInputFormat;
firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat);
encoderInputAudioFormat = audioGraph.getOutputAudioFormat();

View File

@ -28,12 +28,42 @@ import androidx.media3.common.audio.AudioMixingUtil;
import androidx.media3.common.audio.AudioProcessor.AudioFormat;
import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException;
import androidx.media3.common.audio.ChannelMixingMatrix;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/** An {@link AudioMixer} that incrementally mixes source audio into a fixed size mixing buffer. */
/* package */ final class AudioMixerImpl implements AudioMixer {
@UnstableApi
public final class DefaultAudioMixer implements AudioMixer {
/** An {@link AudioMixer.Factory} implementation for {@link DefaultAudioMixer} instances. */
public static final class Factory implements AudioMixer.Factory {
private final boolean outputSilenceWithNoSources;
/**
* Creates an instance that does not {@linkplain #getOutput() output} silence when there are no
* {@linkplain #addSource sources}.
*/
public Factory() {
this(/* outputSilenceWithNoSources= */ false);
}
/**
* Creates an instance.
*
* @param outputSilenceWithNoSources Whether to {@linkplain #getOutput() output} silence when
* there are no {@linkplain #addSource sources}.
*/
public Factory(boolean outputSilenceWithNoSources) {
this.outputSilenceWithNoSources = outputSilenceWithNoSources;
}
@Override
public DefaultAudioMixer create() {
return new DefaultAudioMixer(outputSilenceWithNoSources);
}
}
// TODO(b/290002438, b/276734854): Improve buffer management & determine best default size.
private static final int DEFAULT_BUFFER_SIZE_MS = 500;
@ -60,13 +90,7 @@ import java.nio.ByteOrder;
*/
private long maxPositionOfRemovedSources;
/**
* Creates an instance.
*
* @param outputSilenceWithNoSources Whether {@link #getOutput()} should output buffers of silence
* when there are no {@link #addSource sources}.
*/
public AudioMixerImpl(boolean outputSilenceWithNoSources) {
private DefaultAudioMixer(boolean outputSilenceWithNoSources) {
sources = new SparseArray<>();
outputAudioFormat = AudioFormat.NOT_SET;
bufferSizeFrames = C.LENGTH_UNSET;
@ -79,10 +103,6 @@ import java.nio.ByteOrder;
}
}
public AudioFormat getOutputAudioFormat() {
return outputAudioFormat;
}
@Override
public void configure(AudioFormat outputAudioFormat, int bufferSizeMs, long startTimeUs)
throws UnhandledAudioFormatException {

View File

@ -94,6 +94,7 @@ public final class Transformer {
private boolean flattenForSlowMotion;
private ListenerSet<Transformer.Listener> listeners;
private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory;
private AudioMixer.Factory audioMixerFactory;
private VideoFrameProcessor.Factory videoFrameProcessorFactory;
private Codec.EncoderFactory encoderFactory;
private Muxer.Factory muxerFactory;
@ -110,6 +111,7 @@ public final class Transformer {
this.context = context.getApplicationContext();
audioProcessors = ImmutableList.of();
videoEffects = ImmutableList.of();
audioMixerFactory = new DefaultAudioMixer.Factory();
videoFrameProcessorFactory = new DefaultVideoFrameProcessor.Factory.Builder().build();
encoderFactory = new DefaultEncoderFactory.Builder(this.context).build();
muxerFactory = new DefaultMuxer.Factory();
@ -131,6 +133,7 @@ public final class Transformer {
this.removeVideo = transformer.removeVideo;
this.listeners = transformer.listeners;
this.assetLoaderFactory = transformer.assetLoaderFactory;
this.audioMixerFactory = transformer.audioMixerFactory;
this.videoFrameProcessorFactory = transformer.videoFrameProcessorFactory;
this.encoderFactory = transformer.encoderFactory;
this.muxerFactory = transformer.muxerFactory;
@ -339,7 +342,23 @@ public final class Transformer {
}
/**
* Sets the factory to be used to create {@link VideoFrameProcessor} instances.
* Sets the {@link AudioMixer.Factory} to be used when {@linkplain AudioMixer audio mixing} is
* needed.
*
* <p>The default value is a {@link DefaultAudioMixer.Factory} with default values.
*
* @param audioMixerFactory A {@link AudioMixer.Factory}.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setAudioMixerFactory(AudioMixer.Factory audioMixerFactory) {
this.audioMixerFactory = audioMixerFactory;
return this;
}
/**
* Sets the {@link VideoFrameProcessor.Factory} to be used to create {@link VideoFrameProcessor}
* instances.
*
* <p>The default value is a {@link DefaultVideoFrameProcessor.Factory} built with default
* values.
@ -369,7 +388,7 @@ public final class Transformer {
}
/**
* Sets the factory for muxers that write the media container.
* Sets the {@link Muxer.Factory} for muxers that write the media container.
*
* <p>The default value is a {@link DefaultMuxer.Factory}.
*
@ -468,6 +487,7 @@ public final class Transformer {
flattenForSlowMotion,
listeners,
assetLoaderFactory,
audioMixerFactory,
videoFrameProcessorFactory,
encoderFactory,
muxerFactory,
@ -637,6 +657,7 @@ public final class Transformer {
private final boolean flattenForSlowMotion;
private final ListenerSet<Transformer.Listener> listeners;
@Nullable private final AssetLoader.Factory assetLoaderFactory;
private final AudioMixer.Factory audioMixerFactory;
private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
private final Codec.EncoderFactory encoderFactory;
private final Muxer.Factory muxerFactory;
@ -656,6 +677,7 @@ public final class Transformer {
boolean flattenForSlowMotion,
ListenerSet<Listener> listeners,
@Nullable AssetLoader.Factory assetLoaderFactory,
AudioMixer.Factory audioMixerFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
Codec.EncoderFactory encoderFactory,
Muxer.Factory muxerFactory,
@ -672,6 +694,7 @@ public final class Transformer {
this.flattenForSlowMotion = flattenForSlowMotion;
this.listeners = listeners;
this.assetLoaderFactory = assetLoaderFactory;
this.audioMixerFactory = audioMixerFactory;
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
this.encoderFactory = encoderFactory;
this.muxerFactory = muxerFactory;
@ -839,6 +862,7 @@ public final class Transformer {
path,
transformationRequest,
assetLoaderFactory,
audioMixerFactory,
videoFrameProcessorFactory,
encoderFactory,
muxerFactory,

View File

@ -136,6 +136,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
String outputPath,
TransformationRequest transformationRequest,
AssetLoader.Factory assetLoaderFactory,
AudioMixer.Factory audioMixerFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
Codec.EncoderFactory encoderFactory,
Muxer.Factory muxerFactory,
@ -162,6 +163,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* sequenceIndex= */ i,
composition,
transformationRequest,
audioMixerFactory,
videoFrameProcessorFactory,
fallbackListener,
debugViewProvider);
@ -449,6 +451,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final ImmutableList<EditedMediaItem> editedMediaItems;
private final Composition composition;
private final TransformationRequest transformationRequest;
private final AudioMixer.Factory audioMixerFactory;
private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
private final FallbackListener fallbackListener;
private final DebugViewProvider debugViewProvider;
@ -458,6 +461,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
int sequenceIndex,
Composition composition,
TransformationRequest transformationRequest,
AudioMixer.Factory audioMixerFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
FallbackListener fallbackListener,
DebugViewProvider debugViewProvider) {
@ -465,6 +469,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.editedMediaItems = composition.sequences.get(sequenceIndex).editedMediaItems;
this.composition = composition;
this.transformationRequest = transformationRequest;
this.audioMixerFactory = audioMixerFactory;
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
this.fallbackListener = fallbackListener;
this.debugViewProvider = debugViewProvider;
@ -579,6 +584,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* firstInputFormat= */ assetLoaderOutputFormat,
transformationRequest,
editedMediaItems.get(0),
audioMixerFactory,
encoderFactory,
muxerWrapper,
fallbackListener));

View File

@ -33,7 +33,7 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameter;
import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
/**
* Unit tests for {@link AudioMixerImpl}.
* Unit tests for {@link DefaultAudioMixer}.
*
* <p>The duration of a given buffer can be calculated with the {@link AudioFormat}:
*
@ -48,7 +48,7 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
*/
// TODO(b/290002720): Expand and generalize parameterized test cases.
@RunWith(ParameterizedRobolectricTestRunner.class)
public final class AudioMixerImplTest {
public final class DefaultAudioMixerTest {
@Parameters(name = "outputSilenceWithNoSources={0}")
public static ImmutableList<Boolean> parameters() {
@ -67,7 +67,7 @@ public final class AudioMixerImplTest {
@Before
public void setup() {
mixer = new AudioMixerImpl(outputSilenceWithNoSources);
mixer = new DefaultAudioMixer.Factory(outputSilenceWithNoSources).create();
}
@Test