Move AudioGraph effects to configure method

Move old configure behavior to private configureMixer.
registerInput() is now required after reset().

PiperOrigin-RevId: 612795418
This commit is contained in:
Googler 2024-03-05 04:44:06 -08:00 committed by Copybara-Service
parent c79ac5ba21
commit e175a772db
3 changed files with 80 additions and 77 deletions

View File

@ -18,6 +18,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER; import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER;
import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import android.util.SparseArray; import android.util.SparseArray;
import androidx.media3.common.C; import androidx.media3.common.C;
@ -34,17 +35,17 @@ import java.util.Objects;
/* package */ final class AudioGraph { /* package */ final class AudioGraph {
private final AudioMixer mixer; private final AudioMixer mixer;
private final SparseArray<AudioGraphInput> inputs; private final SparseArray<AudioGraphInput> inputs;
private final AudioProcessingPipeline audioProcessingPipeline;
private AudioProcessingPipeline audioProcessingPipeline;
private AudioFormat mixerAudioFormat; private AudioFormat mixerAudioFormat;
private int finishedInputs; private int finishedInputs;
private ByteBuffer mixerOutput; private ByteBuffer mixerOutput;
/** Creates an instance. */ /** Creates an instance. */
public AudioGraph(AudioMixer.Factory mixerFactory, ImmutableList<AudioProcessor> effects) { public AudioGraph(AudioMixer.Factory mixerFactory) {
mixer = mixerFactory.create(); mixer = mixerFactory.create();
inputs = new SparseArray<>(); inputs = new SparseArray<>();
audioProcessingPipeline = new AudioProcessingPipeline(effects); audioProcessingPipeline = new AudioProcessingPipeline(ImmutableList.of());
mixerOutput = EMPTY_BUFFER; mixerOutput = EMPTY_BUFFER;
mixerAudioFormat = AudioFormat.NOT_SET; mixerAudioFormat = AudioFormat.NOT_SET;
} }
@ -64,28 +65,26 @@ import java.util.Objects;
} }
/** /**
* Configures the graph. * Configures the composition-level audio effects to be applied after mixing.
* *
* <p>Must be called before {@linkplain #getOutput() accessing output}. * <p>Must be called before {@linkplain #registerInput(EditedMediaItem, Format) registering
* inputs}.
* *
* <p>Should be called at most once, before {@link #registerInput registering input}. * @param effects The composition-level audio effects.
* * @throws IllegalStateException If {@link #registerInput(EditedMediaItem, Format)} was already
* @param mixerAudioFormat The {@link AudioFormat} requested for output from the mixer. * called.
* @throws UnhandledAudioFormatException If the audio format is not supported by the {@link
* AudioMixer}.
*/ */
public void configure(AudioFormat mixerAudioFormat) throws UnhandledAudioFormatException { public void configure(ImmutableList<AudioProcessor> effects) {
this.mixerAudioFormat = mixerAudioFormat; checkState(
mixer.configure(mixerAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0); mixerAudioFormat.equals(AudioFormat.NOT_SET),
audioProcessingPipeline.configure(mixerAudioFormat); "AudioGraph can't configure effects after input registration.");
audioProcessingPipeline.flush(); audioProcessingPipeline = new AudioProcessingPipeline(effects);
} }
/** /**
* Returns a new {@link AudioGraphInput} instance. * Returns a new {@link AudioGraphInput} instance.
* *
* <p>Calls {@link #configure} if not already configured, using the {@linkplain * <p>Must be called before {@linkplain #getOutput() accessing output}.
* AudioGraphInput#getOutputAudioFormat() outputAudioFormat} of the input.
*/ */
public AudioGraphInput registerInput(EditedMediaItem editedMediaItem, Format format) public AudioGraphInput registerInput(EditedMediaItem editedMediaItem, Format format)
throws ExportException { throws ExportException {
@ -95,8 +94,8 @@ import java.util.Objects;
new AudioGraphInput(mixerAudioFormat, editedMediaItem, format); new AudioGraphInput(mixerAudioFormat, editedMediaItem, format);
if (Objects.equals(mixerAudioFormat, AudioFormat.NOT_SET)) { if (Objects.equals(mixerAudioFormat, AudioFormat.NOT_SET)) {
// Graph not configured, configure before doing anything else. // Mixer not configured, configure before doing anything else.
configure(audioGraphInput.getOutputAudioFormat()); configureMixer(audioGraphInput.getOutputAudioFormat());
} }
int sourceId = mixer.addSource(audioGraphInput.getOutputAudioFormat(), /* startTimeUs= */ 0); int sourceId = mixer.addSource(audioGraphInput.getOutputAudioFormat(), /* startTimeUs= */ 0);
@ -109,7 +108,8 @@ import java.util.Objects;
/** /**
* Returns the {@link AudioFormat} of the {@linkplain #getOutput() output}, or {@link * Returns the {@link AudioFormat} of the {@linkplain #getOutput() output}, or {@link
* AudioFormat#NOT_SET} if not {@linkplain #configure configured}. * AudioFormat#NOT_SET} if no inputs were {@linkplain #registerInput(EditedMediaItem, Format)
* registered} previously.
*/ */
public AudioFormat getOutputAudioFormat() { public AudioFormat getOutputAudioFormat() {
return audioProcessingPipeline.getOutputAudioFormat(); return audioProcessingPipeline.getOutputAudioFormat();
@ -136,7 +136,11 @@ import java.util.Objects;
return mixerOutput; return mixerOutput;
} }
/** Resets the graph to an unconfigured state, releasing any underlying resources. */ /**
* Resets the graph, un-registering inputs and releasing any underlying resources.
*
* <p>Call {@link #registerInput(EditedMediaItem, Format)} to prepare the audio graph again.
*/
public void reset() { public void reset() {
for (int i = 0; i < inputs.size(); i++) { for (int i = 0; i < inputs.size(); i++) {
inputs.valueAt(i).release(); inputs.valueAt(i).release();
@ -158,6 +162,22 @@ import java.util.Objects;
return isMixerEnded(); return isMixerEnded();
} }
/**
* Configures the mixer.
*
* <p>Must be called before {@linkplain #getOutput() accessing output}.
*
* @param mixerAudioFormat The {@link AudioFormat} requested for output from the mixer.
* @throws UnhandledAudioFormatException If the audio format is not supported by the {@link
* AudioMixer}.
*/
private void configureMixer(AudioFormat mixerAudioFormat) throws UnhandledAudioFormatException {
this.mixerAudioFormat = mixerAudioFormat;
mixer.configure(mixerAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0);
audioProcessingPipeline.configure(mixerAudioFormat);
audioProcessingPipeline.flush();
}
private boolean isMixerEnded() { private boolean isMixerEnded() {
return !mixerOutput.hasRemaining() && finishedInputs >= inputs.size() && mixer.isEnded(); return !mixerOutput.hasRemaining() && finishedInputs >= inputs.size() && mixer.isEnded();
} }

View File

@ -59,7 +59,8 @@ import org.checkerframework.dataflow.qual.Pure;
FallbackListener fallbackListener) FallbackListener fallbackListener)
throws ExportException { throws ExportException {
super(firstAssetLoaderTrackFormat, muxerWrapper); super(firstAssetLoaderTrackFormat, muxerWrapper);
audioGraph = new AudioGraph(mixerFactory, compositionAudioProcessors); audioGraph = new AudioGraph(mixerFactory);
audioGraph.configure(compositionAudioProcessors);
this.firstInputFormat = firstInputFormat; this.firstInputFormat = firstInputFormat;
firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat); firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat);
encoderInputAudioFormat = audioGraph.getOutputAudioFormat(); encoderInputAudioFormat = audioGraph.getOutputAudioFormat();

View File

@ -17,6 +17,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.util.Util.getPcmFormat; import static androidx.media3.common.util.Util.getPcmFormat;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.MediaItem; import androidx.media3.common.MediaItem;
@ -46,8 +47,7 @@ public class AudioGraphTest {
@Test @Test
public void silentItem_outputsCorrectAmountOfBytes() throws Exception { public void silentItem_outputsCorrectAmountOfBytes() throws Exception {
AudioGraph audioGraph = AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
input.onMediaItemChanged( input.onMediaItemChanged(
@ -63,9 +63,8 @@ public class AudioGraphTest {
public void silentItem_withSampleRateChange_outputsCorrectAmountOfBytes() throws Exception { public void silentItem_withSampleRateChange_outputsCorrectAmountOfBytes() throws Exception {
SonicAudioProcessor changeTo100000Hz = new SonicAudioProcessor(); SonicAudioProcessor changeTo100000Hz = new SonicAudioProcessor();
changeTo100000Hz.setOutputSampleRateHz(100_000); changeTo100000Hz.setOutputSampleRateHz(100_000);
AudioGraph audioGraph = AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
new AudioGraph( audioGraph.configure(ImmutableList.of(changeTo100000Hz));
new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(changeTo100000Hz));
GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
input.onMediaItemChanged( input.onMediaItemChanged(
@ -79,48 +78,23 @@ public class AudioGraphTest {
@Test @Test
public void getOutputAudioFormat_afterInitialization_isNotSet() { public void getOutputAudioFormat_afterInitialization_isNotSet() {
AudioGraph audioGraph = AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(AudioFormat.NOT_SET); assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(AudioFormat.NOT_SET);
} }
@Test @Test
public void getOutputAudioFormat_afterRegisterInput_matchesInputFormat() throws Exception { public void getOutputAudioFormat_afterRegisterInput_matchesInputFormat() throws Exception {
AudioGraph audioGraph = AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_48000)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_48000));
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_48000); assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_48000);
} }
@Test
public void getOutputAudioFormat_afterConfigure_matchesConfiguredFormat() throws Exception {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000);
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(SURROUND_50000);
}
@Test
public void registerInput_afterConfigure_doesNotChangeOutputFormat() throws Exception {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
audioGraph.configure(/* mixerAudioFormat= */ STEREO_44100);
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100));
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(STEREO_44100);
}
@Test @Test
public void registerInput_afterRegisterInput_doesNotChangeOutputFormat() throws Exception { public void registerInput_afterRegisterInput_doesNotChangeOutputFormat() throws Exception {
AudioGraph audioGraph = AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100));
@ -130,8 +104,7 @@ public class AudioGraphTest {
@Test @Test
public void registerInput_afterReset_changesOutputFormat() throws Exception { public void registerInput_afterReset_changesOutputFormat() throws Exception {
AudioGraph audioGraph = AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
audioGraph.reset(); audioGraph.reset();
@ -140,26 +113,12 @@ public class AudioGraphTest {
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_44100); assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_44100);
} }
@Test
public void configure_withAudioProcessor_affectsOutputFormat() throws Exception {
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48_000);
AudioGraph audioGraph =
new AudioGraph(
new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(sonicAudioProcessor));
audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000);
assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
}
@Test @Test
public void registerInput_withAudioProcessor_affectsOutputFormat() throws Exception { public void registerInput_withAudioProcessor_affectsOutputFormat() throws Exception {
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48_000); sonicAudioProcessor.setOutputSampleRateHz(48_000);
AudioGraph audioGraph = AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
new AudioGraph( audioGraph.configure(ImmutableList.of(sonicAudioProcessor));
new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(sonicAudioProcessor));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
@ -172,16 +131,39 @@ public class AudioGraphTest {
changeTo96000Hz.setOutputSampleRateHz(96_000); changeTo96000Hz.setOutputSampleRateHz(96_000);
SonicAudioProcessor changeTo48000Hz = new SonicAudioProcessor(); SonicAudioProcessor changeTo48000Hz = new SonicAudioProcessor();
changeTo48000Hz.setOutputSampleRateHz(48_000); changeTo48000Hz.setOutputSampleRateHz(48_000);
AudioGraph audioGraph = AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
new AudioGraph( audioGraph.configure(ImmutableList.of(changeTo96000Hz, changeTo48000Hz));
new DefaultAudioMixer.Factory(),
/* effects= */ ImmutableList.of(changeTo96000Hz, changeTo48000Hz));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000); assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
} }
@Test
public void configure_changesOutputFormat() throws Exception {
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48_000);
audioGraph.configure(ImmutableList.of(sonicAudioProcessor));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_44100));
assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
}
@Test
public void configure_afterRegisterInput_throws() throws Exception {
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48_000);
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_44100));
assertThrows(
IllegalStateException.class,
() -> audioGraph.configure(ImmutableList.of(sonicAudioProcessor)));
}
/** Drains the graph and returns the number of bytes output. */ /** Drains the graph and returns the number of bytes output. */
private static int drainAudioGraph(AudioGraph audioGraph) throws ExportException { private static int drainAudioGraph(AudioGraph audioGraph) throws ExportException {
int bytesOutput = 0; int bytesOutput = 0;