From e175a772dbce90a40e664330a9d3f06dbc9bd7db Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 5 Mar 2024 04:44:06 -0800 Subject: [PATCH] Move AudioGraph effects to configure method Move old configure behavior to private configureMixer. registerInput() is now required after reset(). PiperOrigin-RevId: 612795418 --- .../media3/transformer/AudioGraph.java | 62 ++++++++----- .../transformer/AudioSampleExporter.java | 3 +- .../media3/transformer/AudioGraphTest.java | 92 ++++++++----------- 3 files changed, 80 insertions(+), 77 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java index ac180fecc5..0c5625a5de 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java @@ -18,6 +18,7 @@ package androidx.media3.transformer; import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER; import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkState; import android.util.SparseArray; import androidx.media3.common.C; @@ -34,17 +35,17 @@ import java.util.Objects; /* package */ final class AudioGraph { private final AudioMixer mixer; private final SparseArray inputs; - private final AudioProcessingPipeline audioProcessingPipeline; + private AudioProcessingPipeline audioProcessingPipeline; private AudioFormat mixerAudioFormat; private int finishedInputs; private ByteBuffer mixerOutput; /** Creates an instance. */ - public AudioGraph(AudioMixer.Factory mixerFactory, ImmutableList effects) { + public AudioGraph(AudioMixer.Factory mixerFactory) { mixer = mixerFactory.create(); inputs = new SparseArray<>(); - audioProcessingPipeline = new AudioProcessingPipeline(effects); + audioProcessingPipeline = new AudioProcessingPipeline(ImmutableList.of()); mixerOutput = EMPTY_BUFFER; mixerAudioFormat = AudioFormat.NOT_SET; } @@ -64,28 +65,26 @@ import java.util.Objects; } /** - * Configures the graph. + * Configures the composition-level audio effects to be applied after mixing. * - *

Must be called before {@linkplain #getOutput() accessing output}. + *

Must be called before {@linkplain #registerInput(EditedMediaItem, Format) registering + * inputs}. * - *

Should be called at most once, before {@link #registerInput registering input}. - * - * @param mixerAudioFormat The {@link AudioFormat} requested for output from the mixer. - * @throws UnhandledAudioFormatException If the audio format is not supported by the {@link - * AudioMixer}. + * @param effects The composition-level audio effects. + * @throws IllegalStateException If {@link #registerInput(EditedMediaItem, Format)} was already + * called. */ - public void configure(AudioFormat mixerAudioFormat) throws UnhandledAudioFormatException { - this.mixerAudioFormat = mixerAudioFormat; - mixer.configure(mixerAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0); - audioProcessingPipeline.configure(mixerAudioFormat); - audioProcessingPipeline.flush(); + public void configure(ImmutableList effects) { + checkState( + mixerAudioFormat.equals(AudioFormat.NOT_SET), + "AudioGraph can't configure effects after input registration."); + audioProcessingPipeline = new AudioProcessingPipeline(effects); } /** * Returns a new {@link AudioGraphInput} instance. * - *

Calls {@link #configure} if not already configured, using the {@linkplain - * AudioGraphInput#getOutputAudioFormat() outputAudioFormat} of the input. + *

Must be called before {@linkplain #getOutput() accessing output}. */ public AudioGraphInput registerInput(EditedMediaItem editedMediaItem, Format format) throws ExportException { @@ -95,8 +94,8 @@ import java.util.Objects; new AudioGraphInput(mixerAudioFormat, editedMediaItem, format); if (Objects.equals(mixerAudioFormat, AudioFormat.NOT_SET)) { - // Graph not configured, configure before doing anything else. - configure(audioGraphInput.getOutputAudioFormat()); + // Mixer not configured, configure before doing anything else. + configureMixer(audioGraphInput.getOutputAudioFormat()); } int sourceId = mixer.addSource(audioGraphInput.getOutputAudioFormat(), /* startTimeUs= */ 0); @@ -109,7 +108,8 @@ import java.util.Objects; /** * Returns the {@link AudioFormat} of the {@linkplain #getOutput() output}, or {@link - * AudioFormat#NOT_SET} if not {@linkplain #configure configured}. + * AudioFormat#NOT_SET} if no inputs were {@linkplain #registerInput(EditedMediaItem, Format) + * registered} previously. */ public AudioFormat getOutputAudioFormat() { return audioProcessingPipeline.getOutputAudioFormat(); @@ -136,7 +136,11 @@ import java.util.Objects; return mixerOutput; } - /** Resets the graph to an unconfigured state, releasing any underlying resources. */ + /** + * Resets the graph, un-registering inputs and releasing any underlying resources. + * + *

Call {@link #registerInput(EditedMediaItem, Format)} to prepare the audio graph again. + */ public void reset() { for (int i = 0; i < inputs.size(); i++) { inputs.valueAt(i).release(); @@ -158,6 +162,22 @@ import java.util.Objects; return isMixerEnded(); } + /** + * Configures the mixer. + * + *

Must be called before {@linkplain #getOutput() accessing output}. + * + * @param mixerAudioFormat The {@link AudioFormat} requested for output from the mixer. + * @throws UnhandledAudioFormatException If the audio format is not supported by the {@link + * AudioMixer}. + */ + private void configureMixer(AudioFormat mixerAudioFormat) throws UnhandledAudioFormatException { + this.mixerAudioFormat = mixerAudioFormat; + mixer.configure(mixerAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0); + audioProcessingPipeline.configure(mixerAudioFormat); + audioProcessingPipeline.flush(); + } + private boolean isMixerEnded() { return !mixerOutput.hasRemaining() && finishedInputs >= inputs.size() && mixer.isEnded(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java index 6d434bd74c..16734db52e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java @@ -59,7 +59,8 @@ import org.checkerframework.dataflow.qual.Pure; FallbackListener fallbackListener) throws ExportException { super(firstAssetLoaderTrackFormat, muxerWrapper); - audioGraph = new AudioGraph(mixerFactory, compositionAudioProcessors); + audioGraph = new AudioGraph(mixerFactory); + audioGraph.configure(compositionAudioProcessors); this.firstInputFormat = firstInputFormat; firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat); encoderInputAudioFormat = audioGraph.getOutputAudioFormat(); diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java index 226426264f..14b2e42146 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java @@ -17,6 +17,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Util.getPcmFormat; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; import androidx.media3.common.C; import androidx.media3.common.MediaItem; @@ -46,8 +47,7 @@ public class AudioGraphTest { @Test public void silentItem_outputsCorrectAmountOfBytes() throws Exception { - AudioGraph audioGraph = - new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); input.onMediaItemChanged( @@ -63,9 +63,8 @@ public class AudioGraphTest { public void silentItem_withSampleRateChange_outputsCorrectAmountOfBytes() throws Exception { SonicAudioProcessor changeTo100000Hz = new SonicAudioProcessor(); changeTo100000Hz.setOutputSampleRateHz(100_000); - AudioGraph audioGraph = - new AudioGraph( - new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(changeTo100000Hz)); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + audioGraph.configure(ImmutableList.of(changeTo100000Hz)); GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); input.onMediaItemChanged( @@ -79,48 +78,23 @@ public class AudioGraphTest { @Test public void getOutputAudioFormat_afterInitialization_isNotSet() { - AudioGraph audioGraph = - new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(AudioFormat.NOT_SET); } @Test public void getOutputAudioFormat_afterRegisterInput_matchesInputFormat() throws Exception { - AudioGraph audioGraph = - new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_48000)); assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_48000); } - @Test - public void getOutputAudioFormat_afterConfigure_matchesConfiguredFormat() throws Exception { - AudioGraph audioGraph = - new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of()); - - audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000); - - assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(SURROUND_50000); - } - - @Test - public void registerInput_afterConfigure_doesNotChangeOutputFormat() throws Exception { - AudioGraph audioGraph = - new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of()); - - audioGraph.configure(/* mixerAudioFormat= */ STEREO_44100); - audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000)); - audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100)); - - assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(STEREO_44100); - } - @Test public void registerInput_afterRegisterInput_doesNotChangeOutputFormat() throws Exception { - AudioGraph audioGraph = - new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100)); @@ -130,8 +104,7 @@ public class AudioGraphTest { @Test public void registerInput_afterReset_changesOutputFormat() throws Exception { - AudioGraph audioGraph = - new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000)); audioGraph.reset(); @@ -140,26 +113,12 @@ public class AudioGraphTest { assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_44100); } - @Test - public void configure_withAudioProcessor_affectsOutputFormat() throws Exception { - SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); - sonicAudioProcessor.setOutputSampleRateHz(48_000); - AudioGraph audioGraph = - new AudioGraph( - new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(sonicAudioProcessor)); - - audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000); - - assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000); - } - @Test public void registerInput_withAudioProcessor_affectsOutputFormat() throws Exception { SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); sonicAudioProcessor.setOutputSampleRateHz(48_000); - AudioGraph audioGraph = - new AudioGraph( - new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(sonicAudioProcessor)); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + audioGraph.configure(ImmutableList.of(sonicAudioProcessor)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); @@ -172,16 +131,39 @@ public class AudioGraphTest { changeTo96000Hz.setOutputSampleRateHz(96_000); SonicAudioProcessor changeTo48000Hz = new SonicAudioProcessor(); changeTo48000Hz.setOutputSampleRateHz(48_000); - AudioGraph audioGraph = - new AudioGraph( - new DefaultAudioMixer.Factory(), - /* effects= */ ImmutableList.of(changeTo96000Hz, changeTo48000Hz)); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + audioGraph.configure(ImmutableList.of(changeTo96000Hz, changeTo48000Hz)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000); } + @Test + public void configure_changesOutputFormat() throws Exception { + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); + sonicAudioProcessor.setOutputSampleRateHz(48_000); + audioGraph.configure(ImmutableList.of(sonicAudioProcessor)); + + audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_44100)); + + assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000); + } + + @Test + public void configure_afterRegisterInput_throws() throws Exception { + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); + sonicAudioProcessor.setOutputSampleRateHz(48_000); + + audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_44100)); + + assertThrows( + IllegalStateException.class, + () -> audioGraph.configure(ImmutableList.of(sonicAudioProcessor))); + } + /** Drains the graph and returns the number of bytes output. */ private static int drainAudioGraph(AudioGraph audioGraph) throws ExportException { int bytesOutput = 0;