loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move AudioGraph effects to configure method

Browse Source 
Move old configure behavior to private configureMixer.
registerInput() is now required after reset().

PiperOrigin-RevId: 612795418
This commit is contained in:
Googler 2024-03-05 04:44:06 -08:00 committed by Copybara-Service
parent c79ac5ba21
commit e175a772db
3 changed files with 80 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java
View File

@ -18,6 +18,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import android.util.SparseArray;
import androidx.media3.common.C;
@ -34,17 +35,17 @@ import java.util.Objects;
/* package */ final class AudioGraph {
private final AudioMixer mixer;
private final SparseArray<AudioGraphInput> inputs;
private final AudioProcessingPipeline audioProcessingPipeline;
private AudioProcessingPipeline audioProcessingPipeline;
private AudioFormat mixerAudioFormat;
private int finishedInputs;
private ByteBuffer mixerOutput;
/** Creates an instance. */
public AudioGraph(AudioMixer.Factory mixerFactory, ImmutableList<AudioProcessor> effects) {
public AudioGraph(AudioMixer.Factory mixerFactory) {
mixer = mixerFactory.create();
inputs = new SparseArray<>();
audioProcessingPipeline = new AudioProcessingPipeline(effects);
audioProcessingPipeline = new AudioProcessingPipeline(ImmutableList.of());
mixerOutput = EMPTY_BUFFER;
mixerAudioFormat = AudioFormat.NOT_SET;
}
@ -64,28 +65,26 @@ import java.util.Objects;
}
/**
* Configures the graph.
* Configures the composition-level audio effects to be applied after mixing.
*
* <p>Must be called before {@linkplain #getOutput() accessing output}.
* <p>Must be called before {@linkplain #registerInput(EditedMediaItem, Format) registering
* inputs}.
*
* <p>Should be called at most once, before {@link #registerInput registering input}.
*
* @param mixerAudioFormat The {@link AudioFormat} requested for output from the mixer.
* @throws UnhandledAudioFormatException If the audio format is not supported by the {@link
* AudioMixer}.
* @param effects The composition-level audio effects.
* @throws IllegalStateException If {@link #registerInput(EditedMediaItem, Format)} was already
* called.
*/
public void configure(AudioFormat mixerAudioFormat) throws UnhandledAudioFormatException {
this.mixerAudioFormat = mixerAudioFormat;
mixer.configure(mixerAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0);
audioProcessingPipeline.configure(mixerAudioFormat);
audioProcessingPipeline.flush();
public void configure(ImmutableList<AudioProcessor> effects) {
checkState(
mixerAudioFormat.equals(AudioFormat.NOT_SET),
"AudioGraph can't configure effects after input registration.");
audioProcessingPipeline = new AudioProcessingPipeline(effects);
}
/**
* Returns a new {@link AudioGraphInput} instance.
*
* <p>Calls {@link #configure} if not already configured, using the {@linkplain
* AudioGraphInput#getOutputAudioFormat() outputAudioFormat} of the input.
* <p>Must be called before {@linkplain #getOutput() accessing output}.
*/
public AudioGraphInput registerInput(EditedMediaItem editedMediaItem, Format format)
throws ExportException {
@ -95,8 +94,8 @@ import java.util.Objects;
new AudioGraphInput(mixerAudioFormat, editedMediaItem, format);
if (Objects.equals(mixerAudioFormat, AudioFormat.NOT_SET)) {
// Graph not configured, configure before doing anything else.
configure(audioGraphInput.getOutputAudioFormat());
// Mixer not configured, configure before doing anything else.
configureMixer(audioGraphInput.getOutputAudioFormat());
}
int sourceId = mixer.addSource(audioGraphInput.getOutputAudioFormat(), /* startTimeUs= */ 0);
@ -109,7 +108,8 @@ import java.util.Objects;
/**
* Returns the {@link AudioFormat} of the {@linkplain #getOutput() output}, or {@link
* AudioFormat#NOT_SET} if not {@linkplain #configure configured}.
* AudioFormat#NOT_SET} if no inputs were {@linkplain #registerInput(EditedMediaItem, Format)
* registered} previously.
*/
public AudioFormat getOutputAudioFormat() {
return audioProcessingPipeline.getOutputAudioFormat();
@ -136,7 +136,11 @@ import java.util.Objects;
return mixerOutput;
}
/** Resets the graph to an unconfigured state, releasing any underlying resources. */
/**
* Resets the graph, un-registering inputs and releasing any underlying resources.
*
* <p>Call {@link #registerInput(EditedMediaItem, Format)} to prepare the audio graph again.
*/
public void reset() {
for (int i = 0; i < inputs.size(); i++) {
inputs.valueAt(i).release();
@ -158,6 +162,22 @@ import java.util.Objects;
return isMixerEnded();
}
/**
* Configures the mixer.
*
* <p>Must be called before {@linkplain #getOutput() accessing output}.
*
* @param mixerAudioFormat The {@link AudioFormat} requested for output from the mixer.
* @throws UnhandledAudioFormatException If the audio format is not supported by the {@link
* AudioMixer}.
*/
private void configureMixer(AudioFormat mixerAudioFormat) throws UnhandledAudioFormatException {
this.mixerAudioFormat = mixerAudioFormat;
mixer.configure(mixerAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0);
audioProcessingPipeline.configure(mixerAudioFormat);
audioProcessingPipeline.flush();
}
private boolean isMixerEnded() {
return !mixerOutput.hasRemaining() && finishedInputs >= inputs.size() && mixer.isEnded();
}

3
libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java
View File

@ -59,7 +59,8 @@ import org.checkerframework.dataflow.qual.Pure;
FallbackListener fallbackListener)
throws ExportException {
super(firstAssetLoaderTrackFormat, muxerWrapper);
audioGraph = new AudioGraph(mixerFactory, compositionAudioProcessors);
audioGraph = new AudioGraph(mixerFactory);
audioGraph.configure(compositionAudioProcessors);
this.firstInputFormat = firstInputFormat;
firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat);
encoderInputAudioFormat = audioGraph.getOutputAudioFormat();

92
libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java
View File

@ -17,6 +17,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.util.Util.getPcmFormat;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import androidx.media3.common.C;
import androidx.media3.common.MediaItem;
@ -46,8 +47,7 @@ public class AudioGraphTest {
@Test
public void silentItem_outputsCorrectAmountOfBytes() throws Exception {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
input.onMediaItemChanged(
@ -63,9 +63,8 @@ public class AudioGraphTest {
public void silentItem_withSampleRateChange_outputsCorrectAmountOfBytes() throws Exception {
SonicAudioProcessor changeTo100000Hz = new SonicAudioProcessor();
changeTo100000Hz.setOutputSampleRateHz(100_000);
AudioGraph audioGraph =
new AudioGraph(
new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(changeTo100000Hz));
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
audioGraph.configure(ImmutableList.of(changeTo100000Hz));
GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
input.onMediaItemChanged(
@ -79,48 +78,23 @@ public class AudioGraphTest {
@Test
public void getOutputAudioFormat_afterInitialization_isNotSet() {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(AudioFormat.NOT_SET);
}
@Test
public void getOutputAudioFormat_afterRegisterInput_matchesInputFormat() throws Exception {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_48000));
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_48000);
}
@Test
public void getOutputAudioFormat_afterConfigure_matchesConfiguredFormat() throws Exception {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000);
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(SURROUND_50000);
}
@Test
public void registerInput_afterConfigure_doesNotChangeOutputFormat() throws Exception {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
audioGraph.configure(/* mixerAudioFormat= */ STEREO_44100);
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100));
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(STEREO_44100);
}
@Test
public void registerInput_afterRegisterInput_doesNotChangeOutputFormat() throws Exception {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100));
@ -130,8 +104,7 @@ public class AudioGraphTest {
@Test
public void registerInput_afterReset_changesOutputFormat() throws Exception {
AudioGraph audioGraph =
new AudioGraph(new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of());
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
audioGraph.reset();
@ -140,26 +113,12 @@ public class AudioGraphTest {
assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_44100);
}
@Test
public void configure_withAudioProcessor_affectsOutputFormat() throws Exception {
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48_000);
AudioGraph audioGraph =
new AudioGraph(
new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(sonicAudioProcessor));
audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000);
assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
}
@Test
public void registerInput_withAudioProcessor_affectsOutputFormat() throws Exception {
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48_000);
AudioGraph audioGraph =
new AudioGraph(
new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(sonicAudioProcessor));
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
audioGraph.configure(ImmutableList.of(sonicAudioProcessor));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
@ -172,16 +131,39 @@ public class AudioGraphTest {
changeTo96000Hz.setOutputSampleRateHz(96_000);
SonicAudioProcessor changeTo48000Hz = new SonicAudioProcessor();
changeTo48000Hz.setOutputSampleRateHz(48_000);
AudioGraph audioGraph =
new AudioGraph(
new DefaultAudioMixer.Factory(),
/* effects= */ ImmutableList.of(changeTo96000Hz, changeTo48000Hz));
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
audioGraph.configure(ImmutableList.of(changeTo96000Hz, changeTo48000Hz));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
}
@Test
public void configure_changesOutputFormat() throws Exception {
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48_000);
audioGraph.configure(ImmutableList.of(sonicAudioProcessor));
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_44100));
assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
}
@Test
public void configure_afterRegisterInput_throws() throws Exception {
AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
sonicAudioProcessor.setOutputSampleRateHz(48_000);
audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_44100));
assertThrows(
IllegalStateException.class,
() -> audioGraph.configure(ImmutableList.of(sonicAudioProcessor)));
}
/** Drains the graph and returns the number of bytes output. */
private static int drainAudioGraph(AudioGraph audioGraph) throws ExportException {
int bytesOutput = 0;