Implement a ChannelMixingAudioProcessor.
PiperOrigin-RevId: 506886903
This commit is contained in:
samrobinson
microkatz
parent
c09904e143
commit
eb4b6f812c
@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Copyright 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.google.android.exoplayer2.transformer;
|
||||
|
||||
import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import android.content.Context;
|
||||
import android.net.Uri;
|
||||
import androidx.test.core.app.ApplicationProvider;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.android.exoplayer2.C;
|
||||
import com.google.android.exoplayer2.MediaItem;
|
||||
import com.google.android.exoplayer2.audio.AudioProcessor;
|
||||
import com.google.android.exoplayer2.audio.ToInt16PcmAudioProcessor;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/**
|
||||
* End-to-end instrumentation test for {@link Transformer} audio test cases that cannot be tested
|
||||
* using robolectric.
|
||||
*/
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public class TransformerAudioEndToEndTest {
|
||||
private final Context context = ApplicationProvider.getApplicationContext();
|
||||
|
||||
@Test
|
||||
public void mixMonoToStereo_outputsStereo() throws Exception {
|
||||
String testId = "mixMonoToStereo_outputsStereo";
|
||||
|
||||
Effects effects =
|
||||
createForAudioProcessors(
|
||||
new ChannelMixingAudioProcessor(
|
||||
ChannelMixingMatrix.create(
|
||||
/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)));
|
||||
EditedMediaItem editedMediaItem =
|
||||
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
|
||||
.setRemoveVideo(true)
|
||||
.setEffects(effects)
|
||||
.build();
|
||||
|
||||
TransformationTestResult result =
|
||||
new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
|
||||
.build()
|
||||
.run(testId, editedMediaItem);
|
||||
|
||||
assertThat(result.transformationResult.channelCount).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mixingChannels_outputsFloatPcm() throws Exception {
|
||||
final String testId = "mixingChannels_outputsFloatPcm";
|
||||
|
||||
Effects effects =
|
||||
createForAudioProcessors(
|
||||
new ChannelMixingAudioProcessor(
|
||||
ChannelMixingMatrix.create(
|
||||
/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)));
|
||||
EditedMediaItem editedMediaItem =
|
||||
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
|
||||
.setRemoveVideo(true)
|
||||
.setEffects(effects)
|
||||
.build();
|
||||
|
||||
TransformationTestResult result =
|
||||
new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
|
||||
.build()
|
||||
.run(testId, editedMediaItem);
|
||||
|
||||
assertThat(result.transformationResult.pcmEncoding).isEqualTo(C.ENCODING_PCM_FLOAT);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mixChannelsThenToInt16Pcm_outputsInt16Pcm() throws Exception {
|
||||
final String testId = "mixChannelsThenToInt16Pcm_outputsInt16Pcm";
|
||||
|
||||
Effects effects =
|
||||
createForAudioProcessors(
|
||||
new ChannelMixingAudioProcessor(
|
||||
ChannelMixingMatrix.create(
|
||||
/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)),
|
||||
new ToInt16PcmAudioProcessor());
|
||||
EditedMediaItem editedMediaItem =
|
||||
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
|
||||
.setRemoveVideo(true)
|
||||
.setEffects(effects)
|
||||
.build();
|
||||
|
||||
TransformationTestResult result =
|
||||
new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
|
||||
.build()
|
||||
.run(testId, editedMediaItem);
|
||||
|
||||
assertThat(result.transformationResult.pcmEncoding).isEqualTo(C.ENCODING_PCM_16BIT);
|
||||
}
|
||||
|
||||
private static Effects createForAudioProcessors(AudioProcessor... audioProcessors) {
|
||||
return new Effects(ImmutableList.copyOf(audioProcessors), ImmutableList.of());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,106 @@
|
||||
/*
|
||||
* Copyright 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.google.android.exoplayer2.transformer;
|
||||
|
||||
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
|
||||
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
|
||||
|
||||
import androidx.annotation.Nullable;
|
||||
import com.google.android.exoplayer2.C;
|
||||
import com.google.android.exoplayer2.audio.AudioProcessor;
|
||||
import com.google.android.exoplayer2.audio.BaseAudioProcessor;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* An {@link AudioProcessor} that handles mixing and scaling audio channels.
|
||||
*
|
||||
* <p>The following encodings are supported as input:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link C#ENCODING_PCM_16BIT}
|
||||
* <li>{@link C#ENCODING_PCM_FLOAT}
|
||||
* </ul>
|
||||
*
|
||||
* The output is {@link C#ENCODING_PCM_FLOAT}.
|
||||
*/
|
||||
/* package */ final class ChannelMixingAudioProcessor extends BaseAudioProcessor {
|
||||
|
||||
@Nullable private ChannelMixingMatrix pendingMatrix;
|
||||
@Nullable private ChannelMixingMatrix matrix;
|
||||
@Nullable private AudioMixingAlgorithm pendingAlgorithm;
|
||||
@Nullable private AudioMixingAlgorithm algorithm;
|
||||
|
||||
public ChannelMixingAudioProcessor(ChannelMixingMatrix matrix) {
|
||||
pendingMatrix = matrix;
|
||||
}
|
||||
|
||||
public void setMatrix(ChannelMixingMatrix matrix) {
|
||||
pendingMatrix = matrix;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AudioFormat onConfigure(AudioFormat inputAudioFormat)
|
||||
throws UnhandledAudioFormatException {
|
||||
checkStateNotNull(pendingMatrix);
|
||||
// TODO(b/252538025): Allow for a mapping of input channel count -> matrix to be passed in.
|
||||
if (inputAudioFormat.channelCount != pendingMatrix.getInputChannelCount()) {
|
||||
throw new UnhandledAudioFormatException(
|
||||
"Channel count must match mixing matrix", inputAudioFormat);
|
||||
}
|
||||
|
||||
if (pendingMatrix.isIdentity()) {
|
||||
return AudioFormat.NOT_SET;
|
||||
}
|
||||
|
||||
// TODO(b/264926272): Allow config of output PCM config when other AudioMixingAlgorithms exist.
|
||||
AudioFormat pendingOutputAudioFormat =
|
||||
new AudioFormat(
|
||||
inputAudioFormat.sampleRate,
|
||||
pendingMatrix.getOutputChannelCount(),
|
||||
C.ENCODING_PCM_FLOAT);
|
||||
|
||||
pendingAlgorithm = AudioMixingAlgorithm.create(pendingOutputAudioFormat);
|
||||
if (!pendingAlgorithm.supportsSourceAudioFormat(inputAudioFormat)) {
|
||||
throw new UnhandledAudioFormatException(inputAudioFormat);
|
||||
}
|
||||
|
||||
return pendingOutputAudioFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onFlush() {
|
||||
algorithm = pendingAlgorithm;
|
||||
matrix = pendingMatrix;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onReset() {
|
||||
pendingAlgorithm = null;
|
||||
algorithm = null;
|
||||
pendingMatrix = null;
|
||||
matrix = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void queueInput(ByteBuffer inputBuffer) {
|
||||
int inputFramesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame;
|
||||
ByteBuffer outputBuffer =
|
||||
replaceOutputBuffer(inputFramesToMix * outputAudioFormat.bytesPerFrame);
|
||||
checkNotNull(algorithm)
|
||||
.mix(inputBuffer, inputAudioFormat, checkNotNull(matrix), inputFramesToMix, outputBuffer);
|
||||
outputBuffer.flip();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,167 @@
|
||||
/*
|
||||
* Copyright 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.google.android.exoplayer2.transformer;
|
||||
|
||||
import static com.google.android.exoplayer2.testutil.TestUtil.createByteBuffer;
|
||||
import static com.google.android.exoplayer2.testutil.TestUtil.createFloatArray;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.android.exoplayer2.C;
|
||||
import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
|
||||
import com.google.android.exoplayer2.audio.AudioProcessor.UnhandledAudioFormatException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/** Unit tests for {@link ChannelMixingAudioProcessor}. */
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public class ChannelMixingAudioProcessorTest {
|
||||
|
||||
private static final AudioFormat AUDIO_FORMAT_48KHZ_STEREO_16BIT =
|
||||
new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 2, C.ENCODING_PCM_16BIT);
|
||||
|
||||
@Test
|
||||
public void configure_outputAudioFormat_matchesChannelCountOfMatrix() throws Exception {
|
||||
ChannelMixingAudioProcessor audioProcessor =
|
||||
new ChannelMixingAudioProcessor(
|
||||
ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1));
|
||||
|
||||
AudioFormat outputAudioFormat = audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT);
|
||||
assertThat(outputAudioFormat.channelCount).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void configure_invalidInputAudioChannelCount_throws() {
|
||||
ChannelMixingAudioProcessor audioProcessor =
|
||||
new ChannelMixingAudioProcessor(
|
||||
ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2));
|
||||
|
||||
assertThrows(
|
||||
UnhandledAudioFormatException.class,
|
||||
() -> audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void reconfigure_withDifferentMatrix_outputsCorrectChannelCount() throws Exception {
|
||||
ChannelMixingMatrix stereoTo1 =
|
||||
ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1);
|
||||
ChannelMixingMatrix stereoTo6 =
|
||||
new ChannelMixingMatrix(
|
||||
/* inputChannelCount= */ 2,
|
||||
/* outputChannelCount= */ 6,
|
||||
new float[] {
|
||||
/* L channel factors */ 0.5f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f,
|
||||
/* R channel factors */ 0.1f, 0.5f, 0.1f, 0.1f, 0.1f, 0.1f
|
||||
});
|
||||
|
||||
ChannelMixingAudioProcessor channelMixingAudioProcessor =
|
||||
new ChannelMixingAudioProcessor(stereoTo1);
|
||||
AudioFormat outputAudioFormat =
|
||||
channelMixingAudioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT);
|
||||
assertThat(outputAudioFormat.channelCount).isEqualTo(1);
|
||||
channelMixingAudioProcessor.flush();
|
||||
|
||||
channelMixingAudioProcessor.setMatrix(stereoTo6);
|
||||
outputAudioFormat = channelMixingAudioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT);
|
||||
assertThat(outputAudioFormat.channelCount).isEqualTo(6);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isActive_afterConfigureWithCustomMixingMatrix_returnsTrue() throws Exception {
|
||||
float[] coefficients =
|
||||
new float[] {
|
||||
/* L channel factors */ 0.5f, 0.5f, 0.0f,
|
||||
/* R channel factors */ 0.0f, 0.5f, 0.5f
|
||||
};
|
||||
|
||||
ChannelMixingAudioProcessor audioProcessor =
|
||||
new ChannelMixingAudioProcessor(
|
||||
new ChannelMixingMatrix(
|
||||
/* inputChannelCount= */ 3, /* outputChannelCount= */ 2, coefficients));
|
||||
|
||||
AudioFormat outputAudioFormat =
|
||||
audioProcessor.configure(
|
||||
new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 3, C.ENCODING_PCM_16BIT));
|
||||
|
||||
assertThat(audioProcessor.isActive()).isTrue();
|
||||
assertThat(outputAudioFormat.channelCount).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isActive_afterConfigureWithIdentityMatrix_returnsFalse() throws Exception {
|
||||
ChannelMixingAudioProcessor audioProcessor =
|
||||
new ChannelMixingAudioProcessor(
|
||||
ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2));
|
||||
|
||||
audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT);
|
||||
assertThat(audioProcessor.isActive()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void numberOfFramesOutput_matchesNumberOfFramesInput() throws Exception {
|
||||
ChannelMixingAudioProcessor audioProcessor =
|
||||
new ChannelMixingAudioProcessor(
|
||||
ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1));
|
||||
|
||||
AudioFormat inputAudioFormat =
|
||||
new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_16BIT);
|
||||
AudioFormat outputAudioFormat = audioProcessor.configure(inputAudioFormat);
|
||||
audioProcessor.flush();
|
||||
audioProcessor.queueInput(
|
||||
ByteBuffer.allocateDirect(inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame)
|
||||
.order(ByteOrder.nativeOrder()));
|
||||
|
||||
assertThat(audioProcessor.getOutput().remaining() / outputAudioFormat.bytesPerFrame)
|
||||
.isEqualTo(44100);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void output_stereoToMono_asExpected() throws Exception {
|
||||
ChannelMixingAudioProcessor audioProcessor =
|
||||
new ChannelMixingAudioProcessor(ChannelMixingMatrix.create(2, 1));
|
||||
|
||||
AudioFormat inputAudioFormat = new AudioFormat(44100, 2, C.ENCODING_PCM_FLOAT);
|
||||
audioProcessor.configure(inputAudioFormat);
|
||||
audioProcessor.flush();
|
||||
|
||||
audioProcessor.queueInput(createByteBuffer(new float[] {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f}));
|
||||
|
||||
assertThat(createFloatArray(audioProcessor.getOutput()))
|
||||
.usingTolerance(1.0e-5)
|
||||
.containsExactly(new float[] {0.15f, 0.35f, 0.55f})
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void output_scaled_asExpected() throws Exception {
|
||||
ChannelMixingAudioProcessor audioProcessor =
|
||||
new ChannelMixingAudioProcessor(ChannelMixingMatrix.create(2, 2).scaleBy(0.5f));
|
||||
|
||||
AudioFormat inputAudioFormat = new AudioFormat(44100, 2, C.ENCODING_PCM_FLOAT);
|
||||
audioProcessor.configure(inputAudioFormat);
|
||||
audioProcessor.flush();
|
||||
|
||||
audioProcessor.queueInput(createByteBuffer(new float[] {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f}));
|
||||
|
||||
assertThat(createFloatArray(audioProcessor.getOutput()))
|
||||
.usingTolerance(1.0e-5)
|
||||
.containsExactly(new float[] {0.05f, 0.1f, 0.15f, 0.2f, 0.25f, 0.3f})
|
||||
.inOrder();
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user