diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAudioEndToEndTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAudioEndToEndTest.java new file mode 100644 index 0000000000..5c71129062 --- /dev/null +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAudioEndToEndTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.net.Uri; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.MediaItem; +import com.google.android.exoplayer2.audio.AudioProcessor; +import com.google.android.exoplayer2.audio.ToInt16PcmAudioProcessor; +import com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * End-to-end instrumentation test for {@link Transformer} audio test cases that cannot be tested + * using robolectric. + */ +@RunWith(AndroidJUnit4.class) +public class TransformerAudioEndToEndTest { + private final Context context = ApplicationProvider.getApplicationContext(); + + @Test + public void mixMonoToStereo_outputsStereo() throws Exception { + String testId = "mixMonoToStereo_outputsStereo"; + + Effects effects = + createForAudioProcessors( + new ChannelMixingAudioProcessor( + ChannelMixingMatrix.create( + /* inputChannelCount= */ 1, /* outputChannelCount= */ 2))); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))) + .setRemoveVideo(true) + .setEffects(effects) + .build(); + + TransformationTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, editedMediaItem); + + assertThat(result.transformationResult.channelCount).isEqualTo(2); + } + + @Test + public void mixingChannels_outputsFloatPcm() throws Exception { + final String testId = "mixingChannels_outputsFloatPcm"; + + Effects effects = + createForAudioProcessors( + new ChannelMixingAudioProcessor( + ChannelMixingMatrix.create( + /* inputChannelCount= */ 1, /* outputChannelCount= */ 2))); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))) + .setRemoveVideo(true) + .setEffects(effects) + .build(); + + TransformationTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, editedMediaItem); + + assertThat(result.transformationResult.pcmEncoding).isEqualTo(C.ENCODING_PCM_FLOAT); + } + + @Test + public void mixChannelsThenToInt16Pcm_outputsInt16Pcm() throws Exception { + final String testId = "mixChannelsThenToInt16Pcm_outputsInt16Pcm"; + + Effects effects = + createForAudioProcessors( + new ChannelMixingAudioProcessor( + ChannelMixingMatrix.create( + /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)), + new ToInt16PcmAudioProcessor()); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))) + .setRemoveVideo(true) + .setEffects(effects) + .build(); + + TransformationTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, editedMediaItem); + + assertThat(result.transformationResult.pcmEncoding).isEqualTo(C.ENCODING_PCM_16BIT); + } + + private static Effects createForAudioProcessors(AudioProcessor... audioProcessors) { + return new Effects(ImmutableList.copyOf(audioProcessors), ImmutableList.of()); + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessor.java new file mode 100644 index 0000000000..40e5f1db73 --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessor.java @@ -0,0 +1,106 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; + +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.audio.AudioProcessor; +import com.google.android.exoplayer2.audio.BaseAudioProcessor; +import java.nio.ByteBuffer; + +/** + * An {@link AudioProcessor} that handles mixing and scaling audio channels. + * + *

The following encodings are supported as input: + * + *

+ * + * The output is {@link C#ENCODING_PCM_FLOAT}. + */ +/* package */ final class ChannelMixingAudioProcessor extends BaseAudioProcessor { + + @Nullable private ChannelMixingMatrix pendingMatrix; + @Nullable private ChannelMixingMatrix matrix; + @Nullable private AudioMixingAlgorithm pendingAlgorithm; + @Nullable private AudioMixingAlgorithm algorithm; + + public ChannelMixingAudioProcessor(ChannelMixingMatrix matrix) { + pendingMatrix = matrix; + } + + public void setMatrix(ChannelMixingMatrix matrix) { + pendingMatrix = matrix; + } + + @Override + protected AudioFormat onConfigure(AudioFormat inputAudioFormat) + throws UnhandledAudioFormatException { + checkStateNotNull(pendingMatrix); + // TODO(b/252538025): Allow for a mapping of input channel count -> matrix to be passed in. + if (inputAudioFormat.channelCount != pendingMatrix.getInputChannelCount()) { + throw new UnhandledAudioFormatException( + "Channel count must match mixing matrix", inputAudioFormat); + } + + if (pendingMatrix.isIdentity()) { + return AudioFormat.NOT_SET; + } + + // TODO(b/264926272): Allow config of output PCM config when other AudioMixingAlgorithms exist. + AudioFormat pendingOutputAudioFormat = + new AudioFormat( + inputAudioFormat.sampleRate, + pendingMatrix.getOutputChannelCount(), + C.ENCODING_PCM_FLOAT); + + pendingAlgorithm = AudioMixingAlgorithm.create(pendingOutputAudioFormat); + if (!pendingAlgorithm.supportsSourceAudioFormat(inputAudioFormat)) { + throw new UnhandledAudioFormatException(inputAudioFormat); + } + + return pendingOutputAudioFormat; + } + + @Override + protected void onFlush() { + algorithm = pendingAlgorithm; + matrix = pendingMatrix; + } + + @Override + protected void onReset() { + pendingAlgorithm = null; + algorithm = null; + pendingMatrix = null; + matrix = null; + } + + @Override + public void queueInput(ByteBuffer inputBuffer) { + int inputFramesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame; + ByteBuffer outputBuffer = + replaceOutputBuffer(inputFramesToMix * outputAudioFormat.bytesPerFrame); + checkNotNull(algorithm) + .mix(inputBuffer, inputAudioFormat, checkNotNull(matrix), inputFramesToMix, outputBuffer); + outputBuffer.flip(); + } +} diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessorTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessorTest.java new file mode 100644 index 0000000000..6d8d41df32 --- /dev/null +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessorTest.java @@ -0,0 +1,167 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.testutil.TestUtil.createByteBuffer; +import static com.google.android.exoplayer2.testutil.TestUtil.createFloatArray; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat; +import com.google.android.exoplayer2.audio.AudioProcessor.UnhandledAudioFormatException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link ChannelMixingAudioProcessor}. */ +@RunWith(AndroidJUnit4.class) +public class ChannelMixingAudioProcessorTest { + + private static final AudioFormat AUDIO_FORMAT_48KHZ_STEREO_16BIT = + new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 2, C.ENCODING_PCM_16BIT); + + @Test + public void configure_outputAudioFormat_matchesChannelCountOfMatrix() throws Exception { + ChannelMixingAudioProcessor audioProcessor = + new ChannelMixingAudioProcessor( + ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); + + AudioFormat outputAudioFormat = audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + assertThat(outputAudioFormat.channelCount).isEqualTo(1); + } + + @Test + public void configure_invalidInputAudioChannelCount_throws() { + ChannelMixingAudioProcessor audioProcessor = + new ChannelMixingAudioProcessor( + ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); + + assertThrows( + UnhandledAudioFormatException.class, + () -> audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT)); + } + + @Test + public void reconfigure_withDifferentMatrix_outputsCorrectChannelCount() throws Exception { + ChannelMixingMatrix stereoTo1 = + ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1); + ChannelMixingMatrix stereoTo6 = + new ChannelMixingMatrix( + /* inputChannelCount= */ 2, + /* outputChannelCount= */ 6, + new float[] { + /* L channel factors */ 0.5f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, + /* R channel factors */ 0.1f, 0.5f, 0.1f, 0.1f, 0.1f, 0.1f + }); + + ChannelMixingAudioProcessor channelMixingAudioProcessor = + new ChannelMixingAudioProcessor(stereoTo1); + AudioFormat outputAudioFormat = + channelMixingAudioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + assertThat(outputAudioFormat.channelCount).isEqualTo(1); + channelMixingAudioProcessor.flush(); + + channelMixingAudioProcessor.setMatrix(stereoTo6); + outputAudioFormat = channelMixingAudioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + assertThat(outputAudioFormat.channelCount).isEqualTo(6); + } + + @Test + public void isActive_afterConfigureWithCustomMixingMatrix_returnsTrue() throws Exception { + float[] coefficients = + new float[] { + /* L channel factors */ 0.5f, 0.5f, 0.0f, + /* R channel factors */ 0.0f, 0.5f, 0.5f + }; + + ChannelMixingAudioProcessor audioProcessor = + new ChannelMixingAudioProcessor( + new ChannelMixingMatrix( + /* inputChannelCount= */ 3, /* outputChannelCount= */ 2, coefficients)); + + AudioFormat outputAudioFormat = + audioProcessor.configure( + new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 3, C.ENCODING_PCM_16BIT)); + + assertThat(audioProcessor.isActive()).isTrue(); + assertThat(outputAudioFormat.channelCount).isEqualTo(2); + } + + @Test + public void isActive_afterConfigureWithIdentityMatrix_returnsFalse() throws Exception { + ChannelMixingAudioProcessor audioProcessor = + new ChannelMixingAudioProcessor( + ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2)); + + audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + assertThat(audioProcessor.isActive()).isFalse(); + } + + @Test + public void numberOfFramesOutput_matchesNumberOfFramesInput() throws Exception { + ChannelMixingAudioProcessor audioProcessor = + new ChannelMixingAudioProcessor( + ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); + + AudioFormat inputAudioFormat = + new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_16BIT); + AudioFormat outputAudioFormat = audioProcessor.configure(inputAudioFormat); + audioProcessor.flush(); + audioProcessor.queueInput( + ByteBuffer.allocateDirect(inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame) + .order(ByteOrder.nativeOrder())); + + assertThat(audioProcessor.getOutput().remaining() / outputAudioFormat.bytesPerFrame) + .isEqualTo(44100); + } + + @Test + public void output_stereoToMono_asExpected() throws Exception { + ChannelMixingAudioProcessor audioProcessor = + new ChannelMixingAudioProcessor(ChannelMixingMatrix.create(2, 1)); + + AudioFormat inputAudioFormat = new AudioFormat(44100, 2, C.ENCODING_PCM_FLOAT); + audioProcessor.configure(inputAudioFormat); + audioProcessor.flush(); + + audioProcessor.queueInput(createByteBuffer(new float[] {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f})); + + assertThat(createFloatArray(audioProcessor.getOutput())) + .usingTolerance(1.0e-5) + .containsExactly(new float[] {0.15f, 0.35f, 0.55f}) + .inOrder(); + } + + @Test + public void output_scaled_asExpected() throws Exception { + ChannelMixingAudioProcessor audioProcessor = + new ChannelMixingAudioProcessor(ChannelMixingMatrix.create(2, 2).scaleBy(0.5f)); + + AudioFormat inputAudioFormat = new AudioFormat(44100, 2, C.ENCODING_PCM_FLOAT); + audioProcessor.configure(inputAudioFormat); + audioProcessor.flush(); + + audioProcessor.queueInput(createByteBuffer(new float[] {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f})); + + assertThat(createFloatArray(audioProcessor.getOutput())) + .usingTolerance(1.0e-5) + .containsExactly(new float[] {0.05f, 0.1f, 0.15f, 0.2f, 0.25f, 0.3f}) + .inOrder(); + } +}