From affc23705594a95032c9362d81aa0d9baef2b562 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Wed, 12 Apr 2023 11:29:06 +0100 Subject: [PATCH] Channel mix to 16-bit int not float Previously `ChannelMixingAudioProcessor` output float because it was implemented using the audio mixer's float mixing support. Move the implementation over to just using the `ChannelMixingMatrix` and make it publicly visible in the common module so it can be used by apps for both playback and export. Also resolve a TODO that no longer had a bug attached by implementing support for putting multiple mixing matrices to handle different input audio channel counts, and fix some nits in the test code. Tested via unit tests and manually configuring a `ChannelMixingAudioProcessor` in the transformer demo app and playing an audio stream that identifies channels, and verifying that they are remapped as expected. PiperOrigin-RevId: 523653901 --- RELEASENOTES.md | 2 + .../audio/ChannelMixingAudioProcessor.java | 110 ++++++++++++ .../common/audio}/ChannelMixingMatrix.java | 7 +- .../ChannelMixingAudioProcessorTest.java | 166 +++++++++++++++++ .../TransformerAudioEndToEndTest.java | 66 +------ .../media3/transformer/AudioMixerImpl.java | 1 + .../transformer/AudioMixingAlgorithm.java | 1 + .../ChannelMixingAudioProcessor.java | 106 ----------- .../FloatAudioMixingAlgorithm.java | 3 +- .../ChannelMixingAudioProcessorTest.java | 167 ------------------ .../transformer/ChannelMixingMatrixTest.java | 1 + .../FloatAudioMixingAlgorithmTest.java | 1 + 12 files changed, 294 insertions(+), 337 deletions(-) create mode 100644 libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingAudioProcessor.java rename libraries/{transformer/src/main/java/androidx/media3/transformer => common/src/main/java/androidx/media3/common/audio}/ChannelMixingMatrix.java (97%) create mode 100644 libraries/common/src/test/java/androidx/media3/common/audio/ChannelMixingAudioProcessorTest.java delete mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/ChannelMixingAudioProcessor.java delete mode 100644 libraries/transformer/src/test/java/androidx/media3/transformer/ChannelMixingAudioProcessorTest.java diff --git a/RELEASENOTES.md b/RELEASENOTES.md index fbab31a0d0..834d9496d5 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -56,6 +56,8 @@ `onAudioCapabilitiesChanged` in `AudioSink.Listener` interface, and a new interface `RendererCapabilities.Listener` which triggers `onRendererCapabilitiesChanged` events. + * Add `ChannelMixingAudioProcessor` for applying scaling/mixing to audio + channels. * Metadata: * Deprecate `MediaMetadata.folderType` in favor of `isBrowsable` and `mediaType`. diff --git a/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingAudioProcessor.java b/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingAudioProcessor.java new file mode 100644 index 0000000000..5443ef4815 --- /dev/null +++ b/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingAudioProcessor.java @@ -0,0 +1,110 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.common.audio; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.util.SparseArray; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; +import java.nio.ByteBuffer; + +/** + * An {@link AudioProcessor} that handles mixing and scaling audio channels. Call {@link + * #putChannelMixingMatrix(ChannelMixingMatrix)} specifying mixing matrices to apply for each + * possible input channel count before using the audio processor. Input and output are 16-bit PCM. + */ +@UnstableApi +public final class ChannelMixingAudioProcessor extends BaseAudioProcessor { + + private final SparseArray matrixByInputChannelCount; + + /** Creates a new audio processor for mixing and scaling audio channels. */ + public ChannelMixingAudioProcessor() { + matrixByInputChannelCount = new SparseArray<>(); + } + + /** + * Stores a channel mixing matrix for processing audio with a given {@link + * ChannelMixingMatrix#getInputChannelCount() channel count}. Overwrites any previously stored + * matrix for the same input channel count. + */ + public void putChannelMixingMatrix(ChannelMixingMatrix matrix) { + int inputChannelCount = matrix.getInputChannelCount(); + matrixByInputChannelCount.put(inputChannelCount, matrix); + } + + @Override + protected AudioFormat onConfigure(AudioFormat inputAudioFormat) + throws UnhandledAudioFormatException { + if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT) { + throw new UnhandledAudioFormatException(inputAudioFormat); + } + @Nullable + ChannelMixingMatrix channelMixingMatrix = + matrixByInputChannelCount.get(inputAudioFormat.channelCount); + if (channelMixingMatrix == null) { + throw new UnhandledAudioFormatException( + "No mixing matrix for input channel count", inputAudioFormat); + } + if (channelMixingMatrix.isIdentity()) { + return AudioFormat.NOT_SET; + } + return new AudioFormat( + inputAudioFormat.sampleRate, + channelMixingMatrix.getOutputChannelCount(), + C.ENCODING_PCM_16BIT); + } + + @Override + public void queueInput(ByteBuffer inputBuffer) { + ChannelMixingMatrix channelMixingMatrix = + checkStateNotNull(matrixByInputChannelCount.get(inputAudioFormat.channelCount)); + + int inputFramesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame; + ByteBuffer outputBuffer = + replaceOutputBuffer(inputFramesToMix * outputAudioFormat.bytesPerFrame); + int inputChannelCount = channelMixingMatrix.getInputChannelCount(); + int outputChannelCount = channelMixingMatrix.getOutputChannelCount(); + float[] outputFrame = new float[outputChannelCount]; + while (inputBuffer.hasRemaining()) { + for (int inputChannelIndex = 0; inputChannelIndex < inputChannelCount; inputChannelIndex++) { + short inputValue = inputBuffer.getShort(); + for (int outputChannelIndex = 0; + outputChannelIndex < outputChannelCount; + outputChannelIndex++) { + outputFrame[outputChannelIndex] += + channelMixingMatrix.getMixingCoefficient(inputChannelIndex, outputChannelIndex) + * inputValue; + } + } + for (int outputChannelIndex = 0; + outputChannelIndex < outputChannelCount; + outputChannelIndex++) { + short shortValue = + (short) + Util.constrainValue( + outputFrame[outputChannelIndex], Short.MIN_VALUE, Short.MAX_VALUE); + outputBuffer.put((byte) (shortValue & 0xFF)); + outputBuffer.put((byte) ((shortValue >> 8) & 0xFF)); + outputFrame[outputChannelIndex] = 0; + } + } + outputBuffer.flip(); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ChannelMixingMatrix.java b/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingMatrix.java similarity index 97% rename from libraries/transformer/src/main/java/androidx/media3/transformer/ChannelMixingMatrix.java rename to libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingMatrix.java index 7dbb801034..87d2ef96b9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ChannelMixingMatrix.java +++ b/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingMatrix.java @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.transformer; +package androidx.media3.common.audio; import static androidx.media3.common.util.Assertions.checkArgument; +import androidx.media3.common.util.UnstableApi; + /** * An immutable matrix that describes the mapping of input channels to output channels. * @@ -39,7 +41,8 @@ import static androidx.media3.common.util.Assertions.checkArgument; * 0 0.7] * */ -/* package */ final class ChannelMixingMatrix { +@UnstableApi +public final class ChannelMixingMatrix { private final int inputChannelCount; private final int outputChannelCount; private final float[] coefficients; diff --git a/libraries/common/src/test/java/androidx/media3/common/audio/ChannelMixingAudioProcessorTest.java b/libraries/common/src/test/java/androidx/media3/common/audio/ChannelMixingAudioProcessorTest.java new file mode 100644 index 0000000000..324584d77a --- /dev/null +++ b/libraries/common/src/test/java/androidx/media3/common/audio/ChannelMixingAudioProcessorTest.java @@ -0,0 +1,166 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.common.audio; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import androidx.media3.common.C; +import androidx.media3.common.audio.AudioProcessor.AudioFormat; +import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link ChannelMixingAudioProcessor}. */ +@RunWith(AndroidJUnit4.class) +public final class ChannelMixingAudioProcessorTest { + + private static final AudioFormat AUDIO_FORMAT_48KHZ_STEREO_16BIT = + new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 2, C.ENCODING_PCM_16BIT); + + private ChannelMixingAudioProcessor audioProcessor; + + @Before + public void setUp() { + audioProcessor = new ChannelMixingAudioProcessor(); + audioProcessor.putChannelMixingMatrix( + ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); + audioProcessor.putChannelMixingMatrix( + ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); + } + + @Test + public void configure_outputAudioFormatMatchesChannelCountOfMatrix() throws Exception { + AudioFormat outputAudioFormat = audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + + assertThat(outputAudioFormat.channelCount).isEqualTo(1); + } + + @Test + public void configureUnhandledChannelCount_throws() { + assertThrows( + UnhandledAudioFormatException.class, + () -> + audioProcessor.configure( + new AudioFormat( + /* sampleRate= */ 44100, /* channelCount= */ 3, C.ENCODING_PCM_16BIT))); + } + + @Test + public void reconfigureWithDifferentMatrix_outputsCorrectChannelCount() throws Exception { + AudioFormat outputAudioFormat = audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + assertThat(outputAudioFormat.channelCount).isEqualTo(1); + audioProcessor.flush(); + audioProcessor.putChannelMixingMatrix( + new ChannelMixingMatrix( + /* inputChannelCount= */ 2, + /* outputChannelCount= */ 6, + new float[] { + /* L channel factors */ 0.5f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, + /* R channel factors */ 0.1f, 0.5f, 0.1f, 0.1f, 0.1f, 0.1f + })); + outputAudioFormat = audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + + assertThat(outputAudioFormat.channelCount).isEqualTo(6); + } + + @Test + public void configureWithCustomMixingMatrix_isActiveReturnsTrue() throws Exception { + audioProcessor.putChannelMixingMatrix( + new ChannelMixingMatrix( + /* inputChannelCount= */ 3, + /* outputChannelCount= */ 2, + new float[] { + /* L channel factors */ 0.5f, 0.5f, 0.0f, + /* R channel factors */ 0.0f, 0.5f, 0.5f + })); + AudioFormat outputAudioFormat = + audioProcessor.configure( + new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 3, C.ENCODING_PCM_16BIT)); + + assertThat(audioProcessor.isActive()).isTrue(); + assertThat(outputAudioFormat.channelCount).isEqualTo(2); + } + + @Test + public void configureWithIdentityMatrix_isActiveReturnsFalse() throws Exception { + audioProcessor.putChannelMixingMatrix( + ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2)); + + audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + assertThat(audioProcessor.isActive()).isFalse(); + } + + @Test + public void queueInputGetOutput_frameCountMatches() throws Exception { + AudioFormat inputAudioFormat = AUDIO_FORMAT_48KHZ_STEREO_16BIT; + AudioFormat outputAudioFormat = audioProcessor.configure(inputAudioFormat); + audioProcessor.flush(); + audioProcessor.queueInput( + ByteBuffer.allocateDirect(inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame) + .order(ByteOrder.nativeOrder())); + + assertThat(audioProcessor.getOutput().remaining() / outputAudioFormat.bytesPerFrame) + .isEqualTo(48000); + } + + @Test + public void stereoToMonoMixingMatrix_queueInput_outputIsMono() throws Exception { + audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + audioProcessor.flush(); + audioProcessor.queueInput(getByteBufferFromShortValues(0, 0, 16383, 16383, 32767, 32767)); + + assertThat(audioProcessor.getOutput()).isEqualTo(getByteBufferFromShortValues(0, 16383, 32767)); + } + + @Test + public void scaledMixingMatrix_queueInput_outputIsScaled() throws Exception { + audioProcessor.putChannelMixingMatrix( + ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2) + .scaleBy(0.5f)); + + audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + audioProcessor.flush(); + audioProcessor.queueInput(getByteBufferFromShortValues(0, 0, 16383, 16383, 32767, 16383)); + + assertThat(audioProcessor.getOutput()) + .isEqualTo(getByteBufferFromShortValues(0, 0, 8191, 8191, 16383, 8191)); + } + + @Test + public void queueInputMultipleTimes_getOutputAsExpected() throws Exception { + audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); + audioProcessor.flush(); + audioProcessor.queueInput(getByteBufferFromShortValues(0, 32767, 0, 32767, 0, 0)); + audioProcessor.getOutput(); + audioProcessor.queueInput(getByteBufferFromShortValues(32767, 32767, 0, 0, 32767, 0)); + + assertThat(audioProcessor.getOutput()).isEqualTo(getByteBufferFromShortValues(32767, 0, 16383)); + } + + private static ByteBuffer getByteBufferFromShortValues(int... values) { + ByteBuffer buffer = ByteBuffer.allocateDirect(values.length * 2).order(ByteOrder.nativeOrder()); + for (int s : values) { + buffer.putShort((short) s); + } + buffer.rewind(); + return buffer; + } +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAudioEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAudioEndToEndTest.java index bd66b252b7..193410bbce 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAudioEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAudioEndToEndTest.java @@ -24,7 +24,8 @@ import androidx.media3.common.C; import androidx.media3.common.MediaItem; import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.audio.AudioProcessor.AudioFormat; -import androidx.media3.common.audio.ToInt16PcmAudioProcessor; +import androidx.media3.common.audio.ChannelMixingAudioProcessor; +import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.media3.exoplayer.audio.TeeAudioProcessor; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; @@ -71,11 +72,10 @@ public class TransformerAudioEndToEndTest { public void mixMonoToStereo_outputsStereo() throws Exception { String testId = "mixMonoToStereo_outputsStereo"; - Effects effects = - createForAudioProcessors( - new ChannelMixingAudioProcessor( - ChannelMixingMatrix.create( - /* inputChannelCount= */ 1, /* outputChannelCount= */ 2))); + ChannelMixingAudioProcessor channelMixingAudioProcessor = new ChannelMixingAudioProcessor(); + channelMixingAudioProcessor.putChannelMixingMatrix( + ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); + Effects effects = createForAudioProcessors(channelMixingAudioProcessor); EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))) .setRemoveVideo(true) @@ -90,60 +90,6 @@ public class TransformerAudioEndToEndTest { assertThat(result.exportResult.channelCount).isEqualTo(2); } - @Test - public void channelMixing_outputsFloatPcm() throws Exception { - final String testId = "channelMixing_outputsFloatPcm"; - FormatTrackingAudioBufferSink audioFormatTracker = new FormatTrackingAudioBufferSink(); - - Effects effects = - createForAudioProcessors( - new ChannelMixingAudioProcessor( - ChannelMixingMatrix.create( - /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)), - new TeeAudioProcessor(audioFormatTracker)); - EditedMediaItem editedMediaItem = - new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))) - .setRemoveVideo(true) - .setEffects(effects) - .build(); - - new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) - .build() - .run(testId, editedMediaItem); - - ImmutableList audioFormats = audioFormatTracker.getFlushedAudioFormats().asList(); - assertThat(audioFormats).hasSize(1); - assertThat(audioFormats.get(0).encoding).isEqualTo(C.ENCODING_PCM_FLOAT); - } - - @Test - public void channelMixingThenToInt16Pcm_outputsInt16Pcm() throws Exception { - final String testId = "channelMixingThenToInt16Pcm_outputsInt16Pcm"; - - FormatTrackingAudioBufferSink audioFormatTracker = new FormatTrackingAudioBufferSink(); - - Effects effects = - createForAudioProcessors( - new ChannelMixingAudioProcessor( - ChannelMixingMatrix.create( - /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)), - new ToInt16PcmAudioProcessor(), - new TeeAudioProcessor(audioFormatTracker)); - EditedMediaItem editedMediaItem = - new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))) - .setRemoveVideo(true) - .setEffects(effects) - .build(); - - new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) - .build() - .run(testId, editedMediaItem); - - ImmutableList audioFormats = audioFormatTracker.getFlushedAudioFormats().asList(); - assertThat(audioFormats).hasSize(1); - assertThat(audioFormats.get(0).encoding).isEqualTo(C.ENCODING_PCM_16BIT); - } - private static Effects createForAudioProcessors(AudioProcessor... audioProcessors) { return new Effects(ImmutableList.copyOf(audioProcessors), ImmutableList.of()); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java index 5ae4c72c27..e34552747c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java @@ -26,6 +26,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.audio.AudioProcessor.AudioFormat; import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; +import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.media3.common.util.Util; import java.nio.ByteBuffer; import java.nio.ByteOrder; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixingAlgorithm.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixingAlgorithm.java index e78719d5c7..3a8887dafe 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixingAlgorithm.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixingAlgorithm.java @@ -19,6 +19,7 @@ import android.annotation.SuppressLint; import androidx.media3.common.C; import androidx.media3.common.audio.AudioProcessor.AudioFormat; import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; +import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.media3.common.util.UnstableApi; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.nio.ByteBuffer; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ChannelMixingAudioProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ChannelMixingAudioProcessor.java deleted file mode 100644 index ef3a3a602e..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ChannelMixingAudioProcessor.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkStateNotNull; - -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.audio.AudioProcessor; -import androidx.media3.common.audio.BaseAudioProcessor; -import java.nio.ByteBuffer; - -/** - * An {@link AudioProcessor} that handles mixing and scaling audio channels. - * - *

The following encodings are supported as input: - * - *

    - *
  • {@link C#ENCODING_PCM_16BIT} - *
  • {@link C#ENCODING_PCM_FLOAT} - *
- * - * The output is {@link C#ENCODING_PCM_FLOAT}. - */ -/* package */ final class ChannelMixingAudioProcessor extends BaseAudioProcessor { - - @Nullable private ChannelMixingMatrix pendingMatrix; - @Nullable private ChannelMixingMatrix matrix; - @Nullable private AudioMixingAlgorithm pendingAlgorithm; - @Nullable private AudioMixingAlgorithm algorithm; - - public ChannelMixingAudioProcessor(ChannelMixingMatrix matrix) { - pendingMatrix = matrix; - } - - public void setMatrix(ChannelMixingMatrix matrix) { - pendingMatrix = matrix; - } - - @Override - protected AudioFormat onConfigure(AudioFormat inputAudioFormat) - throws UnhandledAudioFormatException { - checkStateNotNull(pendingMatrix); - // TODO(b/252538025): Allow for a mapping of input channel count -> matrix to be passed in. - if (inputAudioFormat.channelCount != pendingMatrix.getInputChannelCount()) { - throw new UnhandledAudioFormatException( - "Channel count must match mixing matrix", inputAudioFormat); - } - - if (pendingMatrix.isIdentity()) { - return AudioFormat.NOT_SET; - } - - // TODO(b/264926272): Allow config of output PCM config when other AudioMixingAlgorithms exist. - AudioFormat pendingOutputAudioFormat = - new AudioFormat( - inputAudioFormat.sampleRate, - pendingMatrix.getOutputChannelCount(), - C.ENCODING_PCM_FLOAT); - - pendingAlgorithm = AudioMixingAlgorithm.create(pendingOutputAudioFormat); - if (!pendingAlgorithm.supportsSourceAudioFormat(inputAudioFormat)) { - throw new UnhandledAudioFormatException(inputAudioFormat); - } - - return pendingOutputAudioFormat; - } - - @Override - protected void onFlush() { - algorithm = pendingAlgorithm; - matrix = pendingMatrix; - } - - @Override - protected void onReset() { - pendingAlgorithm = null; - algorithm = null; - pendingMatrix = null; - matrix = null; - } - - @Override - public void queueInput(ByteBuffer inputBuffer) { - int inputFramesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame; - ByteBuffer outputBuffer = - replaceOutputBuffer(inputFramesToMix * outputAudioFormat.bytesPerFrame); - checkNotNull(algorithm) - .mix(inputBuffer, inputAudioFormat, checkNotNull(matrix), inputFramesToMix, outputBuffer); - outputBuffer.flip(); - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FloatAudioMixingAlgorithm.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FloatAudioMixingAlgorithm.java index 7ac4199aae..c919d07db7 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FloatAudioMixingAlgorithm.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FloatAudioMixingAlgorithm.java @@ -21,11 +21,10 @@ import android.annotation.SuppressLint; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.audio.AudioProcessor.AudioFormat; -import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.audio.ChannelMixingMatrix; import java.nio.ByteBuffer; /** An {@link AudioMixingAlgorithm} which mixes into float samples. */ -@UnstableApi /* package */ class FloatAudioMixingAlgorithm implements AudioMixingAlgorithm { // Short.MIN_VALUE != -Short.MAX_VALUE so use different scaling factors for positive and diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ChannelMixingAudioProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ChannelMixingAudioProcessorTest.java deleted file mode 100644 index 16b41bc23d..0000000000 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ChannelMixingAudioProcessorTest.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static androidx.media3.test.utils.TestUtil.createByteBuffer; -import static androidx.media3.test.utils.TestUtil.createFloatArray; -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; - -import androidx.media3.common.C; -import androidx.media3.common.audio.AudioProcessor.AudioFormat; -import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** Unit tests for {@link ChannelMixingAudioProcessor}. */ -@RunWith(AndroidJUnit4.class) -public class ChannelMixingAudioProcessorTest { - - private static final AudioFormat AUDIO_FORMAT_48KHZ_STEREO_16BIT = - new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 2, C.ENCODING_PCM_16BIT); - - @Test - public void configure_outputAudioFormat_matchesChannelCountOfMatrix() throws Exception { - ChannelMixingAudioProcessor audioProcessor = - new ChannelMixingAudioProcessor( - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); - - AudioFormat outputAudioFormat = audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); - assertThat(outputAudioFormat.channelCount).isEqualTo(1); - } - - @Test - public void configure_invalidInputAudioChannelCount_throws() { - ChannelMixingAudioProcessor audioProcessor = - new ChannelMixingAudioProcessor( - ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); - - assertThrows( - UnhandledAudioFormatException.class, - () -> audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT)); - } - - @Test - public void reconfigure_withDifferentMatrix_outputsCorrectChannelCount() throws Exception { - ChannelMixingMatrix stereoTo1 = - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1); - ChannelMixingMatrix stereoTo6 = - new ChannelMixingMatrix( - /* inputChannelCount= */ 2, - /* outputChannelCount= */ 6, - new float[] { - /* L channel factors */ 0.5f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, - /* R channel factors */ 0.1f, 0.5f, 0.1f, 0.1f, 0.1f, 0.1f - }); - - ChannelMixingAudioProcessor channelMixingAudioProcessor = - new ChannelMixingAudioProcessor(stereoTo1); - AudioFormat outputAudioFormat = - channelMixingAudioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); - assertThat(outputAudioFormat.channelCount).isEqualTo(1); - channelMixingAudioProcessor.flush(); - - channelMixingAudioProcessor.setMatrix(stereoTo6); - outputAudioFormat = channelMixingAudioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); - assertThat(outputAudioFormat.channelCount).isEqualTo(6); - } - - @Test - public void isActive_afterConfigureWithCustomMixingMatrix_returnsTrue() throws Exception { - float[] coefficients = - new float[] { - /* L channel factors */ 0.5f, 0.5f, 0.0f, - /* R channel factors */ 0.0f, 0.5f, 0.5f - }; - - ChannelMixingAudioProcessor audioProcessor = - new ChannelMixingAudioProcessor( - new ChannelMixingMatrix( - /* inputChannelCount= */ 3, /* outputChannelCount= */ 2, coefficients)); - - AudioFormat outputAudioFormat = - audioProcessor.configure( - new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 3, C.ENCODING_PCM_16BIT)); - - assertThat(audioProcessor.isActive()).isTrue(); - assertThat(outputAudioFormat.channelCount).isEqualTo(2); - } - - @Test - public void isActive_afterConfigureWithIdentityMatrix_returnsFalse() throws Exception { - ChannelMixingAudioProcessor audioProcessor = - new ChannelMixingAudioProcessor( - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2)); - - audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); - assertThat(audioProcessor.isActive()).isFalse(); - } - - @Test - public void numberOfFramesOutput_matchesNumberOfFramesInput() throws Exception { - ChannelMixingAudioProcessor audioProcessor = - new ChannelMixingAudioProcessor( - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); - - AudioFormat inputAudioFormat = - new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_16BIT); - AudioFormat outputAudioFormat = audioProcessor.configure(inputAudioFormat); - audioProcessor.flush(); - audioProcessor.queueInput( - ByteBuffer.allocateDirect(inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame) - .order(ByteOrder.nativeOrder())); - - assertThat(audioProcessor.getOutput().remaining() / outputAudioFormat.bytesPerFrame) - .isEqualTo(44100); - } - - @Test - public void output_stereoToMono_asExpected() throws Exception { - ChannelMixingAudioProcessor audioProcessor = - new ChannelMixingAudioProcessor(ChannelMixingMatrix.create(2, 1)); - - AudioFormat inputAudioFormat = new AudioFormat(44100, 2, C.ENCODING_PCM_FLOAT); - audioProcessor.configure(inputAudioFormat); - audioProcessor.flush(); - - audioProcessor.queueInput(createByteBuffer(new float[] {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f})); - - assertThat(createFloatArray(audioProcessor.getOutput())) - .usingTolerance(1.0e-5) - .containsExactly(new float[] {0.15f, 0.35f, 0.55f}) - .inOrder(); - } - - @Test - public void output_scaled_asExpected() throws Exception { - ChannelMixingAudioProcessor audioProcessor = - new ChannelMixingAudioProcessor(ChannelMixingMatrix.create(2, 2).scaleBy(0.5f)); - - AudioFormat inputAudioFormat = new AudioFormat(44100, 2, C.ENCODING_PCM_FLOAT); - audioProcessor.configure(inputAudioFormat); - audioProcessor.flush(); - - audioProcessor.queueInput(createByteBuffer(new float[] {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f})); - - assertThat(createFloatArray(audioProcessor.getOutput())) - .usingTolerance(1.0e-5) - .containsExactly(new float[] {0.05f, 0.1f, 0.15f, 0.2f, 0.25f, 0.3f}) - .inOrder(); - } -} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ChannelMixingMatrixTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ChannelMixingMatrixTest.java index bade484d6f..defda03c13 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ChannelMixingMatrixTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ChannelMixingMatrixTest.java @@ -17,6 +17,7 @@ package androidx.media3.transformer; import static com.google.common.truth.Truth.assertThat; +import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/FloatAudioMixingAlgorithmTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/FloatAudioMixingAlgorithmTest.java index 09a8d171fb..a492856b60 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/FloatAudioMixingAlgorithmTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/FloatAudioMixingAlgorithmTest.java @@ -22,6 +22,7 @@ import static com.google.common.truth.Truth.assertWithMessage; import androidx.media3.common.C; import androidx.media3.common.audio.AudioProcessor.AudioFormat; +import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.nio.ByteBuffer; import org.junit.Test;