diff --git a/libraries/common/src/main/java/androidx/media3/common/audio/AudioMixingUtil.java b/libraries/common/src/main/java/androidx/media3/common/audio/AudioMixingUtil.java new file mode 100644 index 0000000000..491c6af8a8 --- /dev/null +++ b/libraries/common/src/main/java/androidx/media3/common/audio/AudioMixingUtil.java @@ -0,0 +1,161 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.common.audio; + +import static androidx.media3.common.util.Util.constrainValue; + +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.audio.AudioProcessor.AudioFormat; +import androidx.media3.common.util.UnstableApi; +import java.nio.ByteBuffer; + +/** Utility for mixing audio buffers. */ +@UnstableApi +public final class AudioMixingUtil { + + // Float PCM samples are zero-centred within the [-1.0, 1.0] range. + private static final float FLOAT_PCM_MIN_VALUE = -1.0f; + private static final float FLOAT_PCM_MAX_VALUE = 1.0f; + + public static boolean canMix(AudioFormat audioFormat) { + if (audioFormat.sampleRate == Format.NO_VALUE) { + return false; + } + if (audioFormat.channelCount == Format.NO_VALUE) { + return false; + } + return audioFormat.encoding == C.ENCODING_PCM_16BIT + || audioFormat.encoding == C.ENCODING_PCM_FLOAT; + } + + public static boolean canMix(AudioFormat inputAudioFormat, AudioFormat outputAudioFormat) { + if (inputAudioFormat.sampleRate != outputAudioFormat.sampleRate) { + return false; + } + if (!canMix(inputAudioFormat)) { + return false; + } + if (!canMix(outputAudioFormat)) { + return false; + } + return true; + } + + /** + * Mixes audio from the input buffer into the mixing buffer. + * + *

{@link #canMix(AudioFormat, AudioFormat)} must return {@code true} for the formats. + * + * @param inputBuffer Input audio {@link ByteBuffer}, the position is advanced by the amount of + * bytes read and mixed. + * @param inputAudioFormat {@link AudioFormat} of the {@code inputBuffer}. + * @param mixingBuffer Mixing audio {@link ByteBuffer}, the position is advanced by the amount of + * bytes written. + * @param mixingAudioFormat {@link AudioFormat} of the {@code mixingBuffer}. + * @param matrix Scaled channel mapping from input to output. + * @param framesToMix Number of audio frames to mix. Must be within the bounds of both buffers. + * @param accumulate Whether to accumulate with the existing samples in the mixing buffer. + * @return The {@code mixingBuffer}, for convenience. + */ + public static ByteBuffer mix( + ByteBuffer inputBuffer, + AudioFormat inputAudioFormat, + ByteBuffer mixingBuffer, + AudioFormat mixingAudioFormat, + ChannelMixingMatrix matrix, + int framesToMix, + boolean accumulate) { + + boolean int16Input = inputAudioFormat.encoding == C.ENCODING_PCM_16BIT; + boolean int16Output = mixingAudioFormat.encoding == C.ENCODING_PCM_16BIT; + int inputChannels = matrix.getInputChannelCount(); + int outputChannels = matrix.getOutputChannelCount(); + float[] inputFrame = new float[inputChannels]; + float[] outputFrame = new float[outputChannels]; + + for (int i = 0; i < framesToMix; i++) { + if (accumulate) { + int position = mixingBuffer.position(); + for (int outputChannel = 0; outputChannel < outputChannels; outputChannel++) { + outputFrame[outputChannel] = + getPcmSample(mixingBuffer, /* int16Buffer= */ int16Output, int16Output); + } + mixingBuffer.position(position); + } + + for (int inputChannel = 0; inputChannel < inputChannels; inputChannel++) { + inputFrame[inputChannel] = + getPcmSample(inputBuffer, /* int16Buffer= */ int16Input, int16Output); + } + + for (int outputChannel = 0; outputChannel < outputChannels; outputChannel++) { + for (int inputChannel = 0; inputChannel < inputChannels; inputChannel++) { + outputFrame[outputChannel] += + inputFrame[inputChannel] * matrix.getMixingCoefficient(inputChannel, outputChannel); + } + + if (int16Output) { + mixingBuffer.putShort( + (short) constrainValue(outputFrame[outputChannel], Short.MIN_VALUE, Short.MAX_VALUE)); + } else { + mixingBuffer.putFloat( + constrainValue(outputFrame[outputChannel], FLOAT_PCM_MIN_VALUE, FLOAT_PCM_MAX_VALUE)); + } + + outputFrame[outputChannel] = 0; + } + } + return mixingBuffer; + } + + /** + * Gets the next sample from the {@link ByteBuffer} of raw audio. + * + *

Int16 PCM range of values: [{@link Short#MIN_VALUE}, {@link Short#MAX_VALUE}]. + * + *

Float PCM range of values: [-1.0, 1.0]. + * + * @param buffer The {@link ByteBuffer} containing raw audio. + * @param int16Buffer Whether the buffer contains {@link C#ENCODING_PCM_16BIT} audio. Use {@code + * false} if buffer contains {@link C#ENCODING_PCM_FLOAT} audio. + * @param int16Output Whether the returned sample should be in the {@link C#ENCODING_PCM_16BIT} + * range of values. If {@code false}, Float PCM range is used. + * @return The next sample from the buffer. + */ + private static float getPcmSample(ByteBuffer buffer, boolean int16Buffer, boolean int16Output) { + if (int16Output) { + return int16Buffer ? buffer.getShort() : floatSampleToInt16Pcm(buffer.getFloat()); + } else { + return int16Buffer ? int16SampleToFloatPcm(buffer.getShort()) : buffer.getFloat(); + } + } + + private static float floatSampleToInt16Pcm(float floatPcmValue) { + return constrainValue( + floatPcmValue * (floatPcmValue < 0 ? -Short.MIN_VALUE : Short.MAX_VALUE), + Short.MIN_VALUE, + Short.MAX_VALUE); + } + + private static float int16SampleToFloatPcm(short shortPcmValue) { + // Short.MIN_VALUE != -Short.MAX_VALUE, so use different conversion for positive and negative. + return shortPcmValue / (float) (shortPcmValue < 0 ? -Short.MIN_VALUE : Short.MAX_VALUE); + } + + private AudioMixingUtil() {} +} diff --git a/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingAudioProcessor.java b/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingAudioProcessor.java index 5443ef4815..d82eeae696 100644 --- a/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingAudioProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingAudioProcessor.java @@ -21,7 +21,6 @@ import android.util.SparseArray; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.util.UnstableApi; -import androidx.media3.common.util.Util; import java.nio.ByteBuffer; /** @@ -52,6 +51,7 @@ public final class ChannelMixingAudioProcessor extends BaseAudioProcessor { @Override protected AudioFormat onConfigure(AudioFormat inputAudioFormat) throws UnhandledAudioFormatException { + // TODO(b/290002731): Expand to allow float due to AudioMixingUtil built-in support for float. if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT) { throw new UnhandledAudioFormatException(inputAudioFormat); } @@ -76,35 +76,16 @@ public final class ChannelMixingAudioProcessor extends BaseAudioProcessor { ChannelMixingMatrix channelMixingMatrix = checkStateNotNull(matrixByInputChannelCount.get(inputAudioFormat.channelCount)); - int inputFramesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame; - ByteBuffer outputBuffer = - replaceOutputBuffer(inputFramesToMix * outputAudioFormat.bytesPerFrame); - int inputChannelCount = channelMixingMatrix.getInputChannelCount(); - int outputChannelCount = channelMixingMatrix.getOutputChannelCount(); - float[] outputFrame = new float[outputChannelCount]; - while (inputBuffer.hasRemaining()) { - for (int inputChannelIndex = 0; inputChannelIndex < inputChannelCount; inputChannelIndex++) { - short inputValue = inputBuffer.getShort(); - for (int outputChannelIndex = 0; - outputChannelIndex < outputChannelCount; - outputChannelIndex++) { - outputFrame[outputChannelIndex] += - channelMixingMatrix.getMixingCoefficient(inputChannelIndex, outputChannelIndex) - * inputValue; - } - } - for (int outputChannelIndex = 0; - outputChannelIndex < outputChannelCount; - outputChannelIndex++) { - short shortValue = - (short) - Util.constrainValue( - outputFrame[outputChannelIndex], Short.MIN_VALUE, Short.MAX_VALUE); - outputBuffer.put((byte) (shortValue & 0xFF)); - outputBuffer.put((byte) ((shortValue >> 8) & 0xFF)); - outputFrame[outputChannelIndex] = 0; - } - } + int framesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame; + ByteBuffer outputBuffer = replaceOutputBuffer(framesToMix * outputAudioFormat.bytesPerFrame); + AudioMixingUtil.mix( + inputBuffer, + inputAudioFormat, + outputBuffer, + outputAudioFormat, + channelMixingMatrix, + framesToMix, + /* accumulate= */ false); outputBuffer.flip(); } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/FloatAudioMixingAlgorithmTest.java b/libraries/common/src/test/java/androidx/media3/common/audio/AudioMixingUtilTest.java similarity index 50% rename from libraries/transformer/src/test/java/androidx/media3/transformer/FloatAudioMixingAlgorithmTest.java rename to libraries/common/src/test/java/androidx/media3/common/audio/AudioMixingUtilTest.java index a492856b60..2535289b2f 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/FloatAudioMixingAlgorithmTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/audio/AudioMixingUtilTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.transformer; +package androidx.media3.common.audio; import static androidx.media3.test.utils.TestUtil.createByteBuffer; import static androidx.media3.test.utils.TestUtil.createFloatArray; @@ -22,22 +22,22 @@ import static com.google.common.truth.Truth.assertWithMessage; import androidx.media3.common.C; import androidx.media3.common.audio.AudioProcessor.AudioFormat; -import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.nio.ByteBuffer; import org.junit.Test; import org.junit.runner.RunWith; -/** Unit tests for {@link FloatAudioMixingAlgorithm}. */ +// TODO(b/290002720): Consider parameterization of these test cases. +/** Unit tests for {@link AudioMixingUtil}. */ @RunWith(AndroidJUnit4.class) -public final class FloatAudioMixingAlgorithmTest { - private static final AudioFormat AUDIO_FORMAT_STEREO_PCM_FLOAT = +public final class AudioMixingUtilTest { + private static final AudioFormat STEREO_44100_PCM_FLOAT = new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_FLOAT); - private static final AudioFormat AUDIO_FORMAT_MONO_PCM_FLOAT = + private static final AudioFormat MONO_44100_PCM_FLOAT = new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 1, C.ENCODING_PCM_FLOAT); - private static final AudioFormat AUDIO_FORMAT_STEREO_PCM_16BIT = + private static final AudioFormat STEREO_44100_PCM_16BIT = new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_16BIT); - private static final AudioFormat AUDIO_FORMAT_MONO_PCM_16BIT = + private static final AudioFormat MONO_44100_PCM_16BIT = new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 1, C.ENCODING_PCM_16BIT); private static final ChannelMixingMatrix STEREO_TO_STEREO = @@ -50,95 +50,47 @@ public final class FloatAudioMixingAlgorithmTest { ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 1); @Test - public void supportsSourceAudioFormatsForStereoMixing() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_STEREO_PCM_FLOAT); - assertThat(algorithm.supportsSourceAudioFormat(AUDIO_FORMAT_STEREO_PCM_FLOAT)).isTrue(); - assertThat(algorithm.supportsSourceAudioFormat(AUDIO_FORMAT_MONO_PCM_FLOAT)).isTrue(); - assertThat(algorithm.supportsSourceAudioFormat(AUDIO_FORMAT_STEREO_PCM_16BIT)).isTrue(); - assertThat(algorithm.supportsSourceAudioFormat(AUDIO_FORMAT_MONO_PCM_16BIT)).isTrue(); - } - - @Test - public void supportsSourceAudioFormatsForMonoMixing() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_MONO_PCM_FLOAT); - assertThat(algorithm.supportsSourceAudioFormat(AUDIO_FORMAT_STEREO_PCM_FLOAT)).isTrue(); - assertThat(algorithm.supportsSourceAudioFormat(AUDIO_FORMAT_MONO_PCM_FLOAT)).isTrue(); - assertThat(algorithm.supportsSourceAudioFormat(AUDIO_FORMAT_STEREO_PCM_16BIT)).isTrue(); - assertThat(algorithm.supportsSourceAudioFormat(AUDIO_FORMAT_MONO_PCM_16BIT)).isTrue(); - } - - @Test - public void doesNotSupportSampleRateConversion() { - AudioMixingAlgorithm algorithm = - new FloatAudioMixingAlgorithm( - new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_FLOAT)); - - assertThat( - algorithm.supportsSourceAudioFormat( - new AudioFormat( - /* sampleRate= */ 48000, /* channelCount= */ 2, C.ENCODING_PCM_FLOAT))) - .isFalse(); - } - - @Test - public void doesNotSupportSampleFormats() { - AudioMixingAlgorithm algorithm = - new FloatAudioMixingAlgorithm( - new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_FLOAT)); - - assertThat( - algorithm.supportsSourceAudioFormat( - new AudioFormat( - /* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_24BIT))) - .isFalse(); - assertThat( - algorithm.supportsSourceAudioFormat( - new AudioFormat( - /* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_32BIT))) - .isFalse(); - } - - @Test - public void mixStereoFloatIntoStereoFloat() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_STEREO_PCM_FLOAT); + public void mixToStereoFloat_withStereoFloatInput() { ByteBuffer mixingBuffer = createByteBuffer(new float[] {0.25f, -0.25f, 0.5f, -0.5f}); ByteBuffer sourceBuffer = createByteBuffer(new float[] {-0.5f, 0.25f, -0.25f, 0.5f}); - algorithm.mix( + AudioMixingUtil.mix( sourceBuffer, - AUDIO_FORMAT_STEREO_PCM_FLOAT, + STEREO_44100_PCM_FLOAT, + mixingBuffer, + STEREO_44100_PCM_FLOAT, STEREO_TO_STEREO.scaleBy(0.5f), - /* frameCount= */ 2, - mixingBuffer); + /* framesToMix= */ 2, + /* accumulate= */ true); assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); - mixingBuffer.flip(); + mixingBuffer.rewind(); assertThat(createFloatArray(mixingBuffer)).isEqualTo(new float[] {0f, -0.125f, 0.375f, -0.25f}); } @Test - public void mixMonoFloatIntoStereoFloat() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_STEREO_PCM_FLOAT); + public void mixToStereoFloat_withMonoFloatInput() { ByteBuffer mixingBuffer = createByteBuffer(new float[] {0.25f, -0.25f, 0.5f, -0.5f}); ByteBuffer sourceBuffer = createByteBuffer(new float[] {-0.5f, 0.5f}); - algorithm.mix( + AudioMixingUtil.mix( sourceBuffer, - AUDIO_FORMAT_MONO_PCM_FLOAT, + MONO_44100_PCM_FLOAT, + mixingBuffer, + STEREO_44100_PCM_FLOAT, MONO_TO_STEREO.scaleBy(0.5f), - /* frameCount= */ 2, - mixingBuffer); + /* framesToMix= */ 2, + /* accumulate= */ true); assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); - mixingBuffer.flip(); + mixingBuffer.rewind(); assertThat(createFloatArray(mixingBuffer)).isEqualTo(new float[] {0f, -0.5f, 0.75f, -0.25f}); } @Test - public void mixStereoS16IntoStereoFloat() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_STEREO_PCM_FLOAT); + public void mixToStereoFloat_withStereo16Input() { ByteBuffer mixingBuffer = createByteBuffer(new float[] {0.25f, -0.25f, 0.5f, -0.5f}); ByteBuffer sourceBuffer = createByteBuffer( @@ -149,16 +101,18 @@ public final class FloatAudioMixingAlgorithmTest { 16384 /* 0.50001525925f */ }); - algorithm.mix( + AudioMixingUtil.mix( sourceBuffer, - AUDIO_FORMAT_STEREO_PCM_16BIT, + STEREO_44100_PCM_16BIT, + mixingBuffer, + STEREO_44100_PCM_FLOAT, STEREO_TO_STEREO.scaleBy(0.5f), - /* frameCount= */ 2, - mixingBuffer); + /* framesToMix= */ 2, + /* accumulate= */ true); assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); - mixingBuffer.flip(); + mixingBuffer.rewind(); assertThat(createFloatArray(mixingBuffer)) .usingTolerance(1f / Short.MAX_VALUE) .containsExactly(new float[] {0f, -0.125f, 0.375f, -0.25f}) @@ -166,22 +120,23 @@ public final class FloatAudioMixingAlgorithmTest { } @Test - public void mixMonoS16IntoStereoFloat() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_STEREO_PCM_FLOAT); + public void mixToStereoFloat_withMono16Input() { ByteBuffer mixingBuffer = createByteBuffer(new float[] {0.25f, -0.25f, 0.5f, -0.5f}); ByteBuffer sourceBuffer = createByteBuffer(new short[] {-16384 /* -0.5f */, 16384 /* 0.50001525925f */}); - algorithm.mix( + AudioMixingUtil.mix( sourceBuffer, - AUDIO_FORMAT_MONO_PCM_16BIT, + MONO_44100_PCM_16BIT, + mixingBuffer, + STEREO_44100_PCM_FLOAT, MONO_TO_STEREO.scaleBy(0.5f), - /* frameCount= */ 2, - mixingBuffer); + /* framesToMix= */ 2, + /* accumulate= */ true); assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); - mixingBuffer.flip(); + mixingBuffer.rewind(); assertThat(createFloatArray(mixingBuffer)) .usingTolerance(1f / Short.MAX_VALUE) .containsExactly(new float[] {0f, -0.5f, 0.75f, -0.25f}) @@ -189,45 +144,47 @@ public final class FloatAudioMixingAlgorithmTest { } @Test - public void mixStereoFloatIntoMonoFloat() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_MONO_PCM_FLOAT); + public void mixToMonoFloat_withStereoFloatInput() { ByteBuffer mixingBuffer = createByteBuffer(new float[] {0.25f, 0.5f}); ByteBuffer sourceBuffer = createByteBuffer(new float[] {-0.5f, 0.25f, -0.25f, 0.5f}); - algorithm.mix( + AudioMixingUtil.mix( sourceBuffer, - AUDIO_FORMAT_STEREO_PCM_FLOAT, + STEREO_44100_PCM_FLOAT, + mixingBuffer, + MONO_44100_PCM_FLOAT, STEREO_TO_MONO.scaleBy(0.5f), - /* frameCount= */ 2, - mixingBuffer); + /* framesToMix= */ 2, + /* accumulate= */ true); assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); - mixingBuffer.flip(); + mixingBuffer.rewind(); assertThat(createFloatArray(mixingBuffer)).isEqualTo(new float[] {0.1875f, 0.5625f}); } @Test - public void mixMonoFloatIntoMonoFloat() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_MONO_PCM_FLOAT); + public void mixToMonoFloat_withMonoFloatInput() { ByteBuffer mixingBuffer = createByteBuffer(new float[] {0.25f, -0.25f}); ByteBuffer sourceBuffer = createByteBuffer(new float[] {0.5f, 0.25f}); - algorithm.mix( + + AudioMixingUtil.mix( sourceBuffer, - AUDIO_FORMAT_MONO_PCM_FLOAT, + MONO_44100_PCM_FLOAT, + mixingBuffer, + MONO_44100_PCM_FLOAT, MONO_TO_MONO.scaleBy(0.5f), - /* frameCount= */ 2, - mixingBuffer); + /* framesToMix= */ 2, + /* accumulate= */ true); assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); - mixingBuffer.flip(); + mixingBuffer.rewind(); assertThat(createFloatArray(mixingBuffer)).isEqualTo(new float[] {0.5f, -0.125f}); } @Test - public void mixStereoS16IntoMonoFloat() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_MONO_PCM_FLOAT); + public void mixToMonoFloat_withStereo16Input() { ByteBuffer mixingBuffer = createByteBuffer(new float[] {0.25f, 0.5f}); ByteBuffer sourceBuffer = createByteBuffer( @@ -238,16 +195,18 @@ public final class FloatAudioMixingAlgorithmTest { 16384 /* 0.50001525925f */ }); - algorithm.mix( + AudioMixingUtil.mix( sourceBuffer, - AUDIO_FORMAT_STEREO_PCM_16BIT, + STEREO_44100_PCM_16BIT, + mixingBuffer, + MONO_44100_PCM_FLOAT, STEREO_TO_MONO.scaleBy(0.5f), - /* frameCount= */ 2, - mixingBuffer); + /* framesToMix= */ 2, + /* accumulate= */ true); assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); - mixingBuffer.flip(); + mixingBuffer.rewind(); assertThat(createFloatArray(mixingBuffer)) .usingTolerance(1f / Short.MAX_VALUE) .containsExactly(new float[] {0.1875f, 0.5625f}) @@ -255,25 +214,152 @@ public final class FloatAudioMixingAlgorithmTest { } @Test - public void mixMonoS16IntoMonoFloat() { - AudioMixingAlgorithm algorithm = new FloatAudioMixingAlgorithm(AUDIO_FORMAT_MONO_PCM_FLOAT); + public void mixToMonoFloat_withMono16Input() { ByteBuffer mixingBuffer = createByteBuffer(new float[] {0.25f, 0.5f}); ByteBuffer sourceBuffer = createByteBuffer(new short[] {-16384 /* -0.5f */, 8192 /* 0.25000762962f */}); - algorithm.mix( + AudioMixingUtil.mix( sourceBuffer, - AUDIO_FORMAT_MONO_PCM_16BIT, + MONO_44100_PCM_16BIT, + mixingBuffer, + MONO_44100_PCM_FLOAT, MONO_TO_MONO.scaleBy(0.5f), - /* frameCount= */ 2, - mixingBuffer); + /* framesToMix= */ 2, + /* accumulate= */ true); assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); - mixingBuffer.flip(); + mixingBuffer.rewind(); assertThat(createFloatArray(mixingBuffer)) .usingTolerance(1f / Short.MAX_VALUE) .containsExactly(new float[] {0f, 0.625f}) .inOrder(); } + + @Test + public void mixToStereo16_withMono16Input() { + ByteBuffer mixingBuffer = createByteBuffer(new short[] {0, 0, 0, 0, 0, 0}); + ByteBuffer sourceBuffer = createByteBuffer(new short[] {-1000, -6004, 33}); + ByteBuffer expectedBuffer = createByteBuffer(new short[] {-1000, -1000, -6004, -6004, 33, 33}); + + AudioMixingUtil.mix( + sourceBuffer, + MONO_44100_PCM_16BIT, + mixingBuffer, + STEREO_44100_PCM_16BIT, + MONO_TO_STEREO, + /* framesToMix= */ 3, + /* accumulate= */ true); + + assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); + assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); + + mixingBuffer.rewind(); + assertThat(mixingBuffer).isEqualTo(expectedBuffer); + } + + @Test + public void mixToMono16_withMono16Input() { + ByteBuffer mixingBuffer = createByteBuffer(new short[] {-10, 50, 12, -12}); + ByteBuffer sourceBuffer = createByteBuffer(new short[] {128, -66}); + ByteBuffer expectedBuffer = createByteBuffer(new short[] {118, -16, 12, -12}); + + AudioMixingUtil.mix( + sourceBuffer, + MONO_44100_PCM_16BIT, + mixingBuffer, + MONO_44100_PCM_16BIT, + MONO_TO_MONO, + /* framesToMix= */ 2, + /* accumulate= */ true); + + assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); + assertWithMessage("Mixing buffer") + .that(mixingBuffer.remaining()) + .isEqualTo(2 * MONO_44100_PCM_16BIT.bytesPerFrame); + + mixingBuffer.rewind(); + assertThat(mixingBuffer).isEqualTo(expectedBuffer); + } + + @Test + public void mixToMono16_withMono16Input_clamps() { + ByteBuffer mixingBuffer = + createByteBuffer( + new short[] {Short.MAX_VALUE, Short.MAX_VALUE, Short.MIN_VALUE, Short.MIN_VALUE}); + + ByteBuffer sourceBuffer = createByteBuffer(new short[] {1, -1, 1, -1}); + + ByteBuffer expectedBuffer = + createByteBuffer( + new short[] { + Short.MAX_VALUE, Short.MAX_VALUE - 1, Short.MIN_VALUE + 1, Short.MIN_VALUE + }); + + AudioMixingUtil.mix( + sourceBuffer, + MONO_44100_PCM_16BIT, + mixingBuffer, + MONO_44100_PCM_16BIT, + MONO_TO_MONO, + /* framesToMix= */ 4, + /* accumulate= */ true); + + assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); + assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); + + mixingBuffer.rewind(); + assertThat(mixingBuffer).isEqualTo(expectedBuffer); + } + + @Test + public void mixToStereo16_withStereo16Input() { + ByteBuffer mixingBuffer = createByteBuffer(new short[] {-4, 4, -512, 821, 0, -422}); + ByteBuffer sourceBuffer = + createByteBuffer(new short[] {26000, -26423, -5723, -5723, 23, 12312}); + ByteBuffer expectedBuffer = + createByteBuffer(new short[] {25996, -26419, -6235, -4902, 23, 11890}); + + AudioMixingUtil.mix( + sourceBuffer, + STEREO_44100_PCM_16BIT, + mixingBuffer, + STEREO_44100_PCM_16BIT, + STEREO_TO_STEREO, + /* framesToMix= */ 3, + /* accumulate= */ true); + + assertWithMessage("Source buffer").that(sourceBuffer.remaining()).isEqualTo(0); + assertWithMessage("Mixing buffer").that(mixingBuffer.remaining()).isEqualTo(0); + + mixingBuffer.rewind(); + assertThat(mixingBuffer).isEqualTo(expectedBuffer); + } + + @Test + public void mixToStereo16_withStereo16Input_noAccumulation() { + ByteBuffer mixingBuffer = createByteBuffer(new short[] {-4, 4, -512, 821, 0, -422}); + ByteBuffer sourceBuffer = createByteBuffer(new short[] {260, -26423, -5723, -5723, 23, 12312}); + ByteBuffer expectedBuffer = createByteBuffer(new short[] {260, -26423, -5723, -5723, 0, -422}); + + AudioMixingUtil.mix( + sourceBuffer, + STEREO_44100_PCM_16BIT, + mixingBuffer, + STEREO_44100_PCM_16BIT, + STEREO_TO_STEREO, + /* framesToMix= */ 2, + /* accumulate= */ false); + + assertWithMessage("Source buffer") + .that(sourceBuffer.remaining()) + .isEqualTo(STEREO_44100_PCM_16BIT.bytesPerFrame); + assertWithMessage("Mixing buffer") + .that(mixingBuffer.remaining()) + .isEqualTo(STEREO_44100_PCM_16BIT.bytesPerFrame); + + mixingBuffer.rewind(); + assertThat(mixingBuffer).isEqualTo(expectedBuffer); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java index e34552747c..b4b4f24fa7 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixerImpl.java @@ -16,14 +16,13 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; -import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static java.lang.Math.min; import android.util.SparseArray; -import androidx.annotation.Nullable; import androidx.media3.common.C; +import androidx.media3.common.audio.AudioMixingUtil; import androidx.media3.common.audio.AudioProcessor.AudioFormat; import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; import androidx.media3.common.audio.ChannelMixingMatrix; @@ -40,7 +39,6 @@ import java.nio.ByteOrder; private final SparseArray sources; private int nextSourceId; private AudioFormat outputAudioFormat; - @Nullable private AudioMixingAlgorithm mixingAlgorithm; private int bufferSizeFrames; private MixingBuffer[] mixingBuffers; private long mixerStartTimeUs; @@ -67,10 +65,13 @@ import java.nio.ByteOrder; @Override public void configure(AudioFormat outputAudioFormat, int bufferSizeMs, long startTimeUs) throws UnhandledAudioFormatException { - checkState(!isConfigured(), "Audio mixer already configured."); + checkState( + this.outputAudioFormat.equals(AudioFormat.NOT_SET), "Audio mixer already configured."); - // Create algorithm first in case it throws. - mixingAlgorithm = AudioMixingAlgorithm.create(outputAudioFormat); + if (!AudioMixingUtil.canMix(outputAudioFormat)) { + throw new UnhandledAudioFormatException( + "Can not mix to this AudioFormat.", outputAudioFormat); + } this.outputAudioFormat = outputAudioFormat; bufferSizeFrames = bufferSizeMs * outputAudioFormat.sampleRate / 1000; mixerStartTimeUs = startTimeUs; @@ -97,7 +98,7 @@ import java.nio.ByteOrder; @Override public boolean supportsSourceAudioFormat(AudioFormat sourceFormat) { checkStateIsConfigured(); - return checkStateNotNull(mixingAlgorithm).supportsSourceAudioFormat(sourceFormat); + return AudioMixingUtil.canMix(sourceFormat, outputAudioFormat); } @Override @@ -174,8 +175,8 @@ import java.nio.ByteOrder; source.mixTo( sourceBuffer, min(newSourcePosition, mixingBuffer.limit), - checkNotNull(mixingAlgorithm), - mixingBuffer.buffer); + mixingBuffer.buffer, + outputAudioFormat); mixingBuffer.buffer.reset(); if (source.position == newSourcePosition) { @@ -228,7 +229,6 @@ import java.nio.ByteOrder; sources.clear(); nextSourceId = 0; outputAudioFormat = AudioFormat.NOT_SET; - mixingAlgorithm = null; bufferSizeFrames = C.LENGTH_UNSET; mixingBuffers = new MixingBuffer[0]; mixerStartTimeUs = C.TIME_UNSET; @@ -237,12 +237,8 @@ import java.nio.ByteOrder; endPosition = Long.MAX_VALUE; } - private boolean isConfigured() { - return mixingAlgorithm != null; - } - private void checkStateIsConfigured() { - checkState(isConfigured(), "Audio mixer is not configured."); + checkState(!outputAudioFormat.equals(AudioFormat.NOT_SET), "Audio mixer is not configured."); } private MixingBuffer allocateMixingBuffer(long position) { @@ -331,12 +327,18 @@ import java.nio.ByteOrder; public void mixTo( ByteBuffer sourceBuffer, long newPosition, - AudioMixingAlgorithm mixingAlgorithm, - ByteBuffer mixingBuffer) { + ByteBuffer mixingBuffer, + AudioFormat mixingAudioFormat) { checkArgument(newPosition >= position); int framesToMix = (int) (newPosition - position); - mixingAlgorithm.mix( - sourceBuffer, audioFormat, channelMixingMatrix, framesToMix, mixingBuffer); + AudioMixingUtil.mix( + sourceBuffer, + audioFormat, + mixingBuffer, + mixingAudioFormat, + channelMixingMatrix, + framesToMix, + /* accumulate= */ true); position = newPosition; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixingAlgorithm.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixingAlgorithm.java deleted file mode 100644 index 3a8887dafe..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioMixingAlgorithm.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import android.annotation.SuppressLint; -import androidx.media3.common.C; -import androidx.media3.common.audio.AudioProcessor.AudioFormat; -import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; -import androidx.media3.common.audio.ChannelMixingMatrix; -import androidx.media3.common.util.UnstableApi; -import com.google.errorprone.annotations.CanIgnoreReturnValue; -import java.nio.ByteBuffer; - -/** - * Algorithm for mixing source audio buffers into an audio mixing buffer. - * - *

Each instance is parameterized by the mixing (output) audio format provided to {@link - * #create(AudioFormat)}. An instance may support multiple source audio formats queried via {@link - * #supportsSourceAudioFormat(AudioFormat)}. - * - *

All implementations are stateless and can work with any number of source and mixing buffers. - */ -@UnstableApi -/* package */ interface AudioMixingAlgorithm { - - /** Indicates whether the algorithm supports mixing source buffers with the given audio format. */ - boolean supportsSourceAudioFormat(AudioFormat sourceAudioFormat); - - /** - * Mixes audio from {@code sourceBuffer} into {@code mixingBuffer}. - * - *

The method will read from {@code sourceBuffer} and write to {@code mixingBuffer}, advancing - * the positions of both. The frame count must be in bounds for both buffers. - * - *

The {@code channelMixingMatrix} input and output channel counts must match the channel count - * of the source audio format and mixing audio format respectively. - * - * @param sourceBuffer Source audio. - * @param sourceAudioFormat {@link AudioFormat} of {@code sourceBuffer}. Must be {@linkplain - * #supportsSourceAudioFormat(AudioFormat) supported}. - * @param channelMixingMatrix Scaling factors applied to source samples before mixing. - * @param frameCount Number of audio frames to mix. - * @param mixingBuffer Mixing buffer. - */ - @CanIgnoreReturnValue - ByteBuffer mix( - ByteBuffer sourceBuffer, - AudioFormat sourceAudioFormat, - ChannelMixingMatrix channelMixingMatrix, - int frameCount, - ByteBuffer mixingBuffer); - - /** - * Creates an instance that mixes into the given audio format. - * - * @param mixingAudioFormat The format of audio in the mixing buffer. - * @return The new algorithm instance. - * @throws UnhandledAudioFormatException If the specified format is not supported for mixing. - */ - @SuppressLint("SwitchIntDef") - public static AudioMixingAlgorithm create(AudioFormat mixingAudioFormat) - throws UnhandledAudioFormatException { - switch (mixingAudioFormat.encoding) { - case C.ENCODING_PCM_FLOAT: - return new FloatAudioMixingAlgorithm(mixingAudioFormat); - default: - throw new UnhandledAudioFormatException( - "No supported mixing algorithm available.", mixingAudioFormat); - } - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FloatAudioMixingAlgorithm.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FloatAudioMixingAlgorithm.java deleted file mode 100644 index c919d07db7..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FloatAudioMixingAlgorithm.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static androidx.media3.common.util.Assertions.checkArgument; - -import android.annotation.SuppressLint; -import androidx.media3.common.C; -import androidx.media3.common.Format; -import androidx.media3.common.audio.AudioProcessor.AudioFormat; -import androidx.media3.common.audio.ChannelMixingMatrix; -import java.nio.ByteBuffer; - -/** An {@link AudioMixingAlgorithm} which mixes into float samples. */ -/* package */ class FloatAudioMixingAlgorithm implements AudioMixingAlgorithm { - - // Short.MIN_VALUE != -Short.MAX_VALUE so use different scaling factors for positive and - // negative samples. - private static final float SCALE_S16_FOR_NEGATIVE_INPUT = -1f / Short.MIN_VALUE; - private static final float SCALE_S16_FOR_POSITIVE_INPUT = 1f / Short.MAX_VALUE; - - private final AudioFormat mixingAudioFormat; - - public FloatAudioMixingAlgorithm(AudioFormat mixingAudioFormat) { - checkArgument(mixingAudioFormat.encoding == C.ENCODING_PCM_FLOAT); - checkArgument(mixingAudioFormat.channelCount != Format.NO_VALUE); - this.mixingAudioFormat = mixingAudioFormat; - } - - @Override - @SuppressLint("SwitchIntDef") - public boolean supportsSourceAudioFormat(AudioFormat sourceAudioFormat) { - if (sourceAudioFormat.sampleRate != mixingAudioFormat.sampleRate) { - return false; - } - switch (sourceAudioFormat.encoding) { - case C.ENCODING_PCM_16BIT: - case C.ENCODING_PCM_FLOAT: - return true; - default: - return false; - } - } - - @Override - @SuppressLint("SwitchIntDef") - public ByteBuffer mix( - ByteBuffer sourceBuffer, - AudioFormat sourceAudioFormat, - ChannelMixingMatrix channelMixingMatrix, - int frameCount, - ByteBuffer mixingBuffer) { - checkArgument( - supportsSourceAudioFormat(sourceAudioFormat), "Source audio format is not supported."); - checkArgument( - channelMixingMatrix.getInputChannelCount() == sourceAudioFormat.channelCount, - "Input channel count does not match source format."); - checkArgument( - channelMixingMatrix.getOutputChannelCount() == mixingAudioFormat.channelCount, - "Output channel count does not match mixing format."); - checkArgument( - sourceBuffer.remaining() >= frameCount * sourceAudioFormat.bytesPerFrame, - "Source buffer is too small."); - checkArgument( - mixingBuffer.remaining() >= frameCount * mixingAudioFormat.bytesPerFrame, - "Mixing buffer is too small."); - - switch (sourceAudioFormat.encoding) { - case C.ENCODING_PCM_FLOAT: - return mixFloatIntoFloat(sourceBuffer, channelMixingMatrix, frameCount, mixingBuffer); - case C.ENCODING_PCM_16BIT: - return mixS16IntoFloat(sourceBuffer, channelMixingMatrix, frameCount, mixingBuffer); - default: - throw new IllegalArgumentException("Source encoding is not supported."); - } - } - - private static ByteBuffer mixFloatIntoFloat( - ByteBuffer sourceBuffer, - ChannelMixingMatrix channelMixingMatrix, - int frameCount, - ByteBuffer mixingBuffer) { - if (channelMixingMatrix.isDiagonal()) { - return mixFloatIntoFloatDiagonal(sourceBuffer, channelMixingMatrix, frameCount, mixingBuffer); - } - int sourceChannelCount = channelMixingMatrix.getInputChannelCount(); - float[] sourceFrame = new float[sourceChannelCount]; - for (int i = 0; i < frameCount; i++) { - for (int sourceChannel = 0; sourceChannel < sourceChannelCount; sourceChannel++) { - sourceFrame[sourceChannel] = sourceBuffer.getFloat(); - } - mixFloatFrameIntoFloat(sourceFrame, channelMixingMatrix, mixingBuffer); - } - return mixingBuffer; - } - - private static void mixFloatFrameIntoFloat( - float[] sourceFrame, ChannelMixingMatrix channelMixingMatrix, ByteBuffer mixingBuffer) { - int mixingChannelCount = channelMixingMatrix.getOutputChannelCount(); - for (int mixingChannel = 0; mixingChannel < mixingChannelCount; mixingChannel++) { - float mixedSample = mixingBuffer.getFloat(mixingBuffer.position()); - for (int sourceChannel = 0; sourceChannel < sourceFrame.length; sourceChannel++) { - mixedSample += - channelMixingMatrix.getMixingCoefficient(sourceChannel, mixingChannel) - * sourceFrame[sourceChannel]; - } - mixingBuffer.putFloat(mixedSample); - } - } - - private static ByteBuffer mixFloatIntoFloatDiagonal( - ByteBuffer sourceBuffer, - ChannelMixingMatrix channelMixingMatrix, - int frameCount, - ByteBuffer mixingBuffer) { - int channelCount = channelMixingMatrix.getInputChannelCount(); - for (int i = 0; i < frameCount; i++) { - for (int c = 0; c < channelCount; c++) { - float sourceSample = sourceBuffer.getFloat(); - float mixedSample = - mixingBuffer.getFloat(mixingBuffer.position()) - + channelMixingMatrix.getMixingCoefficient(c, c) * sourceSample; - mixingBuffer.putFloat(mixedSample); - } - } - return mixingBuffer; - } - - private static ByteBuffer mixS16IntoFloat( - ByteBuffer sourceBuffer, - ChannelMixingMatrix channelMixingMatrix, - int frameCount, - ByteBuffer mixingBuffer) { - if (channelMixingMatrix.isDiagonal()) { - return mixS16IntoFloatDiagonal(sourceBuffer, channelMixingMatrix, frameCount, mixingBuffer); - } - int sourceChannelCount = channelMixingMatrix.getInputChannelCount(); - float[] sourceFrame = new float[sourceChannelCount]; - for (int i = 0; i < frameCount; i++) { - for (int sourceChannel = 0; sourceChannel < sourceChannelCount; sourceChannel++) { - sourceFrame[sourceChannel] = s16ToFloat(sourceBuffer.getShort()); - } - mixFloatFrameIntoFloat(sourceFrame, channelMixingMatrix, mixingBuffer); - } - return mixingBuffer; - } - - private static ByteBuffer mixS16IntoFloatDiagonal( - ByteBuffer sourceBuffer, - ChannelMixingMatrix channelMixingMatrix, - int frameCount, - ByteBuffer mixingBuffer) { - int channelCount = channelMixingMatrix.getInputChannelCount(); - for (int i = 0; i < frameCount; i++) { - for (int c = 0; c < channelCount; c++) { - float sourceSample = s16ToFloat(sourceBuffer.getShort()); - float mixedSample = - mixingBuffer.getFloat(mixingBuffer.position()) - + channelMixingMatrix.getMixingCoefficient(c, c) * sourceSample; - mixingBuffer.putFloat(mixedSample); - } - } - return mixingBuffer; - } - - private static float s16ToFloat(short shortValue) { - return shortValue - * (shortValue < 0 ? SCALE_S16_FOR_NEGATIVE_INPUT : SCALE_S16_FOR_POSITIVE_INPUT); - } -}