diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 47cd65022c..68c44f8b61 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -9,6 +9,7 @@ * Extractors: * DataSource: * Audio: + * Allow constant power upmixing/downmixing in DefaultAudioMixer. * Video: * Text: * Metadata: diff --git a/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingMatrix.java b/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingMatrix.java index a4c1753b19..120aa5d1e6 100644 --- a/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingMatrix.java +++ b/libraries/common/src/main/java/androidx/media3/common/audio/ChannelMixingMatrix.java @@ -17,6 +17,7 @@ package androidx.media3.common.audio; import static androidx.media3.common.util.Assertions.checkArgument; +import androidx.annotation.IntRange; import androidx.media3.common.util.UnstableApi; import com.google.errorprone.annotations.CanIgnoreReturnValue; @@ -52,21 +53,41 @@ public final class ChannelMixingMatrix { private final boolean isIdentity; /** - * Creates a basic channel mixing matrix that converts from {@code inputChannelCount} channels to - * {@code outputChannelCount} channels. + * Returns a default constant gain channel mixing matrix that mixes {@code inputChannelCount} + * channels into {@code outputChannelCount} channels. * - *

If the input and output channel counts match then a simple identity matrix will be returned. - * Otherwise, default matrix coefficients will be used to best match channel locations and overall - * power level. + *

This method returns an identity matrix if {@code inputChannelCount} and {@code + * outputChannelCount} are equal. * * @param inputChannelCount Number of input channels. * @param outputChannelCount Number of output channels. - * @return New channel mixing matrix. - * @throws UnsupportedOperationException If no default matrix coefficients are implemented for the - * given input and output channel counts. + * @throws UnsupportedOperationException If no default coefficients are available for the given + * input and output channel counts. + * @deprecated Use {@link #createForConstantGain} instead. */ - // TODO(b/300467493): Modify create() to use constant power defaults and migrate all users. - public static ChannelMixingMatrix create(int inputChannelCount, int outputChannelCount) { + // TODO(b/399861060): Remove in Media3 1.8. + @Deprecated + public static ChannelMixingMatrix create( + @IntRange(from = 1, to = 2) int inputChannelCount, + @IntRange(from = 1, to = 2) int outputChannelCount) { + return createForConstantGain(inputChannelCount, outputChannelCount); + } + + /** + * Returns a default constant gain channel mixing matrix that mixes {@code inputChannelCount} + * channels into {@code outputChannelCount} channels. + * + *

This method returns an identity matrix if {@code inputChannelCount} and {@code + * outputChannelCount} are equal. + * + * @param inputChannelCount Number of input channels. + * @param outputChannelCount Number of output channels. + * @throws UnsupportedOperationException If no default coefficients are available for the given + * input and output channel counts. + */ + public static ChannelMixingMatrix createForConstantGain( + @IntRange(from = 1, to = 2) int inputChannelCount, + @IntRange(from = 1, to = 2) int outputChannelCount) { return new ChannelMixingMatrix( inputChannelCount, outputChannelCount, @@ -74,19 +95,31 @@ public final class ChannelMixingMatrix { } /** - * Returns default constant power matrix for mixing {@code inputChannelCount} channels into {@code - * outputChannelCount} channels. + * Returns a default constant power channel mixing matrix that mixes {@code inputChannelCount} + * channels into {@code outputChannelCount} channels. * - *

If the input and output channel counts match then a simple identity matrix will be returned. + *

This method returns an identity matrix if {@code inputChannelCount} and {@code + * outputChannelCount} are equal. + * + *

Channel counts map to the following layouts: + * + *

    + *
  1. [MONO] + *
  2. [FRONT_LEFT, FRONT_RIGHT] + *
  3. [FRONT_LEFT, FRONT_RIGHT, FRONT_CENTER] + *
  4. [FRONT_LEFT, FRONT_RIGHT, BACK_LEFT, BACK_RIGHT] + *
  5. [FRONT_LEFT, FRONT_RIGHT, FRONT_CENTER, BACK_LEFT, BACK_RIGHT] + *
  6. [FRONT_LEFT, FRONT_RIGHT, FRONT_CENTER, LOW_FREQUENCY, BACK_LEFT, BACK_RIGHT] + *
* * @param inputChannelCount Number of input channels. * @param outputChannelCount Number of output channels. - * @return New channel mixing matrix. - * @throws UnsupportedOperationException If no default coefficients are available for the given - * input and output channel counts. + * @throws UnsupportedOperationException If no default matrix coefficients are implemented for the + * given input and output channel counts. */ public static ChannelMixingMatrix createForConstantPower( - int inputChannelCount, int outputChannelCount) { + @IntRange(from = 1, to = 6) int inputChannelCount, + @IntRange(from = 1, to = 2) int outputChannelCount) { return new ChannelMixingMatrix( inputChannelCount, outputChannelCount, diff --git a/libraries/common/src/test/java/androidx/media3/common/audio/AudioMixingUtilTest.java b/libraries/common/src/test/java/androidx/media3/common/audio/AudioMixingUtilTest.java index c8504345a6..1e4024b722 100644 --- a/libraries/common/src/test/java/androidx/media3/common/audio/AudioMixingUtilTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/audio/AudioMixingUtilTest.java @@ -41,13 +41,17 @@ public final class AudioMixingUtilTest { new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 1, C.ENCODING_PCM_16BIT); private static final ChannelMixingMatrix STEREO_TO_STEREO = - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 2, /* outputChannelCount= */ 2); private static final ChannelMixingMatrix MONO_TO_STEREO = - ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 1, /* outputChannelCount= */ 2); private static final ChannelMixingMatrix STEREO_TO_MONO = - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 2, /* outputChannelCount= */ 1); private static final ChannelMixingMatrix MONO_TO_MONO = - ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 1); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 1, /* outputChannelCount= */ 1); @Test public void mixToStereoFloat_withStereoFloatInput() { diff --git a/libraries/common/src/test/java/androidx/media3/common/audio/ChannelMixingAudioProcessorTest.java b/libraries/common/src/test/java/androidx/media3/common/audio/ChannelMixingAudioProcessorTest.java index 324584d77a..f0b0c5ced6 100644 --- a/libraries/common/src/test/java/androidx/media3/common/audio/ChannelMixingAudioProcessorTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/audio/ChannelMixingAudioProcessorTest.java @@ -41,9 +41,11 @@ public final class ChannelMixingAudioProcessorTest { public void setUp() { audioProcessor = new ChannelMixingAudioProcessor(); audioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); audioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); } @Test @@ -102,7 +104,8 @@ public final class ChannelMixingAudioProcessorTest { @Test public void configureWithIdentityMatrix_isActiveReturnsFalse() throws Exception { audioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2)); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 2, /* outputChannelCount= */ 2)); audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); assertThat(audioProcessor.isActive()).isFalse(); @@ -133,7 +136,8 @@ public final class ChannelMixingAudioProcessorTest { @Test public void scaledMixingMatrix_queueInput_outputIsScaled() throws Exception { audioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2) + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 2, /* outputChannelCount= */ 2) .scaleBy(0.5f)); audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/WaveformAudioBufferSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/WaveformAudioBufferSink.java index c93fadf662..8aa5d55e1f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/WaveformAudioBufferSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/WaveformAudioBufferSink.java @@ -104,7 +104,7 @@ public class WaveformAudioBufferSink implements TeeAudioProcessor.AudioBufferSin * @param barsPerSecond The number of bars that should be generated per each second of audio. * @param outputChannelCount The number of channels that the output waveform should contain. If * this is different than the number of input channels, the audio will be mixed using the - * {@linkplain ChannelMixingMatrix#create default mixing matrix}. + * {@linkplain ChannelMixingMatrix#createForConstantGain default mixing matrix}. * @param listener The listener to be notified when a new waveform bar has been generated. */ public WaveformAudioBufferSink(int barsPerSecond, int outputChannelCount, Listener listener) { @@ -123,7 +123,8 @@ public class WaveformAudioBufferSink implements TeeAudioProcessor.AudioBufferSin samplesPerBar = sampleRateHz / barsPerSecond; inputAudioFormat = new AudioFormat(sampleRateHz, channelCount, encoding); mixingAudioFormat = new AudioFormat(sampleRateHz, outputChannels.size(), C.ENCODING_PCM_FLOAT); - channelMixingMatrix = ChannelMixingMatrix.create(channelCount, outputChannels.size()); + channelMixingMatrix = + ChannelMixingMatrix.createForConstantGain(channelCount, outputChannels.size()); } @Override diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 0de771c3ec..968d3d0609 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -1739,7 +1739,8 @@ public class TransformerEndToEndTest { ChannelMixingAudioProcessor channelMixingAudioProcessor = new ChannelMixingAudioProcessor(); channelMixingAudioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET.uri))) .setRemoveVideo(true) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerWithInAppMp4MuxerEndToEndAndroidTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerWithInAppMp4MuxerEndToEndAndroidTest.java index 74fba5f27c..4e30d003f2 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerWithInAppMp4MuxerEndToEndAndroidTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerWithInAppMp4MuxerEndToEndAndroidTest.java @@ -92,7 +92,8 @@ public class TransformerWithInAppMp4MuxerEndToEndAndroidTest { new Transformer.Builder(context).setMuxerFactory(new InAppMp4Muxer.Factory()).build(); ChannelMixingAudioProcessor channelMixingAudioProcessor = new ChannelMixingAudioProcessor(); channelMixingAudioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)); MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_FILE_ASSET_DIRECTORY + H264_MP4)); EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeForegroundSpeedTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeForegroundSpeedTest.java index 7ef93e673f..f2b72a3c47 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeForegroundSpeedTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeForegroundSpeedTest.java @@ -169,7 +169,8 @@ public class TranscodeForegroundSpeedTest { sonicAudioProcessor.setOutputSampleRateHz(44_100); ChannelMixingAudioProcessor mixingAudioProcessor = new ChannelMixingAudioProcessor(); mixingAudioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); + ChannelMixingMatrix.createForConstantGain( + /* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem) .setEffects( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInput.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInput.java index 89549ba9a5..7bf64d1f70 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInput.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInput.java @@ -536,10 +536,10 @@ import java.util.concurrent.atomic.AtomicLong; || requiredOutputAudioFormat.channelCount == 2) { ChannelMixingAudioProcessor channelCountChanger = new ChannelMixingAudioProcessor(); channelCountChanger.putChannelMixingMatrix( - ChannelMixingMatrix.create( + ChannelMixingMatrix.createForConstantGain( /* inputChannelCount= */ 1, requiredOutputAudioFormat.channelCount)); channelCountChanger.putChannelMixingMatrix( - ChannelMixingMatrix.create( + ChannelMixingMatrix.createForConstantGain( /* inputChannelCount= */ 2, requiredOutputAudioFormat.channelCount)); audioProcessors.add(channelCountChanger); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAudioMixer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAudioMixer.java index aa5c426cf4..a8a90d8580 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAudioMixer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAudioMixer.java @@ -16,6 +16,8 @@ package androidx.media3.transformer; import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER; +import static androidx.media3.common.audio.ChannelMixingMatrix.createForConstantGain; +import static androidx.media3.common.audio.ChannelMixingMatrix.createForConstantPower; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Util.contains; @@ -51,13 +53,36 @@ public final class DefaultAudioMixer implements AudioMixer { public static final class Factory implements AudioMixer.Factory { private final boolean outputSilenceWithNoSources; private final boolean clipFloatOutput; + private final boolean useConstantPowerMixingMatrices; /** - * Creates an instance. This is equivalent to {@link #Factory(boolean, boolean) new - * Factory(false, true)}. + * Creates an instance. This is equivalent to {@link #Factory(boolean, boolean, boolean) new + * Factory(false, true, false)}. */ public Factory() { - this(/* outputSilenceWithNoSources= */ false, /* clipFloatOutput= */ true); + this( + /* outputSilenceWithNoSources= */ false, + /* clipFloatOutput= */ true, + /* useConstantPowerMixingMatrices= */ false); + } + + /** + * Creates an instance. This is equivalent to {@link #Factory(boolean, boolean, boolean) new + * Factory(outputSilenceWithNoSources, clipFloatOutput, false)}. + * + * @param outputSilenceWithNoSources Whether to {@linkplain #getOutput() output} silence when + * there are no {@linkplain #addSource sources}. + * @param clipFloatOutput Whether to clip the output signal to be in the [-1.0, 1.0] range if + * the output encoding is {@link C#ENCODING_PCM_FLOAT}. This parameter is ignored for + * non-float output signals. For float output signals, non-float input signals are converted + * to float signals in the [-1.0, 1.0] range. All input signals (float or non-float) are + * then added and the result is clipped if and only if {@code clipFloatOutput} is true. + */ + public Factory(boolean outputSilenceWithNoSources, boolean clipFloatOutput) { + this( + /* outputSilenceWithNoSources= */ outputSilenceWithNoSources, + /* clipFloatOutput= */ clipFloatOutput, + /* useConstantPowerMixingMatrices= */ false); } /** @@ -70,15 +95,24 @@ public final class DefaultAudioMixer implements AudioMixer { * non-float output signals. For float output signals, non-float input signals are converted * to float signals in the [-1.0, 1.0] range. All input signals (float or non-float) are * then added and the result is clipped if and only if {@code clipFloatOutput} is true. + * @param useConstantPowerMixingMatrices Whether to upmix/downmix using {@linkplain + * ChannelMixingMatrix#createForConstantPower(int, int) constant power mixing matrices}. If + * {@code false}, uses {@linkplain ChannelMixingMatrix#createForConstantGain constant gain + * mixing matrices}. */ - public Factory(boolean outputSilenceWithNoSources, boolean clipFloatOutput) { + public Factory( + boolean outputSilenceWithNoSources, + boolean clipFloatOutput, + boolean useConstantPowerMixingMatrices) { this.outputSilenceWithNoSources = outputSilenceWithNoSources; this.clipFloatOutput = clipFloatOutput; + this.useConstantPowerMixingMatrices = useConstantPowerMixingMatrices; } @Override public DefaultAudioMixer create() { - return new DefaultAudioMixer(outputSilenceWithNoSources, clipFloatOutput); + return new DefaultAudioMixer( + outputSilenceWithNoSources, clipFloatOutput, useConstantPowerMixingMatrices); } } @@ -87,6 +121,7 @@ public final class DefaultAudioMixer implements AudioMixer { private final boolean outputSilenceWithNoSources; private final boolean clipFloatOutput; + private final boolean useConstantPowerMixingMatrices; private final SparseArray sources; private int nextSourceId; private AudioFormat outputAudioFormat; @@ -109,9 +144,13 @@ public final class DefaultAudioMixer implements AudioMixer { */ private long maxPositionOfRemovedSources; - private DefaultAudioMixer(boolean outputSilenceWithNoSources, boolean clipFloatOutput) { + private DefaultAudioMixer( + boolean outputSilenceWithNoSources, + boolean clipFloatOutput, + boolean useConstantPowerMixingMatrices) { this.outputSilenceWithNoSources = outputSilenceWithNoSources; this.clipFloatOutput = clipFloatOutput; + this.useConstantPowerMixingMatrices = useConstantPowerMixingMatrices; sources = new SparseArray<>(); outputAudioFormat = AudioFormat.NOT_SET; bufferSizeFrames = C.LENGTH_UNSET; @@ -188,7 +227,9 @@ public final class DefaultAudioMixer implements AudioMixer { sourceId, new SourceInfo( sourceFormat, - ChannelMixingMatrix.create(sourceFormat.channelCount, outputAudioFormat.channelCount), + useConstantPowerMixingMatrices + ? createForConstantPower(sourceFormat.channelCount, outputAudioFormat.channelCount) + : createForConstantGain(sourceFormat.channelCount, outputAudioFormat.channelCount), startFrameOffset)); DebugTraceUtil.logEvent( diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultAudioMixerTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultAudioMixerTest.java index d4f7f884c8..6c1c8590b1 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultAudioMixerTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultAudioMixerTest.java @@ -68,7 +68,10 @@ public final class DefaultAudioMixerTest { @Before public void setup() { mixer = - new DefaultAudioMixer.Factory(outputSilenceWithNoSources, /* clipFloatOutput= */ true) + new DefaultAudioMixer.Factory( + outputSilenceWithNoSources, + /* clipFloatOutput= */ true, + /* useConstantPowerMixingMatrices= */ false) .create(); } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TestUtil.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TestUtil.java index 195ade200b..53869e80e8 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TestUtil.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TestUtil.java @@ -88,7 +88,7 @@ public final class TestUtil { ChannelMixingAudioProcessor audioProcessor = new ChannelMixingAudioProcessor(); for (int channel = 1; channel <= 6; channel++) { audioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create( + ChannelMixingMatrix.createForConstantGain( /* inputChannelCount= */ channel, /* outputChannelCount= */ channel) .scaleBy(scale)); } @@ -100,7 +100,7 @@ public final class TestUtil { ChannelMixingAudioProcessor audioProcessor = new ChannelMixingAudioProcessor(); for (int inputChannelCount = 1; inputChannelCount <= 2; inputChannelCount++) { audioProcessor.putChannelMixingMatrix( - ChannelMixingMatrix.create(inputChannelCount, outputChannelCount)); + ChannelMixingMatrix.createForConstantGain(inputChannelCount, outputChannelCount)); } return audioProcessor; }