From 92cff6432188e68b62699c367856774cde62ef7d Mon Sep 17 00:00:00 2001 From: ktrajkovski Date: Tue, 13 Aug 2024 09:31:22 -0700 Subject: [PATCH] Add spatial effects to IAMF support in Exoplayer. Check if the output device supports spatialization for the requested output format. If so, return a stream decoded for 6 channels in a 5.1 layout. Otherwise, return a stream decoded for 2 channels in a binaural layout. PiperOrigin-RevId: 662546818 --- .../media3/decoder/iamf/IamfDecoderTest.java | 3 +- .../media3/decoder/iamf/IamfPlaybackTest.java | 28 +++++++++- .../media3/decoder/iamf/IamfDecoder.java | 45 ++++++++++------ .../decoder/iamf/LibiamfAudioRenderer.java | 53 ++++++++++++------- .../decoder_iamf/src/main/jni/iamf_jni.cc | 13 ++--- .../exoplayer/DefaultRenderersFactory.java | 3 +- 6 files changed, 101 insertions(+), 44 deletions(-) diff --git a/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfDecoderTest.java b/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfDecoderTest.java index 89ea7880c6..f6ae3445a4 100644 --- a/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfDecoderTest.java +++ b/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfDecoderTest.java @@ -45,7 +45,8 @@ public final class IamfDecoderTest { @Test public void iamfBinauralLayoutChannelsCount_equalsTwo() throws Exception { - IamfDecoder iamf = new IamfDecoder(ImmutableList.of(IACB_OBUS)); + IamfDecoder iamf = + new IamfDecoder(ImmutableList.of(IACB_OBUS), /* spatializationSupported= */ false); assertThat(iamf.getBinauralLayoutChannelCount()) .isEqualTo(DEFAULT_BINAURAL_LAYOUT_CHANNEL_COUNT); diff --git a/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java b/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java index 7fc0047f1f..5f24c0c4bb 100644 --- a/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java +++ b/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java @@ -18,12 +18,17 @@ package androidx.media3.decoder.iamf; import static com.google.common.truth.Truth.assertWithMessage; import android.content.Context; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.Spatializer; import android.net.Uri; import android.os.Looper; import androidx.annotation.Nullable; +import androidx.media3.common.AudioAttributes; import androidx.media3.common.MediaItem; import androidx.media3.common.PlaybackException; import androidx.media3.common.Player; +import androidx.media3.common.util.Util; import androidx.media3.datasource.DefaultDataSource; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.Renderer; @@ -97,6 +102,26 @@ public class IamfPlaybackTest { @Override public void run() { Looper.prepare(); + if (Util.SDK_INT >= 32) { // Spatializer is only available on API 32 and above. + AudioManager audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + AudioFormat.Builder audioFormat = + new AudioFormat.Builder() + .setEncoding(IamfDecoder.OUTPUT_PCM_ENCODING) + .setChannelMask(IamfDecoder.SPATIALIZED_OUTPUT_LAYOUT); + if (audioManager != null) { + Spatializer spatializer = audioManager.getSpatializer(); + assertWithMessage("Spatializer must be disabled to run this test.") + .that( + spatializer.getImmersiveAudioLevel() + != Spatializer.SPATIALIZER_IMMERSIVE_LEVEL_NONE + && spatializer.isAvailable() + && spatializer.isEnabled() + && spatializer.canBeSpatialized( + AudioAttributes.DEFAULT.getAudioAttributesV21().audioAttributes, + audioFormat.build())) + .isFalse(); + } + } RenderersFactory renderersFactory = (eventHandler, videoRendererEventListener, @@ -104,7 +129,8 @@ public class IamfPlaybackTest { textRendererOutput, metadataRendererOutput) -> new Renderer[] { - new LibiamfAudioRenderer(eventHandler, audioRendererEventListener, audioSink) + new LibiamfAudioRenderer( + context, eventHandler, audioRendererEventListener, audioSink) }; player = new ExoPlayer.Builder(context, renderersFactory).build(); player.addListener(this); diff --git a/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java b/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java index a74f53b861..1f8f7cf2a0 100644 --- a/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java +++ b/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java @@ -17,6 +17,7 @@ package androidx.media3.decoder.iamf; import static android.support.annotation.VisibleForTesting.PACKAGE_PRIVATE; +import android.media.AudioFormat; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.util.Util; @@ -31,31 +32,39 @@ import javax.annotation.Nullable; @VisibleForTesting(otherwise = PACKAGE_PRIVATE) public final class IamfDecoder extends SimpleDecoder { - // TODO(ktrajkovski): Fetch channel count from the device instead of hardcoding. - /* package */ static final int DEFAULT_CHANNEL_COUNT = 2; - /* package */ static final int DEFAULT_OUTPUT_SAMPLE_RATE = 48000; - /* package */ static final @C.PcmEncoding int DEFAULT_PCM_ENCODING = C.ENCODING_PCM_16BIT; + /* package */ static final int OUTPUT_SAMPLE_RATE = 48000; + /* package */ static final int OUTPUT_PCM_ENCODING = AudioFormat.ENCODING_PCM_16BIT; + /* package */ static final int SPATIALIZED_OUTPUT_LAYOUT = AudioFormat.CHANNEL_OUT_5POINT1; + + // Matches IAMF_SoundSystem in IAMF_defines.h + private static final int SOUND_SYSTEM_STEREO = 0; // SOUND_SYSTEM_A + private static final int SOUND_SYSTEM_5POINT1 = 1; // SOUND_SYSTEM_B private final byte[] initializationData; + private final int soundSystem; /** * Creates an IAMF decoder. * * @param initializationData ConfigOBUs data for the decoder. + * @param spatializationSupported Whether spatialization is supported and output should be 6 + * channels in 5.1 layout. * @throws IamfDecoderException Thrown if an exception occurs when initializing the decoder. */ - public IamfDecoder(List initializationData) throws IamfDecoderException { + public IamfDecoder(List initializationData, boolean spatializationSupported) + throws IamfDecoderException { super(new DecoderInputBuffer[1], new SimpleDecoderOutputBuffer[1]); if (initializationData.size() != 1) { throw new IamfDecoderException("Initialization data must contain a single element."); } + soundSystem = spatializationSupported ? SOUND_SYSTEM_5POINT1 : SOUND_SYSTEM_STEREO; this.initializationData = initializationData.get(0); int status = iamfConfigDecoder( this.initializationData, - Util.getByteDepth(DEFAULT_PCM_ENCODING) * C.BITS_PER_BYTE, - DEFAULT_OUTPUT_SAMPLE_RATE, - DEFAULT_CHANNEL_COUNT); + Util.getByteDepth(OUTPUT_PCM_ENCODING) * C.BITS_PER_BYTE, + OUTPUT_SAMPLE_RATE, + soundSystem); if (status != 0) { throw new IamfDecoderException("Failed to configure decoder with returned status: " + status); } @@ -71,6 +80,10 @@ public final class IamfDecoder return iamfLayoutBinauralChannelsCount(); } + public int getChannelCount() { + return iamfGetChannelCount(soundSystem); + } + @Override public String getName() { return "libiamf"; @@ -98,13 +111,13 @@ public final class IamfDecoder if (reset) { iamfClose(); iamfConfigDecoder( - this.initializationData, - Util.getByteDepth(DEFAULT_PCM_ENCODING) * C.BITS_PER_BYTE, - DEFAULT_OUTPUT_SAMPLE_RATE, - DEFAULT_CHANNEL_COUNT); // reconfigure + initializationData, + Util.getByteDepth(OUTPUT_PCM_ENCODING) * C.BITS_PER_BYTE, + OUTPUT_SAMPLE_RATE, + soundSystem); // reconfigure } int bufferSize = - iamfGetMaxFrameSize() * DEFAULT_CHANNEL_COUNT * Util.getByteDepth(DEFAULT_PCM_ENCODING); + iamfGetMaxFrameSize() * getChannelCount() * Util.getByteDepth(OUTPUT_PCM_ENCODING); outputBuffer.init(inputBuffer.timeUs, bufferSize); ByteBuffer outputData = Util.castNonNull(outputBuffer.data); ByteBuffer inputData = Util.castNonNull(inputBuffer.data); @@ -113,14 +126,14 @@ public final class IamfDecoder return new IamfDecoderException("Failed to decode error= " + ret); } outputData.position(0); - outputData.limit(ret * DEFAULT_CHANNEL_COUNT * Util.getByteDepth(DEFAULT_PCM_ENCODING)); + outputData.limit(ret * getChannelCount() * Util.getByteDepth(OUTPUT_PCM_ENCODING)); return null; } private native int iamfLayoutBinauralChannelsCount(); private native int iamfConfigDecoder( - byte[] initializationData, int bitDepth, int sampleRate, int channelCount); + byte[] initializationData, int bitDepth, int sampleRate, int soundSystem); private native void iamfClose(); @@ -131,4 +144,6 @@ public final class IamfDecoder * Used to initialize the output buffer. */ private native int iamfGetMaxFrameSize(); + + private native int iamfGetChannelCount(int soundSystem); } diff --git a/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java b/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java index bae8b5baab..f5b7d6c0cd 100644 --- a/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java +++ b/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java @@ -15,12 +15,16 @@ */ package androidx.media3.decoder.iamf; +import android.content.Context; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.Spatializer; import android.os.Handler; import androidx.annotation.Nullable; +import androidx.media3.common.AudioAttributes; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; -import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.util.TraceUtil; import androidx.media3.common.util.Util; import androidx.media3.decoder.CryptoConfig; @@ -32,35 +36,24 @@ import java.util.Objects; /** Decodes and renders audio using the native IAMF decoder. */ public class LibiamfAudioRenderer extends DecoderAudioRenderer { + private final Context context; /** * Creates a new instance. * - * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be - * null if delivery of events is not required. - * @param eventListener A listener of events. May be null if delivery of events is not required. - * @param audioProcessors Optional {@link AudioProcessor}s that will process audio before output. - */ - public LibiamfAudioRenderer( - @Nullable Handler eventHandler, - @Nullable AudioRendererEventListener eventListener, - AudioProcessor... audioProcessors) { - super(eventHandler, eventListener, audioProcessors); - } - - /** - * Creates a new instance. - * + * @param context The context to use for spatialization capability checks. * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be * null if delivery of events is not required. * @param eventListener A listener of events. May be null if delivery of events is not required. * @param audioSink The sink to which audio will be output. */ public LibiamfAudioRenderer( + Context context, @Nullable Handler eventHandler, @Nullable AudioRendererEventListener eventListener, AudioSink audioSink) { super(eventHandler, eventListener, audioSink); + this.context = context; } @Override @@ -75,7 +68,7 @@ public class LibiamfAudioRenderer extends DecoderAudioRenderer { protected IamfDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig) throws DecoderException { TraceUtil.beginSection("createIamfDecoder"); - IamfDecoder decoder = new IamfDecoder(format.initializationData); + IamfDecoder decoder = new IamfDecoder(format.initializationData, isSpatializationSupported()); TraceUtil.endSection(); return decoder; } @@ -83,13 +76,33 @@ public class LibiamfAudioRenderer extends DecoderAudioRenderer { @Override protected Format getOutputFormat(IamfDecoder decoder) { return Util.getPcmFormat( - IamfDecoder.DEFAULT_PCM_ENCODING, - IamfDecoder.DEFAULT_CHANNEL_COUNT, - IamfDecoder.DEFAULT_OUTPUT_SAMPLE_RATE); + IamfDecoder.OUTPUT_PCM_ENCODING, decoder.getChannelCount(), IamfDecoder.OUTPUT_SAMPLE_RATE); } @Override public String getName() { return "LibiamfAudioRenderer"; } + + private boolean isSpatializationSupported() { + // Spatializer is only available on API 32 and above. + if (Util.SDK_INT < 32) { + return false; + } + + AudioManager audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + AudioFormat.Builder audioFormat = + new AudioFormat.Builder() + .setEncoding(IamfDecoder.OUTPUT_PCM_ENCODING) + .setChannelMask(IamfDecoder.SPATIALIZED_OUTPUT_LAYOUT); + if (audioManager == null) { + return false; + } + Spatializer spatializer = audioManager.getSpatializer(); + return spatializer.getImmersiveAudioLevel() != Spatializer.SPATIALIZER_IMMERSIVE_LEVEL_NONE + && spatializer.isAvailable() + && spatializer.isEnabled() + && spatializer.canBeSpatialized( + AudioAttributes.DEFAULT.getAudioAttributesV21().audioAttributes, audioFormat.build()); + } } diff --git a/libraries/decoder_iamf/src/main/jni/iamf_jni.cc b/libraries/decoder_iamf/src/main/jni/iamf_jni.cc index a51aa1f722..74d3932027 100644 --- a/libraries/decoder_iamf/src/main/jni/iamf_jni.cc +++ b/libraries/decoder_iamf/src/main/jni/iamf_jni.cc @@ -57,16 +57,13 @@ DECODER_FUNC(jint, iamfLayoutBinauralChannelsCount) { IAMF_DecoderHandle handle; DECODER_FUNC(jint, iamfConfigDecoder, jbyteArray initializationDataArray, - jint bitDepth, jint sampleRate, jint channelCount) { + jint bitDepth, jint sampleRate, jint layoutType) { handle = IAMF_decoder_open(); IAMF_decoder_peak_limiter_enable(handle, 0); IAMF_decoder_set_bit_depth(handle, bitDepth); IAMF_decoder_set_sampling_rate(handle, sampleRate); - if (channelCount == 2) { - IAMF_decoder_output_layout_set_binaural(handle); - } else { - IAMF_decoder_output_layout_set_sound_system(handle, SOUND_SYSTEM_INVALID); - } + IAMF_decoder_output_layout_set_sound_system(handle, + (IAMF_SoundSystem)layoutType); uint32_t* bytes_read = nullptr; jbyte* initializationDataBytes = @@ -95,4 +92,8 @@ DECODER_FUNC(jint, iamfGetMaxFrameSize) { return IAMF_decoder_get_stream_info(handle)->max_frame_size; } +DECODER_FUNC(jint, iamfGetChannelCount, jint layoutType) { + return IAMF_layout_sound_system_channels_count((IAMF_SoundSystem)layoutType); +} + DECODER_FUNC(void, iamfClose) { IAMF_decoder_close(handle); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java index 5894061c9d..1c6ce756a7 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java @@ -552,11 +552,12 @@ public class DefaultRenderersFactory implements RenderersFactory { Class clazz = Class.forName("androidx.media3.decoder.iamf.LibiamfAudioRenderer"); Constructor constructor = clazz.getConstructor( + Context.class, android.os.Handler.class, androidx.media3.exoplayer.audio.AudioRendererEventListener.class, androidx.media3.exoplayer.audio.AudioSink.class); Renderer renderer = - (Renderer) constructor.newInstance(eventHandler, eventListener, audioSink); + (Renderer) constructor.newInstance(context, eventHandler, eventListener, audioSink); out.add(extensionRendererIndex++, renderer); } catch (ClassNotFoundException e) { // Expected if the app was built without the extension.