Add spatial effects to IAMF support in Exoplayer.

Check if the output device supports spatialization for the requested output format. If so, return a stream decoded for 6 channels in a 5.1 layout. Otherwise, return a stream decoded for 2 channels in a binaural layout.

PiperOrigin-RevId: 662546818

libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfDecoderTest.java

@@ -45,7 +45,8 @@ public final class IamfDecoderTest {
 
   @Test
   public void iamfBinauralLayoutChannelsCount_equalsTwo() throws Exception {
-    IamfDecoder iamf = new IamfDecoder(ImmutableList.of(IACB_OBUS));
+    IamfDecoder iamf =
+        new IamfDecoder(ImmutableList.of(IACB_OBUS), /* spatializationSupported= */ false);
 
     assertThat(iamf.getBinauralLayoutChannelCount())
         .isEqualTo(DEFAULT_BINAURAL_LAYOUT_CHANNEL_COUNT);

libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java

@@ -18,12 +18,17 @@ package androidx.media3.decoder.iamf;
 import static com.google.common.truth.Truth.assertWithMessage;
 
 import android.content.Context;
+import android.media.AudioFormat;
+import android.media.AudioManager;
+import android.media.Spatializer;
 import android.net.Uri;
 import android.os.Looper;
 import androidx.annotation.Nullable;
+import androidx.media3.common.AudioAttributes;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.PlaybackException;
 import androidx.media3.common.Player;
+import androidx.media3.common.util.Util;
 import androidx.media3.datasource.DefaultDataSource;
 import androidx.media3.exoplayer.ExoPlayer;
 import androidx.media3.exoplayer.Renderer;
@@ -97,6 +102,26 @@ public class IamfPlaybackTest {
     @Override
     public void run() {
       Looper.prepare();
+      if (Util.SDK_INT >= 32) { // Spatializer is only available on API 32 and above.
+        AudioManager audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE);
+        AudioFormat.Builder audioFormat =
+            new AudioFormat.Builder()
+                .setEncoding(IamfDecoder.OUTPUT_PCM_ENCODING)
+                .setChannelMask(IamfDecoder.SPATIALIZED_OUTPUT_LAYOUT);
+        if (audioManager != null) {
+          Spatializer spatializer = audioManager.getSpatializer();
+          assertWithMessage("Spatializer must be disabled to run this test.")
+              .that(
+                  spatializer.getImmersiveAudioLevel()
+                          != Spatializer.SPATIALIZER_IMMERSIVE_LEVEL_NONE
+                      && spatializer.isAvailable()
+                      && spatializer.isEnabled()
+                      && spatializer.canBeSpatialized(
+                          AudioAttributes.DEFAULT.getAudioAttributesV21().audioAttributes,
+                          audioFormat.build()))
+              .isFalse();
+        }
+      }
       RenderersFactory renderersFactory =
           (eventHandler,
               videoRendererEventListener,
@@ -104,7 +129,8 @@ public class IamfPlaybackTest {
               textRendererOutput,
               metadataRendererOutput) ->
               new Renderer[] {
-                new LibiamfAudioRenderer(eventHandler, audioRendererEventListener, audioSink)
+                new LibiamfAudioRenderer(
+                    context, eventHandler, audioRendererEventListener, audioSink)
               };
       player = new ExoPlayer.Builder(context, renderersFactory).build();
       player.addListener(this);

libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java

@@ -17,6 +17,7 @@ package androidx.media3.decoder.iamf;
 
 import static android.support.annotation.VisibleForTesting.PACKAGE_PRIVATE;
 
+import android.media.AudioFormat;
 import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.C;
 import androidx.media3.common.util.Util;
@@ -31,31 +32,39 @@ import javax.annotation.Nullable;
 @VisibleForTesting(otherwise = PACKAGE_PRIVATE)
 public final class IamfDecoder
     extends SimpleDecoder<DecoderInputBuffer, SimpleDecoderOutputBuffer, IamfDecoderException> {
-  // TODO(ktrajkovski): Fetch channel count from the device instead of hardcoding.
+  /* package */ static final int OUTPUT_SAMPLE_RATE = 48000;
-  /* package */ static final int DEFAULT_CHANNEL_COUNT = 2;
+  /* package */ static final int OUTPUT_PCM_ENCODING = AudioFormat.ENCODING_PCM_16BIT;
-  /* package */ static final int DEFAULT_OUTPUT_SAMPLE_RATE = 48000;
+  /* package */ static final int SPATIALIZED_OUTPUT_LAYOUT = AudioFormat.CHANNEL_OUT_5POINT1;
-  /* package */ static final @C.PcmEncoding int DEFAULT_PCM_ENCODING = C.ENCODING_PCM_16BIT;
+
+  // Matches IAMF_SoundSystem in IAMF_defines.h
+  private static final int SOUND_SYSTEM_STEREO = 0; // SOUND_SYSTEM_A
+  private static final int SOUND_SYSTEM_5POINT1 = 1; // SOUND_SYSTEM_B
 
   private final byte[] initializationData;
+  private final int soundSystem;
 
   /**
    * Creates an IAMF decoder.
    *
    * @param initializationData ConfigOBUs data for the decoder.
+   * @param spatializationSupported Whether spatialization is supported and output should be 6
+   *     channels in 5.1 layout.
    * @throws IamfDecoderException Thrown if an exception occurs when initializing the decoder.
    */
-  public IamfDecoder(List<byte[]> initializationData) throws IamfDecoderException {
+  public IamfDecoder(List<byte[]> initializationData, boolean spatializationSupported)
+      throws IamfDecoderException {
     super(new DecoderInputBuffer[1], new SimpleDecoderOutputBuffer[1]);
     if (initializationData.size() != 1) {
       throw new IamfDecoderException("Initialization data must contain a single element.");
     }
+    soundSystem = spatializationSupported ? SOUND_SYSTEM_5POINT1 : SOUND_SYSTEM_STEREO;
     this.initializationData = initializationData.get(0);
     int status =
         iamfConfigDecoder(
             this.initializationData,
-            Util.getByteDepth(DEFAULT_PCM_ENCODING) * C.BITS_PER_BYTE,
+            Util.getByteDepth(OUTPUT_PCM_ENCODING) * C.BITS_PER_BYTE,
-            DEFAULT_OUTPUT_SAMPLE_RATE,
+            OUTPUT_SAMPLE_RATE,
-            DEFAULT_CHANNEL_COUNT);
+            soundSystem);
     if (status != 0) {
       throw new IamfDecoderException("Failed to configure decoder with returned status: " + status);
     }
@@ -71,6 +80,10 @@ public final class IamfDecoder
     return iamfLayoutBinauralChannelsCount();
   }
 
+  public int getChannelCount() {
+    return iamfGetChannelCount(soundSystem);
+  }
+
   @Override
   public String getName() {
     return "libiamf";
@@ -98,13 +111,13 @@ public final class IamfDecoder
     if (reset) {
       iamfClose();
       iamfConfigDecoder(
-          this.initializationData,
+          initializationData,
-          Util.getByteDepth(DEFAULT_PCM_ENCODING) * C.BITS_PER_BYTE,
+          Util.getByteDepth(OUTPUT_PCM_ENCODING) * C.BITS_PER_BYTE,
-          DEFAULT_OUTPUT_SAMPLE_RATE,
+          OUTPUT_SAMPLE_RATE,
-          DEFAULT_CHANNEL_COUNT); // reconfigure
+          soundSystem); // reconfigure
     }
     int bufferSize =
-        iamfGetMaxFrameSize() * DEFAULT_CHANNEL_COUNT * Util.getByteDepth(DEFAULT_PCM_ENCODING);
+        iamfGetMaxFrameSize() * getChannelCount() * Util.getByteDepth(OUTPUT_PCM_ENCODING);
     outputBuffer.init(inputBuffer.timeUs, bufferSize);
     ByteBuffer outputData = Util.castNonNull(outputBuffer.data);
     ByteBuffer inputData = Util.castNonNull(inputBuffer.data);
@@ -113,14 +126,14 @@ public final class IamfDecoder
       return new IamfDecoderException("Failed to decode error= " + ret);
     }
     outputData.position(0);
-    outputData.limit(ret * DEFAULT_CHANNEL_COUNT * Util.getByteDepth(DEFAULT_PCM_ENCODING));
+    outputData.limit(ret * getChannelCount() * Util.getByteDepth(OUTPUT_PCM_ENCODING));
     return null;
   }
 
   private native int iamfLayoutBinauralChannelsCount();
 
   private native int iamfConfigDecoder(
-      byte[] initializationData, int bitDepth, int sampleRate, int channelCount);
+      byte[] initializationData, int bitDepth, int sampleRate, int soundSystem);
 
   private native void iamfClose();
 
@@ -131,4 +144,6 @@ public final class IamfDecoder
    * Used to initialize the output buffer.
    */
   private native int iamfGetMaxFrameSize();
+
+  private native int iamfGetChannelCount(int soundSystem);
 }

libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java

@@ -15,12 +15,16 @@
  */
 package androidx.media3.decoder.iamf;
 
+import android.content.Context;
+import android.media.AudioFormat;
+import android.media.AudioManager;
+import android.media.Spatializer;
 import android.os.Handler;
 import androidx.annotation.Nullable;
+import androidx.media3.common.AudioAttributes;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.MimeTypes;
-import androidx.media3.common.audio.AudioProcessor;
 import androidx.media3.common.util.TraceUtil;
 import androidx.media3.common.util.Util;
 import androidx.media3.decoder.CryptoConfig;
@@ -32,35 +36,24 @@ import java.util.Objects;
 
 /** Decodes and renders audio using the native IAMF decoder. */
 public class LibiamfAudioRenderer extends DecoderAudioRenderer<IamfDecoder> {
+  private final Context context;
 
   /**
    * Creates a new instance.
    *
-   * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
+   * @param context The context to use for spatialization capability checks.
-   *     null if delivery of events is not required.
-   * @param eventListener A listener of events. May be null if delivery of events is not required.
-   * @param audioProcessors Optional {@link AudioProcessor}s that will process audio before output.
-   */
-  public LibiamfAudioRenderer(
-      @Nullable Handler eventHandler,
-      @Nullable AudioRendererEventListener eventListener,
-      AudioProcessor... audioProcessors) {
-    super(eventHandler, eventListener, audioProcessors);
-  }
-
-  /**
-   * Creates a new instance.
-   *
    * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
    *     null if delivery of events is not required.
    * @param eventListener A listener of events. May be null if delivery of events is not required.
    * @param audioSink The sink to which audio will be output.
    */
   public LibiamfAudioRenderer(
+      Context context,
       @Nullable Handler eventHandler,
       @Nullable AudioRendererEventListener eventListener,
       AudioSink audioSink) {
     super(eventHandler, eventListener, audioSink);
+    this.context = context;
   }
 
   @Override
@@ -75,7 +68,7 @@ public class LibiamfAudioRenderer extends DecoderAudioRenderer<IamfDecoder> {
   protected IamfDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig)
       throws DecoderException {
     TraceUtil.beginSection("createIamfDecoder");
-    IamfDecoder decoder = new IamfDecoder(format.initializationData);
+    IamfDecoder decoder = new IamfDecoder(format.initializationData, isSpatializationSupported());
     TraceUtil.endSection();
     return decoder;
   }
@@ -83,13 +76,33 @@ public class LibiamfAudioRenderer extends DecoderAudioRenderer<IamfDecoder> {
   @Override
   protected Format getOutputFormat(IamfDecoder decoder) {
     return Util.getPcmFormat(
-        IamfDecoder.DEFAULT_PCM_ENCODING,
+        IamfDecoder.OUTPUT_PCM_ENCODING, decoder.getChannelCount(), IamfDecoder.OUTPUT_SAMPLE_RATE);
-        IamfDecoder.DEFAULT_CHANNEL_COUNT,
-        IamfDecoder.DEFAULT_OUTPUT_SAMPLE_RATE);
   }
 
   @Override
   public String getName() {
     return "LibiamfAudioRenderer";
   }
+
+  private boolean isSpatializationSupported() {
+    // Spatializer is only available on API 32 and above.
+    if (Util.SDK_INT < 32) {
+      return false;
+    }
+
+    AudioManager audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE);
+    AudioFormat.Builder audioFormat =
+        new AudioFormat.Builder()
+            .setEncoding(IamfDecoder.OUTPUT_PCM_ENCODING)
+            .setChannelMask(IamfDecoder.SPATIALIZED_OUTPUT_LAYOUT);
+    if (audioManager == null) {
+      return false;
+    }
+    Spatializer spatializer = audioManager.getSpatializer();
+    return spatializer.getImmersiveAudioLevel() != Spatializer.SPATIALIZER_IMMERSIVE_LEVEL_NONE
+        && spatializer.isAvailable()
+        && spatializer.isEnabled()
+        && spatializer.canBeSpatialized(
+            AudioAttributes.DEFAULT.getAudioAttributesV21().audioAttributes, audioFormat.build());
+  }
 }

libraries/decoder_iamf/src/main/jni/iamf_jni.cc

@@ -57,16 +57,13 @@ DECODER_FUNC(jint, iamfLayoutBinauralChannelsCount) {
 IAMF_DecoderHandle handle;
 
 DECODER_FUNC(jint, iamfConfigDecoder, jbyteArray initializationDataArray,
-             jint bitDepth, jint sampleRate, jint channelCount) {
+             jint bitDepth, jint sampleRate, jint layoutType) {
   handle = IAMF_decoder_open();
   IAMF_decoder_peak_limiter_enable(handle, 0);
   IAMF_decoder_set_bit_depth(handle, bitDepth);
   IAMF_decoder_set_sampling_rate(handle, sampleRate);
-  if (channelCount == 2) {
+  IAMF_decoder_output_layout_set_sound_system(handle,
-    IAMF_decoder_output_layout_set_binaural(handle);
+                                              (IAMF_SoundSystem)layoutType);
-  } else {
-    IAMF_decoder_output_layout_set_sound_system(handle, SOUND_SYSTEM_INVALID);
-  }
 
   uint32_t* bytes_read = nullptr;
   jbyte* initializationDataBytes =
@@ -95,4 +92,8 @@ DECODER_FUNC(jint, iamfGetMaxFrameSize) {
   return IAMF_decoder_get_stream_info(handle)->max_frame_size;
 }
 
+DECODER_FUNC(jint, iamfGetChannelCount, jint layoutType) {
+  return IAMF_layout_sound_system_channels_count((IAMF_SoundSystem)layoutType);
+}
+
 DECODER_FUNC(void, iamfClose) { IAMF_decoder_close(handle); }

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java

@@ -552,11 +552,12 @@ public class DefaultRenderersFactory implements RenderersFactory {
       Class<?> clazz = Class.forName("androidx.media3.decoder.iamf.LibiamfAudioRenderer");
       Constructor<?> constructor =
           clazz.getConstructor(
+              Context.class,
               android.os.Handler.class,
               androidx.media3.exoplayer.audio.AudioRendererEventListener.class,
               androidx.media3.exoplayer.audio.AudioSink.class);
       Renderer renderer =
-          (Renderer) constructor.newInstance(eventHandler, eventListener, audioSink);
+          (Renderer) constructor.newInstance(context, eventHandler, eventListener, audioSink);
       out.add(extensionRendererIndex++, renderer);
     } catch (ClassNotFoundException e) {
       // Expected if the app was built without the extension.