loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add spatial effects to IAMF support in Exoplayer.

Browse Source 
Check if the output device supports spatialization for the requested output format. If so, return a stream decoded for 6 channels in a 5.1 layout. Otherwise, return a stream decoded for 2 channels in a binaural layout.

PiperOrigin-RevId: 662546818
This commit is contained in:
ktrajkovski 2024-08-13 09:31:22 -07:00 committed by Copybara-Service
parent c48c051ce2
commit 92cff64321
6 changed files with 101 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfDecoderTest.java
View File

@ -45,7 +45,8 @@ public final class IamfDecoderTest {
@Test
public void iamfBinauralLayoutChannelsCount_equalsTwo() throws Exception {
IamfDecoder iamf = new IamfDecoder(ImmutableList.of(IACB_OBUS));
IamfDecoder iamf =
new IamfDecoder(ImmutableList.of(IACB_OBUS), /* spatializationSupported= */ false);
assertThat(iamf.getBinauralLayoutChannelCount())
.isEqualTo(DEFAULT_BINAURAL_LAYOUT_CHANNEL_COUNT);

28
libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java
View File

@ -18,12 +18,17 @@ package androidx.media3.decoder.iamf;
import static com.google.common.truth.Truth.assertWithMessage;
import android.content.Context;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.Spatializer;
import android.net.Uri;
import android.os.Looper;
import androidx.annotation.Nullable;
import androidx.media3.common.AudioAttributes;
import androidx.media3.common.MediaItem;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.Player;
import androidx.media3.common.util.Util;
import androidx.media3.datasource.DefaultDataSource;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.Renderer;
@ -97,6 +102,26 @@ public class IamfPlaybackTest {
@Override
public void run() {
Looper.prepare();
if (Util.SDK_INT >= 32) { // Spatializer is only available on API 32 and above.
AudioManager audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE);
AudioFormat.Builder audioFormat =
new AudioFormat.Builder()
.setEncoding(IamfDecoder.OUTPUT_PCM_ENCODING)
.setChannelMask(IamfDecoder.SPATIALIZED_OUTPUT_LAYOUT);
if (audioManager != null) {
Spatializer spatializer = audioManager.getSpatializer();
assertWithMessage("Spatializer must be disabled to run this test.")
.that(
spatializer.getImmersiveAudioLevel()
!= Spatializer.SPATIALIZER_IMMERSIVE_LEVEL_NONE
&& spatializer.isAvailable()
&& spatializer.isEnabled()
&& spatializer.canBeSpatialized(
AudioAttributes.DEFAULT.getAudioAttributesV21().audioAttributes,
audioFormat.build()))
.isFalse();
}
}
RenderersFactory renderersFactory =
(eventHandler,
videoRendererEventListener,
@ -104,7 +129,8 @@ public class IamfPlaybackTest {
textRendererOutput,
metadataRendererOutput) ->
new Renderer[] {
new LibiamfAudioRenderer(eventHandler, audioRendererEventListener, audioSink)
new LibiamfAudioRenderer(
context, eventHandler, audioRendererEventListener, audioSink)
};
player = new ExoPlayer.Builder(context, renderersFactory).build();
player.addListener(this);

45
libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java
View File

@ -17,6 +17,7 @@ package androidx.media3.decoder.iamf;
import static android.support.annotation.VisibleForTesting.PACKAGE_PRIVATE;
import android.media.AudioFormat;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.util.Util;
@ -31,31 +32,39 @@ import javax.annotation.Nullable;
@VisibleForTesting(otherwise = PACKAGE_PRIVATE)
public final class IamfDecoder
extends SimpleDecoder<DecoderInputBuffer, SimpleDecoderOutputBuffer, IamfDecoderException> {
// TODO(ktrajkovski): Fetch channel count from the device instead of hardcoding.
/* package */ static final int DEFAULT_CHANNEL_COUNT = 2;
/* package */ static final int DEFAULT_OUTPUT_SAMPLE_RATE = 48000;
/* package */ static final @C.PcmEncoding int DEFAULT_PCM_ENCODING = C.ENCODING_PCM_16BIT;
/* package */ static final int OUTPUT_SAMPLE_RATE = 48000;
/* package */ static final int OUTPUT_PCM_ENCODING = AudioFormat.ENCODING_PCM_16BIT;
/* package */ static final int SPATIALIZED_OUTPUT_LAYOUT = AudioFormat.CHANNEL_OUT_5POINT1;
// Matches IAMF_SoundSystem in IAMF_defines.h
private static final int SOUND_SYSTEM_STEREO = 0; // SOUND_SYSTEM_A
private static final int SOUND_SYSTEM_5POINT1 = 1; // SOUND_SYSTEM_B
private final byte[] initializationData;
private final int soundSystem;
/**
* Creates an IAMF decoder.
*
* @param initializationData ConfigOBUs data for the decoder.
* @param spatializationSupported Whether spatialization is supported and output should be 6
* channels in 5.1 layout.
* @throws IamfDecoderException Thrown if an exception occurs when initializing the decoder.
*/
public IamfDecoder(List<byte[]> initializationData) throws IamfDecoderException {
public IamfDecoder(List<byte[]> initializationData, boolean spatializationSupported)
throws IamfDecoderException {
super(new DecoderInputBuffer[1], new SimpleDecoderOutputBuffer[1]);
if (initializationData.size() != 1) {
throw new IamfDecoderException("Initialization data must contain a single element.");
}
soundSystem = spatializationSupported ? SOUND_SYSTEM_5POINT1 : SOUND_SYSTEM_STEREO;
this.initializationData = initializationData.get(0);
int status =
iamfConfigDecoder(
this.initializationData,
Util.getByteDepth(DEFAULT_PCM_ENCODING) * C.BITS_PER_BYTE,
DEFAULT_OUTPUT_SAMPLE_RATE,
DEFAULT_CHANNEL_COUNT);
Util.getByteDepth(OUTPUT_PCM_ENCODING) * C.BITS_PER_BYTE,
OUTPUT_SAMPLE_RATE,
soundSystem);
if (status != 0) {
throw new IamfDecoderException("Failed to configure decoder with returned status: " + status);
}
@ -71,6 +80,10 @@ public final class IamfDecoder
return iamfLayoutBinauralChannelsCount();
}
public int getChannelCount() {
return iamfGetChannelCount(soundSystem);
}
@Override
public String getName() {
return "libiamf";
@ -98,13 +111,13 @@ public final class IamfDecoder
if (reset) {
iamfClose();
iamfConfigDecoder(
this.initializationData,
Util.getByteDepth(DEFAULT_PCM_ENCODING) * C.BITS_PER_BYTE,
DEFAULT_OUTPUT_SAMPLE_RATE,
DEFAULT_CHANNEL_COUNT); // reconfigure
initializationData,
Util.getByteDepth(OUTPUT_PCM_ENCODING) * C.BITS_PER_BYTE,
OUTPUT_SAMPLE_RATE,
soundSystem); // reconfigure
}
int bufferSize =
iamfGetMaxFrameSize() * DEFAULT_CHANNEL_COUNT * Util.getByteDepth(DEFAULT_PCM_ENCODING);
iamfGetMaxFrameSize() * getChannelCount() * Util.getByteDepth(OUTPUT_PCM_ENCODING);
outputBuffer.init(inputBuffer.timeUs, bufferSize);
ByteBuffer outputData = Util.castNonNull(outputBuffer.data);
ByteBuffer inputData = Util.castNonNull(inputBuffer.data);
@ -113,14 +126,14 @@ public final class IamfDecoder
return new IamfDecoderException("Failed to decode error= " + ret);
}
outputData.position(0);
outputData.limit(ret * DEFAULT_CHANNEL_COUNT * Util.getByteDepth(DEFAULT_PCM_ENCODING));
outputData.limit(ret * getChannelCount() * Util.getByteDepth(OUTPUT_PCM_ENCODING));
return null;
}
private native int iamfLayoutBinauralChannelsCount();
private native int iamfConfigDecoder(
byte[] initializationData, int bitDepth, int sampleRate, int channelCount);
byte[] initializationData, int bitDepth, int sampleRate, int soundSystem);
private native void iamfClose();
@ -131,4 +144,6 @@ public final class IamfDecoder
* Used to initialize the output buffer.
*/
private native int iamfGetMaxFrameSize();
private native int iamfGetChannelCount(int soundSystem);
}

53
libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java
View File

@ -15,12 +15,16 @@
*/
package androidx.media3.decoder.iamf;
import android.content.Context;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.Spatializer;
import android.os.Handler;
import androidx.annotation.Nullable;
import androidx.media3.common.AudioAttributes;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.audio.AudioProcessor;
import androidx.media3.common.util.TraceUtil;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.CryptoConfig;
@ -32,35 +36,24 @@ import java.util.Objects;
/** Decodes and renders audio using the native IAMF decoder. */
public class LibiamfAudioRenderer extends DecoderAudioRenderer<IamfDecoder> {
private final Context context;
/**
* Creates a new instance.
*
* @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
* null if delivery of events is not required.
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param audioProcessors Optional {@link AudioProcessor}s that will process audio before output.
*/
public LibiamfAudioRenderer(
@Nullable Handler eventHandler,
@Nullable AudioRendererEventListener eventListener,
AudioProcessor... audioProcessors) {
super(eventHandler, eventListener, audioProcessors);
}
/**
* Creates a new instance.
*
* @param context The context to use for spatialization capability checks.
* @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
* null if delivery of events is not required.
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param audioSink The sink to which audio will be output.
*/
public LibiamfAudioRenderer(
Context context,
@Nullable Handler eventHandler,
@Nullable AudioRendererEventListener eventListener,
AudioSink audioSink) {
super(eventHandler, eventListener, audioSink);
this.context = context;
}
@Override
@ -75,7 +68,7 @@ public class LibiamfAudioRenderer extends DecoderAudioRenderer<IamfDecoder> {
protected IamfDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig)
throws DecoderException {
TraceUtil.beginSection("createIamfDecoder");
IamfDecoder decoder = new IamfDecoder(format.initializationData);
IamfDecoder decoder = new IamfDecoder(format.initializationData, isSpatializationSupported());
TraceUtil.endSection();
return decoder;
}
@ -83,13 +76,33 @@ public class LibiamfAudioRenderer extends DecoderAudioRenderer<IamfDecoder> {
@Override
protected Format getOutputFormat(IamfDecoder decoder) {
return Util.getPcmFormat(
IamfDecoder.DEFAULT_PCM_ENCODING,
IamfDecoder.DEFAULT_CHANNEL_COUNT,
IamfDecoder.DEFAULT_OUTPUT_SAMPLE_RATE);
IamfDecoder.OUTPUT_PCM_ENCODING, decoder.getChannelCount(), IamfDecoder.OUTPUT_SAMPLE_RATE);
}
@Override
public String getName() {
return "LibiamfAudioRenderer";
}
private boolean isSpatializationSupported() {
// Spatializer is only available on API 32 and above.
if (Util.SDK_INT < 32) {
return false;
}
AudioManager audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE);
AudioFormat.Builder audioFormat =
new AudioFormat.Builder()
.setEncoding(IamfDecoder.OUTPUT_PCM_ENCODING)
.setChannelMask(IamfDecoder.SPATIALIZED_OUTPUT_LAYOUT);
if (audioManager == null) {
return false;
}
Spatializer spatializer = audioManager.getSpatializer();
return spatializer.getImmersiveAudioLevel() != Spatializer.SPATIALIZER_IMMERSIVE_LEVEL_NONE
&& spatializer.isAvailable()
&& spatializer.isEnabled()
&& spatializer.canBeSpatialized(
AudioAttributes.DEFAULT.getAudioAttributesV21().audioAttributes, audioFormat.build());
}
}

13
libraries/decoder_iamf/src/main/jni/iamf_jni.cc
View File

@ -57,16 +57,13 @@ DECODER_FUNC(jint, iamfLayoutBinauralChannelsCount) {
IAMF_DecoderHandle handle;
DECODER_FUNC(jint, iamfConfigDecoder, jbyteArray initializationDataArray,
jint bitDepth, jint sampleRate, jint channelCount) {
jint bitDepth, jint sampleRate, jint layoutType) {
handle = IAMF_decoder_open();
IAMF_decoder_peak_limiter_enable(handle, 0);
IAMF_decoder_set_bit_depth(handle, bitDepth);
IAMF_decoder_set_sampling_rate(handle, sampleRate);
if (channelCount == 2) {
IAMF_decoder_output_layout_set_binaural(handle);
} else {
IAMF_decoder_output_layout_set_sound_system(handle, SOUND_SYSTEM_INVALID);
}
IAMF_decoder_output_layout_set_sound_system(handle,
(IAMF_SoundSystem)layoutType);
uint32_t* bytes_read = nullptr;
jbyte* initializationDataBytes =
@ -95,4 +92,8 @@ DECODER_FUNC(jint, iamfGetMaxFrameSize) {
return IAMF_decoder_get_stream_info(handle)->max_frame_size;
}
DECODER_FUNC(jint, iamfGetChannelCount, jint layoutType) {
return IAMF_layout_sound_system_channels_count((IAMF_SoundSystem)layoutType);
}
DECODER_FUNC(void, iamfClose) { IAMF_decoder_close(handle); }

3
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java
View File

@ -552,11 +552,12 @@ public class DefaultRenderersFactory implements RenderersFactory {
Class<?> clazz = Class.forName("androidx.media3.decoder.iamf.LibiamfAudioRenderer");
Constructor<?> constructor =
clazz.getConstructor(
Context.class,
android.os.Handler.class,
androidx.media3.exoplayer.audio.AudioRendererEventListener.class,
androidx.media3.exoplayer.audio.AudioSink.class);
Renderer renderer =
(Renderer) constructor.newInstance(eventHandler, eventListener, audioSink);
(Renderer) constructor.newInstance(context, eventHandler, eventListener, audioSink);
out.add(extensionRendererIndex++, renderer);
} catch (ClassNotFoundException e) {
// Expected if the app was built without the extension.