diff --git a/demos/main/src/main/assets/media.exolist.json b/demos/main/src/main/assets/media.exolist.json index 37846cb906..63bbe26b47 100644 --- a/demos/main/src/main/assets/media.exolist.json +++ b/demos/main/src/main/assets/media.exolist.json @@ -758,6 +758,10 @@ { "name": "One hour frame counter (MP4)", "uri": "https://storage.googleapis.com/exoplayer-test-media-1/mp4/frame-counter-one-hour.mp4" + }, + { + "name": "Immersive Audio Format Sample (MP4, IAMF)", + "uri": "https://github.com/AOMediaCodec/libiamf/raw/main/tests/test_000036_s.mp4" } ] }, diff --git a/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/DefaultRenderersFactoryTest.java b/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/DefaultRenderersFactoryTest.java new file mode 100644 index 0000000000..6617fdc84f --- /dev/null +++ b/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/DefaultRenderersFactoryTest.java @@ -0,0 +1,33 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.decoder.iamf; + +import androidx.media3.common.C; +import androidx.media3.test.utils.DefaultRenderersFactoryAsserts; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link DefaultRenderersFactoryTest} with {@link LibiamfAudioRenderer}. */ +@RunWith(AndroidJUnit4.class) +public final class DefaultRenderersFactoryTest { + + @Test + public void createRenderers_instantiatesIamfRenderer() { + DefaultRenderersFactoryAsserts.assertExtensionRendererCreated( + LibiamfAudioRenderer.class, C.TRACK_TYPE_AUDIO); + } +} diff --git a/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java b/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java new file mode 100644 index 0000000000..7fc0047f1f --- /dev/null +++ b/libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java @@ -0,0 +1,136 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.decoder.iamf; + +import static com.google.common.truth.Truth.assertWithMessage; + +import android.content.Context; +import android.net.Uri; +import android.os.Looper; +import androidx.annotation.Nullable; +import androidx.media3.common.MediaItem; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.Player; +import androidx.media3.datasource.DefaultDataSource; +import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.Renderer; +import androidx.media3.exoplayer.RenderersFactory; +import androidx.media3.exoplayer.audio.AudioSink; +import androidx.media3.exoplayer.audio.DefaultAudioSink; +import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.exoplayer.source.ProgressiveMediaSource; +import androidx.media3.extractor.mp4.Mp4Extractor; +import androidx.media3.extractor.text.DefaultSubtitleParserFactory; +import androidx.media3.test.utils.CapturingAudioSink; +import androidx.media3.test.utils.DumpFileAsserts; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Playback tests using {@link LibiamfAudioRenderer}. */ +@RunWith(AndroidJUnit4.class) +public class IamfPlaybackTest { + private static final String IAMF_SAMPLE = "mp4/sample_iamf.mp4"; + + @Before + public void setUp() { + assertWithMessage("Iamf library not available").that(IamfLibrary.isAvailable()).isTrue(); + } + + @Test + public void playIamf() throws Exception { + playAndAssertAudioSinkOutput(IAMF_SAMPLE); + } + + private static void playAndAssertAudioSinkOutput(String fileName) throws Exception { + CapturingAudioSink audioSink = + new CapturingAudioSink( + new DefaultAudioSink.Builder(ApplicationProvider.getApplicationContext()).build()); + + TestPlaybackRunnable testPlaybackRunnable = + new TestPlaybackRunnable( + Uri.parse("asset:///media/" + fileName), + ApplicationProvider.getApplicationContext(), + audioSink); + Thread thread = new Thread(testPlaybackRunnable); + thread.start(); + thread.join(); + if (testPlaybackRunnable.playbackException != null) { + throw testPlaybackRunnable.playbackException; + } + DumpFileAsserts.assertOutput( + ApplicationProvider.getApplicationContext(), + audioSink, + "audiosinkdumps/" + fileName + ".audiosink.dump"); + } + + private static class TestPlaybackRunnable implements Player.Listener, Runnable { + + private final Context context; + private final Uri uri; + private final AudioSink audioSink; + + @Nullable private ExoPlayer player; + @Nullable private PlaybackException playbackException; + + public TestPlaybackRunnable(Uri uri, Context context, AudioSink audioSink) { + this.uri = uri; + this.context = context; + this.audioSink = audioSink; + } + + @Override + public void run() { + Looper.prepare(); + RenderersFactory renderersFactory = + (eventHandler, + videoRendererEventListener, + audioRendererEventListener, + textRendererOutput, + metadataRendererOutput) -> + new Renderer[] { + new LibiamfAudioRenderer(eventHandler, audioRendererEventListener, audioSink) + }; + player = new ExoPlayer.Builder(context, renderersFactory).build(); + player.addListener(this); + MediaSource mediaSource = + new ProgressiveMediaSource.Factory( + new DefaultDataSource.Factory(context), + Mp4Extractor.newFactory(new DefaultSubtitleParserFactory())) + .createMediaSource(MediaItem.fromUri(uri)); + player.setMediaSource(mediaSource); + player.prepare(); + player.play(); + Looper.loop(); + } + + @Override + public void onPlayerError(PlaybackException error) { + playbackException = error; + } + + @Override + public void onPlaybackStateChanged(@Player.State int playbackState) { + if (playbackState == Player.STATE_ENDED + || (playbackState == Player.STATE_IDLE && playbackException != null)) { + player.release(); + Looper.myLooper().quit(); + } + } + } +} diff --git a/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java b/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java index 99da0c9587..81ad9b9b3a 100644 --- a/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java +++ b/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java @@ -18,16 +18,24 @@ package androidx.media3.decoder.iamf; import static android.support.annotation.VisibleForTesting.PACKAGE_PRIVATE; import androidx.annotation.VisibleForTesting; +import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.decoder.SimpleDecoder; import androidx.media3.decoder.SimpleDecoderOutputBuffer; +import java.nio.ByteBuffer; import java.util.List; +import javax.annotation.Nullable; /** IAMF decoder. */ @VisibleForTesting(otherwise = PACKAGE_PRIVATE) public final class IamfDecoder extends SimpleDecoder { + // TODO(ktrajkovski): Find the maximum acceptable output buffer size. + private static final int DEFAULT_OUTPUT_BUFFER_SIZE = 4096; + + private final byte[] initializationData; + /** * Creates an IAMF decoder. * @@ -35,8 +43,12 @@ public final class IamfDecoder * @throws IamfDecoderException Thrown if an exception occurs when initializing the decoder. */ public IamfDecoder(List initializationData) throws IamfDecoderException { - super(new DecoderInputBuffer[0], new SimpleDecoderOutputBuffer[0]); - int status = iamfConfigDecoder(initializationData.get(0)); + super(new DecoderInputBuffer[1], new SimpleDecoderOutputBuffer[1]); + if (initializationData.size() != 1) { + throw new IamfDecoderException("Initialization data must contain a single element."); + } + this.initializationData = initializationData.get(0); + int status = iamfConfigDecoder(this.initializationData); if (status != 0) { throw new IamfDecoderException("Failed to configure decoder with returned status: " + status); } @@ -73,9 +85,25 @@ public final class IamfDecoder } @Override + @Nullable protected IamfDecoderException decode( DecoderInputBuffer inputBuffer, SimpleDecoderOutputBuffer outputBuffer, boolean reset) { - throw new UnsupportedOperationException(); + if (reset) { + iamfClose(); + iamfConfigDecoder(this.initializationData); // reconfigure + } + outputBuffer.init(inputBuffer.timeUs, DEFAULT_OUTPUT_BUFFER_SIZE); + ByteBuffer outputData = Util.castNonNull(outputBuffer.data); + ByteBuffer inputData = Util.castNonNull(inputBuffer.data); + int ret = iamfDecode(inputData, inputData.limit(), outputData); + if (ret < 0) { + return new IamfDecoderException("Failed to decode error= " + ret); + } + outputData.position(0); + // TODO(ktrajkovski): Extract the outputData limit from the iamfDecode return value, given + // channel count, and given bit depth. + outputData.limit(ret * 4); // x2 for expected bit depth, x2 for channel count + return null; } private native int iamfLayoutBinauralChannelsCount(); @@ -83,4 +111,6 @@ public final class IamfDecoder private native int iamfConfigDecoder(byte[] initializationData); private native void iamfClose(); + + private native int iamfDecode(ByteBuffer inputBuffer, int inputSize, ByteBuffer outputBuffer); } diff --git a/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java b/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java index 40345d29d3..ab9d639a43 100644 --- a/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java +++ b/libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java @@ -22,6 +22,7 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.util.TraceUtil; +import androidx.media3.common.util.Util; import androidx.media3.decoder.CryptoConfig; import androidx.media3.decoder.DecoderException; import androidx.media3.exoplayer.audio.AudioRendererEventListener; @@ -32,6 +33,12 @@ import java.util.Objects; /** Decodes and renders audio using the native IAMF decoder. */ public class LibiamfAudioRenderer extends DecoderAudioRenderer { + // TODO(ktrajkovski): Values need to be configured and must come from the same source of truth as + // in {@link IamfDecoder}. + private static final int BINAURAL_CHANNEL_COUNT = 2; + private static final int DEFAULT_OUTPUT_SAMPLE_RATE = 48000; + private static final int DEFAULT_PCM_ENCODING = C.ENCODING_PCM_16BIT; + /** * Creates a new instance. * @@ -81,7 +88,8 @@ public class LibiamfAudioRenderer extends DecoderAudioRenderer { @Override protected Format getOutputFormat(IamfDecoder decoder) { - throw new UnsupportedOperationException(); + return Util.getPcmFormat( + DEFAULT_PCM_ENCODING, BINAURAL_CHANNEL_COUNT, DEFAULT_OUTPUT_SAMPLE_RATE); } @Override diff --git a/libraries/decoder_iamf/src/main/jni/iamf_jni.cc b/libraries/decoder_iamf/src/main/jni/iamf_jni.cc index 46879c6d2f..dd3938245a 100644 --- a/libraries/decoder_iamf/src/main/jni/iamf_jni.cc +++ b/libraries/decoder_iamf/src/main/jni/iamf_jni.cc @@ -58,6 +58,9 @@ IAMF_DecoderHandle handle; DECODER_FUNC(jint, iamfConfigDecoder, jbyteArray initializationDataArray) { handle = IAMF_decoder_open(); + + // TODO(ktrajkovski): Values need to be aligned with IamfDecoder and + // LibiamfAudioRenderer and/or extracted from ConfigOBUs. IAMF_decoder_peak_limiter_enable(handle, 0); IAMF_decoder_peak_limiter_set_threshold(handle, -1.0f); IAMF_decoder_set_normalization_loudness(handle, 0.0f); @@ -78,4 +81,15 @@ DECODER_FUNC(jint, iamfConfigDecoder, jbyteArray initializationDataArray) { return status; } +DECODER_FUNC(jint, iamfDecode, jobject inputBuffer, jint inputSize, + jobject outputBuffer) { + uint32_t* rsize = nullptr; + return IAMF_decoder_decode( + handle, + reinterpret_cast( + env->GetDirectBufferAddress(inputBuffer)), + inputSize, rsize, + reinterpret_cast(env->GetDirectBufferAddress(outputBuffer))); +} + DECODER_FUNC(void, iamfClose) { IAMF_decoder_close(handle); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java index 6c28504521..1b50d7b974 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java @@ -547,6 +547,23 @@ public class DefaultRenderersFactory implements RenderersFactory { // The extension is present, but instantiation failed. throw new RuntimeException("Error instantiating FFmpeg extension", e); } + try { + // Full class names used for constructor args so the LINT rule triggers if any of them move. + Class clazz = Class.forName("androidx.media3.decoder.iamf.LibiamfAudioRenderer"); + Constructor constructor = + clazz.getConstructor( + android.os.Handler.class, + androidx.media3.exoplayer.audio.AudioRendererEventListener.class, + androidx.media3.exoplayer.audio.AudioSink.class); + Renderer renderer = + (Renderer) constructor.newInstance(eventHandler, eventListener, audioSink); + out.add(extensionRendererIndex++, renderer); + } catch (ClassNotFoundException e) { + // Expected if the app was built without the extension. + } catch (Exception e) { + // The extension is present, but instantiation failed. + throw new RuntimeException("Error instantiating IAMF extension", e); + } } /** diff --git a/libraries/test_data/src/test/assets/audiosinkdumps/mp4/sample_iamf.mp4.audiosink.dump b/libraries/test_data/src/test/assets/audiosinkdumps/mp4/sample_iamf.mp4.audiosink.dump new file mode 100644 index 0000000000..c2a334c6eb --- /dev/null +++ b/libraries/test_data/src/test/assets/audiosinkdumps/mp4/sample_iamf.mp4.audiosink.dump @@ -0,0 +1,382 @@ +AudioSink: + buffer count = 125 + discontinuity: + config: + pcmEncoding = 2 + channelCount = 2 + sampleRate = 48000 + buffer #0: + time = 1000000000000 + data = -1112365151 + buffer #1: + time = 1000000004000 + data = -1667344575 + buffer #2: + time = 1000000008000 + data = -1973614204 + buffer #3: + time = 1000000012000 + data = 1093907329 + buffer #4: + time = 1000000016000 + data = 1166333761 + buffer #5: + time = 1000000020000 + data = -1080159356 + buffer #6: + time = 1000000024000 + data = 399031681 + buffer #7: + time = 1000000028000 + data = -1744152767 + buffer #8: + time = 1000000032000 + data = -1965613692 + buffer #9: + time = 1000000036000 + data = 595533569 + buffer #10: + time = 1000000040000 + data = 1026981825 + buffer #11: + time = 1000000044000 + data = -1945622652 + buffer #12: + time = 1000000048000 + data = 699779201 + buffer #13: + time = 1000000052000 + data = -1931845567 + buffer #14: + time = 1000000056000 + data = 1664168324 + buffer #15: + time = 1000000060000 + data = -19505471 + buffer #16: + time = 1000000064000 + data = -1395371007 + buffer #17: + time = 1000000068000 + data = -1336788092 + buffer #18: + time = 1000000072000 + data = -302449791 + buffer #19: + time = 1000000076000 + data = -164347583 + buffer #20: + time = 1000000080000 + data = 325797252 + buffer #21: + time = 1000000084000 + data = -1215027391 + buffer #22: + time = 1000000088000 + data = 783523713 + buffer #23: + time = 1000000092000 + data = 746344324 + buffer #24: + time = 1000000096000 + data = 1618650945 + buffer #25: + time = 1000000100000 + data = -1158174335 + buffer #26: + time = 1000000104000 + data = -1667344575 + buffer #27: + time = 1000000108000 + data = -1973614204 + buffer #28: + time = 1000000112000 + data = 1093907329 + buffer #29: + time = 1000000116000 + data = 1166333761 + buffer #30: + time = 1000000120000 + data = -1080159356 + buffer #31: + time = 1000000124000 + data = 399031681 + buffer #32: + time = 1000000128000 + data = -1744152767 + buffer #33: + time = 1000000132000 + data = -1965613692 + buffer #34: + time = 1000000136000 + data = 595533569 + buffer #35: + time = 1000000140000 + data = 1026981825 + buffer #36: + time = 1000000144000 + data = -1945622652 + buffer #37: + time = 1000000148000 + data = 699779201 + buffer #38: + time = 1000000152000 + data = -1931845567 + buffer #39: + time = 1000000156000 + data = 1664168324 + buffer #40: + time = 1000000160000 + data = -19505471 + buffer #41: + time = 1000000164000 + data = -1395371007 + buffer #42: + time = 1000000168000 + data = -1336788092 + buffer #43: + time = 1000000172000 + data = -302449791 + buffer #44: + time = 1000000176000 + data = -164347583 + buffer #45: + time = 1000000180000 + data = 325797252 + buffer #46: + time = 1000000184000 + data = -1215027391 + buffer #47: + time = 1000000188000 + data = 783523713 + buffer #48: + time = 1000000192000 + data = 746344324 + buffer #49: + time = 1000000196000 + data = 1618650945 + buffer #50: + time = 1000000200000 + data = -1158174335 + buffer #51: + time = 1000000204000 + data = -1667344575 + buffer #52: + time = 1000000208000 + data = -1973614204 + buffer #53: + time = 1000000212000 + data = 1093907329 + buffer #54: + time = 1000000216000 + data = 1166333761 + buffer #55: + time = 1000000220000 + data = -1080159356 + buffer #56: + time = 1000000224000 + data = 399031681 + buffer #57: + time = 1000000228000 + data = -1744152767 + buffer #58: + time = 1000000232000 + data = -1965613692 + buffer #59: + time = 1000000236000 + data = 595533569 + buffer #60: + time = 1000000240000 + data = 1026981825 + buffer #61: + time = 1000000244000 + data = -1945622652 + buffer #62: + time = 1000000248000 + data = 699779201 + buffer #63: + time = 1000000252000 + data = -1931845567 + buffer #64: + time = 1000000256000 + data = 1664168324 + buffer #65: + time = 1000000260000 + data = -19505471 + buffer #66: + time = 1000000264000 + data = -1395371007 + buffer #67: + time = 1000000268000 + data = -1336788092 + buffer #68: + time = 1000000272000 + data = -302449791 + buffer #69: + time = 1000000276000 + data = -164347583 + buffer #70: + time = 1000000280000 + data = 325797252 + buffer #71: + time = 1000000284000 + data = -1215027391 + buffer #72: + time = 1000000288000 + data = 783523713 + buffer #73: + time = 1000000292000 + data = 746344324 + buffer #74: + time = 1000000296000 + data = 1618650945 + buffer #75: + time = 1000000300000 + data = -1158174335 + buffer #76: + time = 1000000304000 + data = -1667344575 + buffer #77: + time = 1000000308000 + data = -1973614204 + buffer #78: + time = 1000000312000 + data = 1093907329 + buffer #79: + time = 1000000316000 + data = 1166333761 + buffer #80: + time = 1000000320000 + data = -1080159356 + buffer #81: + time = 1000000324000 + data = 399031681 + buffer #82: + time = 1000000328000 + data = -1744152767 + buffer #83: + time = 1000000332000 + data = -1965613692 + buffer #84: + time = 1000000336000 + data = 595533569 + buffer #85: + time = 1000000340000 + data = 1026981825 + buffer #86: + time = 1000000344000 + data = -1945622652 + buffer #87: + time = 1000000348000 + data = 699779201 + buffer #88: + time = 1000000352000 + data = -1931845567 + buffer #89: + time = 1000000356000 + data = 1664168324 + buffer #90: + time = 1000000360000 + data = -19505471 + buffer #91: + time = 1000000364000 + data = -1395371007 + buffer #92: + time = 1000000368000 + data = -1336788092 + buffer #93: + time = 1000000372000 + data = -302449791 + buffer #94: + time = 1000000376000 + data = -164347583 + buffer #95: + time = 1000000380000 + data = 325797252 + buffer #96: + time = 1000000384000 + data = -1215027391 + buffer #97: + time = 1000000388000 + data = 783523713 + buffer #98: + time = 1000000392000 + data = 746344324 + buffer #99: + time = 1000000396000 + data = 1618650945 + buffer #100: + time = 1000000400000 + data = -1158174335 + buffer #101: + time = 1000000404000 + data = -1667344575 + buffer #102: + time = 1000000408000 + data = -1973614204 + buffer #103: + time = 1000000412000 + data = 1093907329 + buffer #104: + time = 1000000416000 + data = 1166333761 + buffer #105: + time = 1000000420000 + data = -1080159356 + buffer #106: + time = 1000000424000 + data = 399031681 + buffer #107: + time = 1000000428000 + data = -1744152767 + buffer #108: + time = 1000000432000 + data = -1965613692 + buffer #109: + time = 1000000436000 + data = 595533569 + buffer #110: + time = 1000000440000 + data = 1026981825 + buffer #111: + time = 1000000444000 + data = -1945622652 + buffer #112: + time = 1000000448000 + data = 699779201 + buffer #113: + time = 1000000452000 + data = -1931845567 + buffer #114: + time = 1000000456000 + data = 1664168324 + buffer #115: + time = 1000000460000 + data = -19505471 + buffer #116: + time = 1000000464000 + data = -1395371007 + buffer #117: + time = 1000000468000 + data = -1336788092 + buffer #118: + time = 1000000472000 + data = -302449791 + buffer #119: + time = 1000000476000 + data = -164347583 + buffer #120: + time = 1000000480000 + data = 325797252 + buffer #121: + time = 1000000484000 + data = -1215027391 + buffer #122: + time = 1000000488000 + data = 783523713 + buffer #123: + time = 1000000492000 + data = 746344324 + buffer #124: + time = 1000000496000 + data = 1026348167