loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add decoding functions to IamfDecoder and LibiamfAudioRender.

Browse Source 
PiperOrigin-RevId: 657621223
This commit is contained in:
ktrajkovski 2024-07-30 09:30:21 -07:00 committed by Copybara-Service
parent e9787c4196
commit 04bfeec751
8 changed files with 628 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
demos/main/src/main/assets/media.exolist.json
View File

@ -758,6 +758,10 @@
{
"name": "One hour frame counter (MP4)",
"uri": "https://storage.googleapis.com/exoplayer-test-media-1/mp4/frame-counter-one-hour.mp4"
},
{
"name": "Immersive Audio Format Sample (MP4, IAMF)",
"uri": "https://github.com/AOMediaCodec/libiamf/raw/main/tests/test_000036_s.mp4"
}
]
},

33
libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/DefaultRenderersFactoryTest.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Copyright 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.decoder.iamf;
import androidx.media3.common.C;
import androidx.media3.test.utils.DefaultRenderersFactoryAsserts;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Unit test for {@link DefaultRenderersFactoryTest} with {@link LibiamfAudioRenderer}. */
@RunWith(AndroidJUnit4.class)
public final class DefaultRenderersFactoryTest {
@Test
public void createRenderers_instantiatesIamfRenderer() {
DefaultRenderersFactoryAsserts.assertExtensionRendererCreated(
LibiamfAudioRenderer.class, C.TRACK_TYPE_AUDIO);
}
}

136
libraries/decoder_iamf/src/androidTest/java/androidx/media3/decoder/iamf/IamfPlaybackTest.java Normal file
View File

@ -0,0 +1,136 @@
/*
* Copyright 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.decoder.iamf;
import static com.google.common.truth.Truth.assertWithMessage;
import android.content.Context;
import android.net.Uri;
import android.os.Looper;
import androidx.annotation.Nullable;
import androidx.media3.common.MediaItem;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.Player;
import androidx.media3.datasource.DefaultDataSource;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.RenderersFactory;
import androidx.media3.exoplayer.audio.AudioSink;
import androidx.media3.exoplayer.audio.DefaultAudioSink;
import androidx.media3.exoplayer.source.MediaSource;
import androidx.media3.exoplayer.source.ProgressiveMediaSource;
import androidx.media3.extractor.mp4.Mp4Extractor;
import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
import androidx.media3.test.utils.CapturingAudioSink;
import androidx.media3.test.utils.DumpFileAsserts;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Playback tests using {@link LibiamfAudioRenderer}. */
@RunWith(AndroidJUnit4.class)
public class IamfPlaybackTest {
private static final String IAMF_SAMPLE = "mp4/sample_iamf.mp4";
@Before
public void setUp() {
assertWithMessage("Iamf library not available").that(IamfLibrary.isAvailable()).isTrue();
}
@Test
public void playIamf() throws Exception {
playAndAssertAudioSinkOutput(IAMF_SAMPLE);
}
private static void playAndAssertAudioSinkOutput(String fileName) throws Exception {
CapturingAudioSink audioSink =
new CapturingAudioSink(
new DefaultAudioSink.Builder(ApplicationProvider.getApplicationContext()).build());
TestPlaybackRunnable testPlaybackRunnable =
new TestPlaybackRunnable(
Uri.parse("asset:///media/" + fileName),
ApplicationProvider.getApplicationContext(),
audioSink);
Thread thread = new Thread(testPlaybackRunnable);
thread.start();
thread.join();
if (testPlaybackRunnable.playbackException != null) {
throw testPlaybackRunnable.playbackException;
}
DumpFileAsserts.assertOutput(
ApplicationProvider.getApplicationContext(),
audioSink,
"audiosinkdumps/" + fileName + ".audiosink.dump");
}
private static class TestPlaybackRunnable implements Player.Listener, Runnable {
private final Context context;
private final Uri uri;
private final AudioSink audioSink;
@Nullable private ExoPlayer player;
@Nullable private PlaybackException playbackException;
public TestPlaybackRunnable(Uri uri, Context context, AudioSink audioSink) {
this.uri = uri;
this.context = context;
this.audioSink = audioSink;
}
@Override
public void run() {
Looper.prepare();
RenderersFactory renderersFactory =
(eventHandler,
videoRendererEventListener,
audioRendererEventListener,
textRendererOutput,
metadataRendererOutput) ->
new Renderer[] {
new LibiamfAudioRenderer(eventHandler, audioRendererEventListener, audioSink)
};
player = new ExoPlayer.Builder(context, renderersFactory).build();
player.addListener(this);
MediaSource mediaSource =
new ProgressiveMediaSource.Factory(
new DefaultDataSource.Factory(context),
Mp4Extractor.newFactory(new DefaultSubtitleParserFactory()))
.createMediaSource(MediaItem.fromUri(uri));
player.setMediaSource(mediaSource);
player.prepare();
player.play();
Looper.loop();
}
@Override
public void onPlayerError(PlaybackException error) {
playbackException = error;
}
@Override
public void onPlaybackStateChanged(@Player.State int playbackState) {
if (playbackState == Player.STATE_ENDED
|| (playbackState == Player.STATE_IDLE && playbackException != null)) {
player.release();
Looper.myLooper().quit();
}
}
}
}

36
libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/IamfDecoder.java
View File

@ -18,16 +18,24 @@ package androidx.media3.decoder.iamf;
import static android.support.annotation.VisibleForTesting.PACKAGE_PRIVATE;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.decoder.SimpleDecoder;
import androidx.media3.decoder.SimpleDecoderOutputBuffer;
import java.nio.ByteBuffer;
import java.util.List;
import javax.annotation.Nullable;
/** IAMF decoder. */
@VisibleForTesting(otherwise = PACKAGE_PRIVATE)
public final class IamfDecoder
extends SimpleDecoder<DecoderInputBuffer, SimpleDecoderOutputBuffer, IamfDecoderException> {
// TODO(ktrajkovski): Find the maximum acceptable output buffer size.
private static final int DEFAULT_OUTPUT_BUFFER_SIZE = 4096;
private final byte[] initializationData;
/**
* Creates an IAMF decoder.
*
@ -35,8 +43,12 @@ public final class IamfDecoder
* @throws IamfDecoderException Thrown if an exception occurs when initializing the decoder.
*/
public IamfDecoder(List<byte[]> initializationData) throws IamfDecoderException {
super(new DecoderInputBuffer[0], new SimpleDecoderOutputBuffer[0]);
int status = iamfConfigDecoder(initializationData.get(0));
super(new DecoderInputBuffer[1], new SimpleDecoderOutputBuffer[1]);
if (initializationData.size() != 1) {
throw new IamfDecoderException("Initialization data must contain a single element.");
}
this.initializationData = initializationData.get(0);
int status = iamfConfigDecoder(this.initializationData);
if (status != 0) {
throw new IamfDecoderException("Failed to configure decoder with returned status: " + status);
}
@ -73,9 +85,25 @@ public final class IamfDecoder
}
@Override
@Nullable
protected IamfDecoderException decode(
DecoderInputBuffer inputBuffer, SimpleDecoderOutputBuffer outputBuffer, boolean reset) {
throw new UnsupportedOperationException();
if (reset) {
iamfClose();
iamfConfigDecoder(this.initializationData); // reconfigure
}
outputBuffer.init(inputBuffer.timeUs, DEFAULT_OUTPUT_BUFFER_SIZE);
ByteBuffer outputData = Util.castNonNull(outputBuffer.data);
ByteBuffer inputData = Util.castNonNull(inputBuffer.data);
int ret = iamfDecode(inputData, inputData.limit(), outputData);
if (ret < 0) {
return new IamfDecoderException("Failed to decode error= " + ret);
}
outputData.position(0);
// TODO(ktrajkovski): Extract the outputData limit from the iamfDecode return value, given
// channel count, and given bit depth.
outputData.limit(ret * 4); // x2 for expected bit depth, x2 for channel count
return null;
}
private native int iamfLayoutBinauralChannelsCount();
@ -83,4 +111,6 @@ public final class IamfDecoder
private native int iamfConfigDecoder(byte[] initializationData);
private native void iamfClose();
private native int iamfDecode(ByteBuffer inputBuffer, int inputSize, ByteBuffer outputBuffer);
}

10
libraries/decoder_iamf/src/main/java/androidx/media3/decoder/iamf/LibiamfAudioRenderer.java
View File

@ -22,6 +22,7 @@ import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.audio.AudioProcessor;
import androidx.media3.common.util.TraceUtil;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.CryptoConfig;
import androidx.media3.decoder.DecoderException;
import androidx.media3.exoplayer.audio.AudioRendererEventListener;
@ -32,6 +33,12 @@ import java.util.Objects;
/** Decodes and renders audio using the native IAMF decoder. */
public class LibiamfAudioRenderer extends DecoderAudioRenderer<IamfDecoder> {
// TODO(ktrajkovski): Values need to be configured and must come from the same source of truth as
// in {@link IamfDecoder}.
private static final int BINAURAL_CHANNEL_COUNT = 2;
private static final int DEFAULT_OUTPUT_SAMPLE_RATE = 48000;
private static final int DEFAULT_PCM_ENCODING = C.ENCODING_PCM_16BIT;
/**
* Creates a new instance.
*
@ -81,7 +88,8 @@ public class LibiamfAudioRenderer extends DecoderAudioRenderer<IamfDecoder> {
@Override
protected Format getOutputFormat(IamfDecoder decoder) {
throw new UnsupportedOperationException();
return Util.getPcmFormat(
DEFAULT_PCM_ENCODING, BINAURAL_CHANNEL_COUNT, DEFAULT_OUTPUT_SAMPLE_RATE);
}
@Override

14
libraries/decoder_iamf/src/main/jni/iamf_jni.cc
View File

@ -58,6 +58,9 @@ IAMF_DecoderHandle handle;
DECODER_FUNC(jint, iamfConfigDecoder, jbyteArray initializationDataArray) {
handle = IAMF_decoder_open();
// TODO(ktrajkovski): Values need to be aligned with IamfDecoder and
// LibiamfAudioRenderer and/or extracted from ConfigOBUs.
IAMF_decoder_peak_limiter_enable(handle, 0);
IAMF_decoder_peak_limiter_set_threshold(handle, -1.0f);
IAMF_decoder_set_normalization_loudness(handle, 0.0f);
@ -78,4 +81,15 @@ DECODER_FUNC(jint, iamfConfigDecoder, jbyteArray initializationDataArray) {
return status;
}
DECODER_FUNC(jint, iamfDecode, jobject inputBuffer, jint inputSize,
jobject outputBuffer) {
uint32_t* rsize = nullptr;
return IAMF_decoder_decode(
handle,
reinterpret_cast<const uint8_t*>(
env->GetDirectBufferAddress(inputBuffer)),
inputSize, rsize,
reinterpret_cast<void*>(env->GetDirectBufferAddress(outputBuffer)));
}
DECODER_FUNC(void, iamfClose) { IAMF_decoder_close(handle); }

17
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java
View File

@ -547,6 +547,23 @@ public class DefaultRenderersFactory implements RenderersFactory {
// The extension is present, but instantiation failed.
throw new RuntimeException("Error instantiating FFmpeg extension", e);
}
try {
// Full class names used for constructor args so the LINT rule triggers if any of them move.
Class<?> clazz = Class.forName("androidx.media3.decoder.iamf.LibiamfAudioRenderer");
Constructor<?> constructor =
clazz.getConstructor(
android.os.Handler.class,
androidx.media3.exoplayer.audio.AudioRendererEventListener.class,
androidx.media3.exoplayer.audio.AudioSink.class);
Renderer renderer =
(Renderer) constructor.newInstance(eventHandler, eventListener, audioSink);
out.add(extensionRendererIndex++, renderer);
} catch (ClassNotFoundException e) {
// Expected if the app was built without the extension.
} catch (Exception e) {
// The extension is present, but instantiation failed.
throw new RuntimeException("Error instantiating IAMF extension", e);
}
}
/**

382
libraries/test_data/src/test/assets/audiosinkdumps/mp4/sample_iamf.mp4.audiosink.dump Normal file
View File

@ -0,0 +1,382 @@
AudioSink:
buffer count = 125
discontinuity:
config:
pcmEncoding = 2
channelCount = 2
sampleRate = 48000
buffer #0:
time = 1000000000000
data = -1112365151
buffer #1:
time = 1000000004000
data = -1667344575
buffer #2:
time = 1000000008000
data = -1973614204
buffer #3:
time = 1000000012000
data = 1093907329
buffer #4:
time = 1000000016000
data = 1166333761
buffer #5:
time = 1000000020000
data = -1080159356
buffer #6:
time = 1000000024000
data = 399031681
buffer #7:
time = 1000000028000
data = -1744152767
buffer #8:
time = 1000000032000
data = -1965613692
buffer #9:
time = 1000000036000
data = 595533569
buffer #10:
time = 1000000040000
data = 1026981825
buffer #11:
time = 1000000044000
data = -1945622652
buffer #12:
time = 1000000048000
data = 699779201
buffer #13:
time = 1000000052000
data = -1931845567
buffer #14:
time = 1000000056000
data = 1664168324
buffer #15:
time = 1000000060000
data = -19505471
buffer #16:
time = 1000000064000
data = -1395371007
buffer #17:
time = 1000000068000
data = -1336788092
buffer #18:
time = 1000000072000
data = -302449791
buffer #19:
time = 1000000076000
data = -164347583
buffer #20:
time = 1000000080000
data = 325797252
buffer #21:
time = 1000000084000
data = -1215027391
buffer #22:
time = 1000000088000
data = 783523713
buffer #23:
time = 1000000092000
data = 746344324
buffer #24:
time = 1000000096000
data = 1618650945
buffer #25:
time = 1000000100000
data = -1158174335
buffer #26:
time = 1000000104000
data = -1667344575
buffer #27:
time = 1000000108000
data = -1973614204
buffer #28:
time = 1000000112000
data = 1093907329
buffer #29:
time = 1000000116000
data = 1166333761
buffer #30:
time = 1000000120000
data = -1080159356
buffer #31:
time = 1000000124000
data = 399031681
buffer #32:
time = 1000000128000
data = -1744152767
buffer #33:
time = 1000000132000
data = -1965613692
buffer #34:
time = 1000000136000
data = 595533569
buffer #35:
time = 1000000140000
data = 1026981825
buffer #36:
time = 1000000144000
data = -1945622652
buffer #37:
time = 1000000148000
data = 699779201
buffer #38:
time = 1000000152000
data = -1931845567
buffer #39:
time = 1000000156000
data = 1664168324
buffer #40:
time = 1000000160000
data = -19505471
buffer #41:
time = 1000000164000
data = -1395371007
buffer #42:
time = 1000000168000
data = -1336788092
buffer #43:
time = 1000000172000
data = -302449791
buffer #44:
time = 1000000176000
data = -164347583
buffer #45:
time = 1000000180000
data = 325797252
buffer #46:
time = 1000000184000
data = -1215027391
buffer #47:
time = 1000000188000
data = 783523713
buffer #48:
time = 1000000192000
data = 746344324
buffer #49:
time = 1000000196000
data = 1618650945
buffer #50:
time = 1000000200000
data = -1158174335
buffer #51:
time = 1000000204000
data = -1667344575
buffer #52:
time = 1000000208000
data = -1973614204
buffer #53:
time = 1000000212000
data = 1093907329
buffer #54:
time = 1000000216000
data = 1166333761
buffer #55:
time = 1000000220000
data = -1080159356
buffer #56:
time = 1000000224000
data = 399031681
buffer #57:
time = 1000000228000
data = -1744152767
buffer #58:
time = 1000000232000
data = -1965613692
buffer #59:
time = 1000000236000
data = 595533569
buffer #60:
time = 1000000240000
data = 1026981825
buffer #61:
time = 1000000244000
data = -1945622652
buffer #62:
time = 1000000248000
data = 699779201
buffer #63:
time = 1000000252000
data = -1931845567
buffer #64:
time = 1000000256000
data = 1664168324
buffer #65:
time = 1000000260000
data = -19505471
buffer #66:
time = 1000000264000
data = -1395371007
buffer #67:
time = 1000000268000
data = -1336788092
buffer #68:
time = 1000000272000
data = -302449791
buffer #69:
time = 1000000276000
data = -164347583
buffer #70:
time = 1000000280000
data = 325797252
buffer #71:
time = 1000000284000
data = -1215027391
buffer #72:
time = 1000000288000
data = 783523713
buffer #73:
time = 1000000292000
data = 746344324
buffer #74:
time = 1000000296000
data = 1618650945
buffer #75:
time = 1000000300000
data = -1158174335
buffer #76:
time = 1000000304000
data = -1667344575
buffer #77:
time = 1000000308000
data = -1973614204
buffer #78:
time = 1000000312000
data = 1093907329
buffer #79:
time = 1000000316000
data = 1166333761
buffer #80:
time = 1000000320000
data = -1080159356
buffer #81:
time = 1000000324000
data = 399031681
buffer #82:
time = 1000000328000
data = -1744152767
buffer #83:
time = 1000000332000
data = -1965613692
buffer #84:
time = 1000000336000
data = 595533569
buffer #85:
time = 1000000340000
data = 1026981825
buffer #86:
time = 1000000344000
data = -1945622652
buffer #87:
time = 1000000348000
data = 699779201
buffer #88:
time = 1000000352000
data = -1931845567
buffer #89:
time = 1000000356000
data = 1664168324
buffer #90:
time = 1000000360000
data = -19505471
buffer #91:
time = 1000000364000
data = -1395371007
buffer #92:
time = 1000000368000
data = -1336788092
buffer #93:
time = 1000000372000
data = -302449791
buffer #94:
time = 1000000376000
data = -164347583
buffer #95:
time = 1000000380000
data = 325797252
buffer #96:
time = 1000000384000
data = -1215027391
buffer #97:
time = 1000000388000
data = 783523713
buffer #98:
time = 1000000392000
data = 746344324
buffer #99:
time = 1000000396000
data = 1618650945
buffer #100:
time = 1000000400000
data = -1158174335
buffer #101:
time = 1000000404000
data = -1667344575
buffer #102:
time = 1000000408000
data = -1973614204
buffer #103:
time = 1000000412000
data = 1093907329
buffer #104:
time = 1000000416000
data = 1166333761
buffer #105:
time = 1000000420000
data = -1080159356
buffer #106:
time = 1000000424000
data = 399031681
buffer #107:
time = 1000000428000
data = -1744152767
buffer #108:
time = 1000000432000
data = -1965613692
buffer #109:
time = 1000000436000
data = 595533569
buffer #110:
time = 1000000440000
data = 1026981825
buffer #111:
time = 1000000444000
data = -1945622652
buffer #112:
time = 1000000448000
data = 699779201
buffer #113:
time = 1000000452000
data = -1931845567
buffer #114:
time = 1000000456000
data = 1664168324
buffer #115:
time = 1000000460000
data = -19505471
buffer #116:
time = 1000000464000
data = -1395371007
buffer #117:
time = 1000000468000
data = -1336788092
buffer #118:
time = 1000000472000
data = -302449791
buffer #119:
time = 1000000476000
data = -164347583
buffer #120:
time = 1000000480000
data = 325797252
buffer #121:
time = 1000000484000
data = -1215027391
buffer #122:
time = 1000000488000
data = 783523713
buffer #123:
time = 1000000492000
data = 746344324
buffer #124:
time = 1000000496000
data = 1026348167