Merge Issue: androidx/media#1826: add extension for MPEG-H decoding

Imported from GitHub PR https://github.com/androidx/media/pull/1826

Merge 6b59a1602b022ebc44411ae3440e274c51c223a7 into b5615d5e919b297def6450b45320a3165c34548c

COPYBARA_INTEGRATE_REVIEW=https://github.com/androidx/media/pull/1826 from androidx:mpegh_extension 6b59a1602b022ebc44411ae3440e274c51c223a7
PiperOrigin-RevId: 689417378

RELEASENOTES.md

@@ -52,6 +52,9 @@
 *   Smooth Streaming Extension:
 *   RTSP Extension:
 *   Decoder Extensions (FFmpeg, VP9, AV1, etc.):
+    *   Add the MPEG-H decoder module which uses the native MPEG-H decoder
+        module to decode MPEG-H audio
+        ([#1826](https://github.com/androidx/media/pull/1826)).
 *   MIDI extension:
 *   Leanback extension:
 *   Cast Extension:

core_settings.gradle

@@ -83,6 +83,8 @@ if (gradle.ext.has('androidxMediaEnableMidiModule') && gradle.ext.androidxMediaE
     include modulePrefix + 'lib-decoder-midi'
     project(modulePrefix + 'lib-decoder-midi').projectDir = new File(rootDir, 'libraries/decoder_midi')
 }
+include modulePrefix + 'lib-decoder-mpegh'
+project(modulePrefix + 'lib-decoder-mpegh').projectDir = new File(rootDir, 'libraries/decoder_mpegh')
 include modulePrefix + 'lib-decoder-opus'
 project(modulePrefix + 'lib-decoder-opus').projectDir = new File(rootDir, 'libraries/decoder_opus')
 include modulePrefix + 'lib-decoder-vp9'

demos/main/build.gradle

@@ -90,6 +90,7 @@ dependencies {
     withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-iamf')
     withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-vp9')
     withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-midi')
+    withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-mpegh')
     withDecoderExtensionsImplementation project(modulePrefix + 'lib-datasource-rtmp')
 }
 

demos/main/src/main/assets/media.exolist.json

@@ -762,6 +762,10 @@
       {
         "name": "Immersive Audio Format Sample (MP4, IAMF)",
         "uri": "https://github.com/AOMediaCodec/libiamf/raw/main/tests/test_000036_s.mp4"
+      },
+      {
+        "name": "MPEG-H HD (MP4, H265)",
+        "uri": "https://media.githubusercontent.com/media/Fraunhofer-IIS/mpegh-test-content/main/TRI_Fileset_17_514H_D1_D2_D3_O1_24bit1080p50.mp4"
       }
     ]
   },

libraries/decoder_mpegh/README.md

@@ -0,0 +1,89 @@
+# MPEG-H decoder module
+
+The MPEG-H module provides `MpeghAudioRenderer`, which uses the libmpegh native
+library to decode MPEG-H audio.
+
+## License note
+
+Please note that while the code in this repository is licensed under
+[Apache 2.0][], using this module also requires building and including the
+[Fraunhofer GitHub MPEG-H decoder][] which is licensed under the Fraunhofer
+GitHub MPEG-H decoder project.
+
+[Apache 2.0]: ../../LICENSE
+[Fraunhofer GitHub MPEG-H decoder]: https://github.com/Fraunhofer-IIS/mpeghdec
+
+## Build instructions (Linux, macOS)
+
+To use the module you need to clone this GitHub project and depend on its
+modules locally. Instructions for doing this can be found in the
+[top level README][].
+
+In addition, it's necessary to fetch libmpegh library as follows:
+
+*   Set the following environment variables:
+
+```
+cd "<path to project checkout>"
+MPEGH_MODULE_PATH="$(pwd)/libraries/decoder_mpegh/src/main"
+```
+
+*   Fetch libmpegh library:
+
+```
+cd "${MPEGH_MODULE_PATH}/jni" && \
+git clone https://github.com/Fraunhofer-IIS/mpeghdec.git --branch r2.0.0 libmpegh
+```
+
+*   [Install CMake][].
+
+Having followed these steps, gradle will build the module automatically when run
+on the command line or via Android Studio, using [CMake][] and [Ninja][] to
+configure and build mpeghdec and the module's [JNI wrapper library][].
+
+[top level README]: ../../README.md
+[Install CMake]: https://developer.android.com/studio/projects/install-ndk
+[CMake]: https://cmake.org/
+[Ninja]: https://ninja-build.org
+[JNI wrapper library]: src/main/jni/mpeghdec_jni.cc
+
+## Build instructions (Windows)
+
+We do not provide support for building this module on Windows, however it should
+be possible to follow the Linux instructions in [Windows PowerShell][].
+
+[Windows PowerShell]: https://docs.microsoft.com/en-us/powershell/scripting/getting-started/getting-started-with-windows-powershell
+
+## Using the module with ExoPlayer
+
+Once you've followed the instructions above to check out, build and depend on
+the module, the next step is to tell ExoPlayer to use `MpeghAudioRenderer`. How
+you do this depends on which player API you're using:
+
+*   If you're passing a `DefaultRenderersFactory` to `ExoPlayer.Builder`, you
+    can enable using the module by setting the `extensionRendererMode` parameter
+    of the `DefaultRenderersFactory` constructor to
+    `EXTENSION_RENDERER_MODE_ON`. This will use `MpeghAudioRenderer` for
+    playback if `MediaCodecAudioRenderer` doesn't support the input format. Pass
+    `EXTENSION_RENDERER_MODE_PREFER` to give `MpeghAudioRenderer` priority over
+    `MediaCodecAudioRenderer`.
+*   If you've subclassed `DefaultRenderersFactory`, add a `MpeghAudioRenderer`
+    to the output list in `buildAudioRenderers`. ExoPlayer will use the first
+    `Renderer` in the list that supports the input media format.
+*   If you've implemented your own `RenderersFactory`, return a
+    `MpeghAudioRenderer` instance from `createRenderers`. ExoPlayer will use the
+    first `Renderer` in the returned array that supports the input media format.
+*   If you're using `ExoPlayer.Builder`, pass a `MpeghAudioRenderer` in the
+    array of `Renderer`s. ExoPlayer will use the first `Renderer` in the list
+    that supports the input media format.
+
+Note: These instructions assume you're using `DefaultTrackSelector`. If you have
+a custom track selector the choice of `Renderer` is up to your implementation,
+so you need to make sure you are passing an `MpeghAudioRenderer` to the player,
+then implement your own logic to use the renderer for a given track.
+
+## Links
+
+*   [Troubleshooting using decoding extensions][]
+
+[Troubleshooting using decoding extensions]: https://developer.android.com/media/media3/exoplayer/troubleshooting#how-can-i-get-a-decoding-library-to-load-and-be-used-for-playback

libraries/decoder_mpegh/build.gradle

@@ -0,0 +1,65 @@
+// Copyright 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+apply from: "$gradle.ext.androidxMediaSettingsDir/common_library_config.gradle"
+
+android {
+    namespace 'androidx.media3.decoder.mpegh'
+
+    sourceSets {
+        androidTest.assets.srcDir '../test_data/src/test/assets'
+    }
+
+    defaultConfig {
+        externalNativeBuild {
+            cmake {
+                targets "mpeghJNI"
+            }
+        }
+    }
+
+    // TODO(Internal: b/372449691): Remove packagingOptions once AGP is updated
+    // to version 8.5.1 or higher.
+    packagingOptions {
+        jniLibs {
+            useLegacyPackaging true
+        }
+    }
+}
+
+// Configure the native build only if libmpegh is present to avoid gradle sync
+// failures if libmpegh hasn't been built according to the README instructions.
+if (project.file('src/main/jni/libmpegh').exists()) {
+    android.externalNativeBuild.cmake {
+        path = 'src/main/jni/CMakeLists.txt'
+        version = '3.21.0+'
+        if (project.hasProperty('externalNativeBuildDir')) {
+            if (!new File(externalNativeBuildDir).isAbsolute()) {
+                ext.externalNativeBuildDir =
+                        new File(rootDir, it.externalNativeBuildDir)
+            }
+            buildStagingDirectory = "${externalNativeBuildDir}/${project.name}"
+        }
+    }
+}
+
+dependencies {
+    implementation project(modulePrefix + 'lib-decoder')
+    // TODO(b/203752526): Remove this dependency.
+    implementation project(modulePrefix + 'lib-exoplayer')
+    implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
+    compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion
+    compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion
+    testImplementation project(modulePrefix + 'test-utils')
+    testImplementation 'org.robolectric:robolectric:' + robolectricVersion
+}

libraries/decoder_mpegh/proguard-rules.txt

@@ -0,0 +1,13 @@
+# Proguard rules specific to the MPEG-H extension.
+
+# This prevents the names of native methods from being obfuscated.
+-keepclasseswithmembernames class * {
+    native <methods>;
+}
+
+# Some members of this class are being accessed from native methods. Keep them unobfuscated.
+-keep class androidx.media3.decoder.SimpleDecoderOutputBuffer {
+    *;
+}
+
+-keep class androidx.media3.decoder.mpegh** { *; }

libraries/decoder_mpegh/src/main/AndroidManifest.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2024 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+
+<manifest package="androidx.media3.decoder.mpegh"/>

libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.decoder.mpegh;
+
+import android.os.Handler;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.audio.AudioProcessor;
+import androidx.media3.common.util.TraceUtil;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.CryptoConfig;
+import androidx.media3.exoplayer.DecoderReuseEvaluation;
+import androidx.media3.exoplayer.audio.AudioRendererEventListener;
+import androidx.media3.exoplayer.audio.AudioSink;
+import androidx.media3.exoplayer.audio.DecoderAudioRenderer;
+import java.util.Objects;
+
+/** Decodes and renders audio using the native MPEG-H decoder. */
+@UnstableApi
+public final class MpeghAudioRenderer extends DecoderAudioRenderer<MpeghDecoder> {
+
+  private static final String TAG = "MpeghAudioRenderer";
+
+  /** The number of input and output buffers. */
+  private static final int NUM_BUFFERS = 16;
+
+  /*  Creates a new instance. */
+  public MpeghAudioRenderer() {
+    this(/* eventHandler= */ null, /* eventListener= */ null);
+  }
+
+  /**
+   * Creates a new instance.
+   *
+   * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
+   *     null if delivery of events is not required.
+   * @param eventListener A listener of events. May be null if delivery of events is not required.
+   * @param audioProcessors Optional {@link AudioProcessor}s that will process audio before output.
+   */
+  public MpeghAudioRenderer(
+      Handler eventHandler,
+      AudioRendererEventListener eventListener,
+      AudioProcessor... audioProcessors) {
+    super(eventHandler, eventListener, audioProcessors);
+  }
+
+  /**
+   * Creates a new instance.
+   *
+   * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
+   *     null if delivery of events is not required.
+   * @param eventListener A listener of events. May be null if delivery of events is not required.
+   * @param audioSink The sink to which audio will be output.
+   */
+  public MpeghAudioRenderer(
+      Handler eventHandler, AudioRendererEventListener eventListener, AudioSink audioSink) {
+    super(eventHandler, eventListener, audioSink);
+  }
+
+  @Override
+  public String getName() {
+    return TAG;
+  }
+
+  @Override
+  protected @C.FormatSupport int supportsFormatInternal(Format format) {
+    // Check if JNI library is available.
+    if (!MpeghLibrary.isAvailable()) {
+      return C.FORMAT_UNSUPPORTED_TYPE;
+    }
+
+    // Check if MIME type is supported.
+    if (!(Objects.equals(format.sampleMimeType, MimeTypes.AUDIO_MPEGH_MHM1)
+        || Objects.equals(format.sampleMimeType, MimeTypes.AUDIO_MPEGH_MHA1))) {
+      return C.FORMAT_UNSUPPORTED_TYPE;
+    }
+    return C.FORMAT_HANDLED;
+  }
+
+  @Override
+  protected DecoderReuseEvaluation canReuseDecoder(
+      String decoderName, Format oldFormat, Format newFormat) {
+    if (Objects.equals(oldFormat.sampleMimeType, newFormat.sampleMimeType)
+        && Objects.equals(oldFormat.sampleMimeType, MimeTypes.AUDIO_MPEGH_MHM1)) {
+      return new DecoderReuseEvaluation(
+          decoderName,
+          oldFormat,
+          newFormat,
+          DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION,
+          /* discardReasons= */ 0);
+    }
+    return super.canReuseDecoder(decoderName, oldFormat, newFormat);
+  }
+
+  @Override
+  protected MpeghDecoder createDecoder(Format format, CryptoConfig cryptoConfig)
+      throws MpeghDecoderException {
+    TraceUtil.beginSection("createMpeghDecoder");
+    MpeghDecoder decoder = new MpeghDecoder(format, NUM_BUFFERS, NUM_BUFFERS);
+    TraceUtil.endSection();
+    return decoder;
+  }
+
+  @Override
+  protected Format getOutputFormat(MpeghDecoder decoder) {
+    return new Format.Builder()
+        .setChannelCount(decoder.getChannelCount())
+        .setSampleRate(decoder.getSampleRate())
+        .setSampleMimeType(MimeTypes.AUDIO_RAW)
+        .setPcmEncoding(C.ENCODING_PCM_16BIT)
+        .build();
+  }
+}

libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java

@@ -0,0 +1,188 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.decoder.mpegh;
+
+import androidx.annotation.Nullable;
+import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.common.util.Util;
+import androidx.media3.decoder.DecoderInputBuffer;
+import androidx.media3.decoder.SimpleDecoder;
+import androidx.media3.decoder.SimpleDecoderOutputBuffer;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+/** MPEG-H decoder. */
+@UnstableApi
+public final class MpeghDecoder
+    extends SimpleDecoder<DecoderInputBuffer, SimpleDecoderOutputBuffer, MpeghDecoderException> {
+
+  /** The default input buffer size. */
+  private static final int DEFAULT_INPUT_BUFFER_SIZE = 2048 * 6;
+
+  private static final int TARGET_LAYOUT_CICP = 2;
+
+  private final ByteBuffer tmpOutputBuffer;
+
+  private MpeghDecoderJni decoder;
+  private long outPtsUs;
+  private int outChannels;
+  private int outSampleRate;
+
+  /**
+   * Creates an MPEG-H decoder.
+   *
+   * @param format The input {@link Format}.
+   * @param numInputBuffers The number of input buffers.
+   * @param numOutputBuffers The number of output buffers.
+   * @throws MpeghDecoderException If an exception occurs when initializing the decoder.
+   */
+  public MpeghDecoder(Format format, int numInputBuffers, int numOutputBuffers)
+      throws MpeghDecoderException {
+    super(new DecoderInputBuffer[numInputBuffers], new SimpleDecoderOutputBuffer[numOutputBuffers]);
+    if (!MpeghLibrary.isAvailable()) {
+      throw new MpeghDecoderException("Failed to load decoder native libraries.");
+    }
+
+    byte[] configData = new byte[0];
+    if (!format.initializationData.isEmpty()
+        && Objects.equals(format.sampleMimeType, MimeTypes.AUDIO_MPEGH_MHA1)) {
+      configData = format.initializationData.get(0);
+    }
+
+    // Initialize the native MPEG-H decoder.
+    decoder = new MpeghDecoderJni();
+    decoder.init(TARGET_LAYOUT_CICP, configData, configData.length);
+
+    int initialInputBufferSize =
+        format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE;
+    setInitialInputBufferSize(initialInputBufferSize);
+
+    // Allocate memory for the temporary output of the native MPEG-H decoder.
+    tmpOutputBuffer =
+        ByteBuffer.allocateDirect(
+            3072 * 24 * 6
+                * 2); // MAX_FRAME_LENGTH * MAX_NUM_CHANNELS * MAX_NUM_FRAMES * BYTES_PER_SAMPLE
+  }
+
+  @Override
+  public String getName() {
+    return "libmpegh";
+  }
+
+  @Override
+  protected DecoderInputBuffer createInputBuffer() {
+    return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
+  }
+
+  @Override
+  protected SimpleDecoderOutputBuffer createOutputBuffer() {
+    return new SimpleDecoderOutputBuffer(this::releaseOutputBuffer);
+  }
+
+  @Override
+  protected MpeghDecoderException createUnexpectedDecodeException(Throwable error) {
+    return new MpeghDecoderException("Unexpected decode error", error);
+  }
+
+  @Override
+  @Nullable
+  protected MpeghDecoderException decode(
+      DecoderInputBuffer inputBuffer, SimpleDecoderOutputBuffer outputBuffer, boolean reset) {
+    if (reset) {
+      try {
+        decoder.flush();
+      } catch (MpeghDecoderException e) {
+        return e;
+      }
+    }
+
+    // Get the data from the input buffer.
+    ByteBuffer inputData = Util.castNonNull(inputBuffer.data);
+    int inputSize = inputData.limit();
+    long inputPtsUs = inputBuffer.timeUs;
+
+    // Process/decode the incoming data.
+    try {
+      decoder.process(inputData, inputSize, inputPtsUs);
+    } catch (MpeghDecoderException e) {
+      return e;
+    }
+
+    // Get as many decoded samples as possible.
+    int outputSize = 0;
+    int numBytes = 0;
+    int cnt = 0;
+    tmpOutputBuffer.clear();
+    do {
+      try {
+        outputSize = decoder.getSamples(tmpOutputBuffer, numBytes);
+      } catch (MpeghDecoderException e) {
+        return e;
+      }
+      // To concatenate possible additional audio frames, increase the write position.
+      numBytes += outputSize;
+
+      if (cnt == 0 && outputSize > 0) {
+        // Only use the first frame for info about PTS, number of channels and sample rate.
+        outPtsUs = decoder.getPts();
+        outChannels = decoder.getNumChannels();
+        outSampleRate = decoder.getSamplerate();
+      }
+
+      cnt++;
+    } while (outputSize > 0);
+
+    int outputSizeTotal = numBytes;
+    tmpOutputBuffer.limit(outputSizeTotal);
+
+    if (outputSizeTotal > 0) {
+      // There is output data available
+
+      // initialize the output buffer
+      outputBuffer.clear();
+      outputBuffer.init(outPtsUs, outputSizeTotal);
+
+      // copy temporary output to output buffer
+      outputBuffer.data.asShortBuffer().put(tmpOutputBuffer.asShortBuffer());
+      outputBuffer.data.rewind();
+    } else {
+      // if no output data is available signalize that only decoding/processing was possible
+      outputBuffer.shouldBeSkipped = true;
+    }
+    return null;
+  }
+
+  @Override
+  public void release() {
+    super.release();
+    if (decoder != null) {
+      decoder.destroy();
+      decoder = null;
+    }
+  }
+
+  /** Returns the channel count of output audio. */
+  public int getChannelCount() {
+    return outChannels;
+  }
+
+  /** Returns the sample rate of output audio. */
+  public int getSampleRate() {
+    return outSampleRate;
+  }
+}

libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoderException.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.decoder.mpegh;
+
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.DecoderException;
+
+/** Thrown when an MPEG-H decoder error occurs. */
+@UnstableApi
+public class MpeghDecoderException extends DecoderException {
+
+  public MpeghDecoderException(String message) {
+    super(message, new Throwable());
+  }
+
+  public MpeghDecoderException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoderJni.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.decoder.mpegh;
+
+import java.nio.ByteBuffer;
+
+/** JNI wrapper for the libmpegh MPEG-H decoder. */
+public class MpeghDecoderJni {
+
+  private long decoderHandle; // used by JNI only to hold the native context.
+
+  public MpeghDecoderJni() {}
+
+  /**
+   * Initializes the native MPEG-H decoder.
+   *
+   * @param cicpIndex The desired target layout CICP index.
+   * @param mhaConfig The byte array holding the audio specific configuration for MHA content.
+   * @param mhaConfigLength Length of audio specific configuration.
+   * @throws MpeghDecoderException If initialization fails.
+   */
+  public native void init(int cicpIndex, byte[] mhaConfig, int mhaConfigLength)
+      throws MpeghDecoderException;
+
+  /** Destroys the native MPEG-H decoder. */
+  public native void destroy();
+
+  /**
+   * Processes data (access units) and corresponding PTS inside of the native MPEG-H decoder.
+   *
+   * @param inputBuffer The direct byte buffer holding the access unit.
+   * @param inputLength The length of the direct byte buffer.
+   * @param timestampUs The presentation timestamp of the access unit, in microseconds.
+   * @throws MpeghDecoderException If processing fails.
+   */
+  public native void process(ByteBuffer inputBuffer, int inputLength, long timestampUs)
+      throws MpeghDecoderException;
+
+  /**
+   * Obtains decoded samples from the native MPEG-H decoder and writes them into {@code buffer} at
+   * position {@code writePos}.
+   *
+   * <p>NOTE: The decoder returns the samples as 16bit values.
+   *
+   * @param buffer The direct byte buffer to write the decoded samples to.
+   * @param writePos The start position in the byte buffer to write the decoded samples to.
+   * @return The number of bytes written to buffer.
+   * @throws MpeghDecoderException If obtaining samples fails.
+   */
+  public native int getSamples(ByteBuffer buffer, int writePos) throws MpeghDecoderException;
+
+  /**
+   * Flushes the native MPEG-H decoder and writes available output samples into a sample queue.
+   *
+   * @throws MpeghDecoderException If flushing fails.
+   */
+  public native void flushAndGet() throws MpeghDecoderException;
+
+  /**
+   * Gets the number of output channels from the native MPEG-H decoder.
+   *
+   * <p>NOTE: This information belongs to the last audio frame obtained from {@link
+   * #getSamples(ByteBuffer, int)} or {@link #flushAndGet()}.
+   *
+   * @return The number of output channels.
+   */
+  public native int getNumChannels();
+
+  /**
+   * Gets the output sample rate from the native MPEG-H decoder.
+   *
+   * <p>NOTE: This information belongs to the last audio frame obtained from {@link
+   * #getSamples(ByteBuffer, int)} or {@link #flushAndGet()}.
+   *
+   * @return The output sample rate.
+   */
+  public native int getSamplerate();
+
+  /**
+   * Gets the PTS from the native MPEG-H decoder, in microseconds.
+   *
+   * <p>NOTE: This information belongs to the last audio frame obtained from {@link
+   * #getSamples(ByteBuffer, int)} or {@link #flushAndGet()}.
+   *
+   * @return The output presentation timestamp.
+   */
+  public native long getPts();
+
+  /**
+   * Flushes the native MPEG-H decoder.
+   *
+   * @throws MpeghDecoderException If flushing fails.
+   */
+  public native void flush() throws MpeghDecoderException;
+}

libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghLibrary.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.decoder.mpegh;
+
+import androidx.media3.common.MediaLibraryInfo;
+import androidx.media3.common.util.LibraryLoader;
+import androidx.media3.common.util.UnstableApi;
+
+/** Configures and queries the underlying native library. */
+@UnstableApi
+public final class MpeghLibrary {
+
+  static {
+    MediaLibraryInfo.registerModule("media3.decoder.mpegh");
+  }
+
+  private static final LibraryLoader LOADER =
+      new LibraryLoader("mpeghJNI") {
+        @Override
+        protected void loadLibrary(String name) {
+          System.loadLibrary(name);
+        }
+      };
+
+  private MpeghLibrary() {}
+
+  /**
+   * Override the names of the MPEG-H native libraries. If an application wishes to call this
+   * method, it must do so before calling any other method defined by this class, and before
+   * instantiating a {@link MpeghAudioRenderer} instance.
+   *
+   * @param libraries The names of the MPEG-H native libraries.
+   */
+  public static void setLibraries(String... libraries) {
+    LOADER.setLibraries(libraries);
+  }
+
+  /** Returns whether the underlying library is available, loading it if necessary. */
+  public static boolean isAvailable() {
+    return LOADER.isAvailable();
+  }
+}

libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/package-info.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@NonNullApi
+package androidx.media3.decoder.mpegh;
+
+import androidx.media3.common.util.NonNullApi;

libraries/decoder_mpegh/src/main/jni/CMakeLists.txt

@@ -0,0 +1,57 @@
+#
+# Copyright 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+# Enable C++11 features.
+set(CMAKE_CXX_STANDARD 11)
+
+# Define project name for your JNI module
+project(libmpeghJNI C CXX)
+
+if(${ANDROID_ABI} MATCHES "armeabi-v7a")
+  add_compile_options("-mfpu=neon")
+  add_compile_options("-marm")
+  add_compile_options("-fPIC")
+endif()
+
+set(libmpegh_jni_root "${CMAKE_CURRENT_SOURCE_DIR}")
+
+# Build libmpegh.
+add_subdirectory("${libmpegh_jni_root}/libmpegh"
+                 EXCLUDE_FROM_ALL)
+
+# Add the include directory from libmpegh.
+include_directories ("${libmpegh_jni_root}/libmpegh/include")
+
+# Build libmpeghJNI.
+add_library(mpeghJNI
+            SHARED
+            mpegh_jni.cpp)
+
+# Locate NDK log library.
+find_library(android_log_lib log)
+
+# Link libmpeghJNI against used libraries.
+target_link_libraries(mpeghJNI
+                      PRIVATE android
+                      PRIVATE mpeghdec
+                      PRIVATE ${android_log_lib})
+
+# Enable 16 KB ELF alignment.
+target_link_options(mpeghJNI
+                    PRIVATE "-Wl,-z,max-page-size=16384")
+

libraries/decoder_mpegh/src/main/jni/mpegh_jni.cpp

@@ -0,0 +1,255 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <android/log.h>
+#include <jni.h>
+
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+
+#include "../include/mpeghdecoder.h"
+
+#define LOG_TAG "mpeghdec_jni"
+#define LOGE(...) \
+  ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__))
+#define LOGW(...) \
+  ((void)__android_log_print(ANDROID_LOG_WARN, LOG_TAG, __VA_ARGS__))
+#define LOGI(...) \
+  ((void)__android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__))
+#define LOGD(...) \
+  ((void)__android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__))
+
+#define DECODER_FUNC(RETURN_TYPE, NAME, ...)                     \
+  extern "C" {                                                   \
+  JNIEXPORT RETURN_TYPE                                          \
+      Java_androidx_media3_decoder_mpegh_MpeghDecoderJni_##NAME( \
+          JNIEnv *env, jobject obj, ##__VA_ARGS__);              \
+  }                                                              \
+  JNIEXPORT RETURN_TYPE                                          \
+      Java_androidx_media3_decoder_mpegh_MpeghDecoderJni_##NAME( \
+          JNIEnv *env, jobject obj, ##__VA_ARGS__)
+
+#define EXCEPTION_PATH "androidx/media3/decoder/mpegh/MpeghDecoderException"
+
+#define MAX_NUM_FRAMES (6)
+#define MAX_FRAME_LENGTH (3072)
+#define MAX_NUM_CHANNELS (24)
+#define BYTES_PER_SAMPLE (2)
+#define MAX_OUTBUF_SIZE_SAMPLES \
+  (MAX_NUM_FRAMES * MAX_FRAME_LENGTH * MAX_NUM_CHANNELS)
+
+typedef struct DECODER_CONTEXT {
+  int32_t outSampleRate;
+  int32_t outNumChannels;
+  int64_t outPts;
+
+  HANDLE_MPEGH_DECODER_CONTEXT handle;
+  int32_t samples[MAX_OUTBUF_SIZE_SAMPLES];
+} DECODER_CONTEXT;
+
+jfieldID getHandleFieldID(JNIEnv *env, jobject obj) {
+  jclass cls = env->GetObjectClass(obj);
+  return env->GetFieldID(cls, "decoderHandle", "J");
+}
+
+void setContext(JNIEnv *env, jobject obj, DECODER_CONTEXT *ctx) {
+  jfieldID decoderHandle_fid = getHandleFieldID(env, obj);
+  env->SetLongField(obj, decoderHandle_fid, (jlong)ctx);
+}
+
+DECODER_CONTEXT *getContext(JNIEnv *env, jobject obj) {
+  jfieldID decoderHandle_fid = getHandleFieldID(env, obj);
+  return (DECODER_CONTEXT *)env->GetLongField(obj, decoderHandle_fid);
+}
+
+/*
+ * Method: init
+ * will be used to initialize the JNI MPEG-H decoder wrapper.
+ */
+DECODER_FUNC(void, init, jint cicpIndex, jbyteArray mhaConfig,
+             jint mhaConfigLength) {
+  // create JNI decoder wrapper context
+  auto *ctx = (DECODER_CONTEXT *)calloc(1, sizeof(DECODER_CONTEXT));
+  if (ctx == nullptr) {
+    LOGE("Unable to allocate memory for DECODER_CONTEXT!");
+    jclass atscExCls = env->FindClass(EXCEPTION_PATH);
+    env->ThrowNew(atscExCls, "cannot create DECODER_CONTEXT");
+    return;
+  }
+
+  // create MPEG-H decoder
+  ctx->handle = mpeghdecoder_init(cicpIndex);
+  if (ctx->handle == nullptr) {
+    LOGE("Cannot create mpeghdecoder with CICP = %d!", cicpIndex);
+    jclass atscExCls = env->FindClass(EXCEPTION_PATH);
+    env->ThrowNew(atscExCls, "Cannot create mpeghdecoder");
+    return;
+  }
+
+  if (mhaConfigLength > 0) {
+    auto *cData = (jbyte *)calloc(mhaConfigLength, sizeof(jbyte));
+    env->GetByteArrayRegion(mhaConfig, 0, mhaConfigLength, cData);
+
+    MPEGH_DECODER_ERROR result = mpeghdecoder_setMhaConfig(
+        ctx->handle, (unsigned char *)cData, (uint32_t)mhaConfigLength);
+    free(cData);
+    if (result != MPEGH_DEC_OK) {
+      LOGE("Cannot set MHA config!");
+      jclass atscExCls = env->FindClass(EXCEPTION_PATH);
+      env->ThrowNew(atscExCls, "Cannot set MHA config");
+      return;
+    }
+  }
+
+  // store the wrapper context in JNI env
+  setContext(env, obj, ctx);
+}
+
+/*
+ * Method: destroy
+ * will be called to destroy the JNI MPEG-H decoder wrapper.
+ */
+DECODER_FUNC(void, destroy) {
+  DECODER_CONTEXT *ctx = getContext(env, obj);
+
+  mpeghdecoder_destroy(ctx->handle);
+  free(ctx);
+}
+
+/*
+ * Method: process
+ * will be called to pass the received MHAS frame to the decoder.
+ */
+DECODER_FUNC(void, process, jobject inputBuffer, jint inputLength,
+             jlong timestampUs) {
+  DECODER_CONTEXT *ctx = getContext(env, obj);
+
+  // get memory pointer to the buffer of the corresponding JAVA input parameter
+  auto *inData = (const uint8_t *)env->GetDirectBufferAddress(inputBuffer);
+  auto inDataLen = (uint32_t)inputLength;
+  auto ptsIn = (uint64_t)timestampUs;
+
+  MPEGH_DECODER_ERROR result =
+      mpeghdecoder_process(ctx->handle, inData, inDataLen, ptsIn * 1000);
+  if (result != MPEGH_DEC_OK) {
+    LOGW("Unable to feed new data with return value = %d", result);
+    jclass atscExCls = env->FindClass(EXCEPTION_PATH);
+    env->ThrowNew(atscExCls, "Unable to feed new data!");
+  }
+}
+
+/*
+ * Method: getSamples
+ * will be called to receive the decoded PCM.
+ */
+DECODER_FUNC(jint, getSamples, jobject buffer, jint writePos) {
+  DECODER_CONTEXT *ctx = getContext(env, obj);
+
+  // get memory pointer to the buffer of the corresponding JAVA input parameter
+  auto *outData = (uint8_t *)env->GetDirectBufferAddress(buffer);
+  if (outData == nullptr) {
+    LOGE("not possible to get direct byte buffer!");
+    jclass atscExCls = env->FindClass(EXCEPTION_PATH);
+    env->ThrowNew(atscExCls, "not possible to get direct byte buffer!");
+    return 0;
+  }
+
+  unsigned int outNumSamples = 0;
+  MPEGH_DECODER_OUTPUT_INFO outInfo;
+
+  MPEGH_DECODER_ERROR result = mpeghdecoder_getSamples(
+      ctx->handle, ctx->samples, MAX_OUTBUF_SIZE_SAMPLES, &outInfo);
+
+  if (result == MPEGH_DEC_OK) {
+    outNumSamples = outInfo.numSamplesPerChannel;
+    if (outNumSamples > 0) {
+      for (int i = 0; i < outNumSamples * outInfo.numChannels; i++) {
+        ctx->samples[i] = ctx->samples[i] >> 16;
+        outData[writePos + i * 2 + 0] = (ctx->samples[i] >> 8) & 0xFF;
+        outData[writePos + i * 2 + 1] = (ctx->samples[i]) & 0xFF;
+      }
+    }
+    ctx->outSampleRate = outInfo.sampleRate;
+    ctx->outNumChannels = outInfo.numChannels;
+    ctx->outPts = outInfo.pts / 1000;
+  } else {
+    ctx->outSampleRate = -1;
+    ctx->outNumChannels = -1;
+    ctx->outPts = -1;
+  }
+
+  return outNumSamples * ctx->outNumChannels * BYTES_PER_SAMPLE;
+}
+
+/*
+ * Method: flushAndGet
+ * will be called to force the decoder to flush the internal PCM buffer and
+ * write available output samples into a sample queue.
+ */
+DECODER_FUNC(void, flushAndGet) {
+  DECODER_CONTEXT *ctx = getContext(env, obj);
+
+  MPEGH_DECODER_ERROR result = mpeghdecoder_flushAndGet(ctx->handle);
+  if (result != MPEGH_DEC_OK) {
+    LOGE("Unable to flush data with return value = %d", result);
+    jclass atscExCls = env->FindClass(EXCEPTION_PATH);
+    env->ThrowNew(atscExCls, "Unable to flush data!");
+  }
+}
+
+/*
+ * Method: getNumChannels
+ * will be called to receive the number of output channels.
+ */
+DECODER_FUNC(jint, getNumChannels) {
+  DECODER_CONTEXT *ctx = getContext(env, obj);
+  return ctx->outNumChannels;
+}
+
+/*
+ * Method: getSamplerate
+ * will be called to receive the output sample rate.
+ */
+DECODER_FUNC(jint, getSamplerate) {
+  // get wrapper context from JNI env
+  DECODER_CONTEXT *ctx = getContext(env, obj);
+  return ctx->outSampleRate;
+}
+
+/*
+ * Method: getPts
+ * will be called to receive the output PTS.
+ */
+DECODER_FUNC(jlong, getPts) {
+  DECODER_CONTEXT *ctx = getContext(env, obj);
+  return ctx->outPts;
+}
+
+/*
+ * Method: flush
+ * will be called to force the decoder to flush the internal PCM buffer.
+ */
+DECODER_FUNC(void, flush) {
+  DECODER_CONTEXT *ctx = getContext(env, obj);
+
+  MPEGH_DECODER_ERROR result = mpeghdecoder_flush(ctx->handle);
+  if (result != MPEGH_DEC_OK) {
+    LOGE("Unable to flush data with return value = %d", result);
+    jclass atscExCls = env->FindClass(EXCEPTION_PATH);
+    env->ThrowNew(atscExCls, "Unable to flush data!");
+  }
+}

libraries/decoder_mpegh/src/test/AndroidManifest.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2024 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+
+<manifest package="androidx.media3.decoder.mpegh.test">
+  <uses-sdk/>
+</manifest>

libraries/decoder_mpegh/src/test/java/androidx/media3/decoder/mpegh/DefaultRenderersFactoryTest.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.decoder.mpegh;
+
+import androidx.media3.common.C;
+import androidx.media3.test.utils.DefaultRenderersFactoryAsserts;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit test for {@link DefaultRenderersFactoryTest} with {@link MpeghAudioRenderer}. */
+@RunWith(AndroidJUnit4.class)
+public final class DefaultRenderersFactoryTest {
+
+  @Test
+  public void createRenderers_instantiatesMpeghAudioRenderer() {
+    DefaultRenderersFactoryAsserts.assertExtensionRendererCreated(
+        MpeghAudioRenderer.class, C.TRACK_TYPE_AUDIO);
+  }
+}

libraries/exoplayer/proguard-rules.txt

@@ -33,6 +33,10 @@
 -keepclassmembers class androidx.media3.decoder.midi.MidiRenderer {
   <init>(android.content.Context);
 }
+-dontnote androidx.media3.decoder.mpegh.MpeghAudioRenderer
+-keepclassmembers class androidx.media3.decoder.mpegh.MpeghAudioRenderer {
+  <init>(android.os.Handler, androidx.media3.exoplayer.audio.AudioRendererEventListener, androidx.media3.exoplayer.audio.AudioSink);
+}
 
 # Constructors accessed via reflection in DefaultDownloaderFactory
 -dontnote androidx.media3.exoplayer.dash.offline.DashDownloader

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java

@@ -567,6 +567,25 @@ public class DefaultRenderersFactory implements RenderersFactory {
       // The extension is present, but instantiation failed.
       throw new IllegalStateException("Error instantiating IAMF extension", e);
     }
+
+    try {
+      // Full class names used for constructor args so the LINT rule triggers if any of them move.
+      Class<?> clazz = Class.forName("androidx.media3.decoder.mpegh.MpeghAudioRenderer");
+      Constructor<?> constructor =
+          clazz.getConstructor(
+              android.os.Handler.class,
+              androidx.media3.exoplayer.audio.AudioRendererEventListener.class,
+              androidx.media3.exoplayer.audio.AudioSink.class);
+      Renderer renderer =
+          (Renderer) constructor.newInstance(eventHandler, eventListener, audioSink);
+      out.add(extensionRendererIndex++, renderer);
+      Log.i(TAG, "Loaded MpeghAudioRenderer.");
+    } catch (ClassNotFoundException e) {
+      // Expected if the app was built without the extension.
+    } catch (Exception e) {
+      // The extension is present, but instantiation failed.
+      throw new IllegalStateException("Error instantiating MPEG-H extension", e);
+    }
   }
 
   /**