diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 96733f848c..581eb4fd10 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -20,7 +20,7 @@ * Track Selection: * Extractors: * Audio: -* Audio Offload: + * Add support for Opus gapless metadata during offload playback. * Video: * Text: * Metadata: diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java index f0941fcf56..e8597bb228 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java @@ -523,11 +523,18 @@ public interface AudioSink { /** * Sets audio offload mode, if possible. Enabling offload is only possible if the sink is based on * a platform {@link AudioTrack}, and requires platform API version 29 onwards. - * - * @throws IllegalStateException Thrown if enabling offload on platform API version < 29. */ + @RequiresApi(29) default void setOffloadMode(@OffloadMode int offloadMode) {} + /** + * Sets offload delay padding on the {@link AudioTrack}, if possible. Setting the offload delay + * padding is only possible if the sink is based on a platform {@link AudioTrack} in offload mode. + * Also requires platform API version 29 onwards. + */ + @RequiresApi(29) + default void setOffloadDelayPadding(int delayInFrames, int paddingInFrames) {} + /** * Sets the playback volume. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java index 0e56471b09..6d1cd4c99a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java @@ -1356,12 +1356,24 @@ public final class DefaultAudioSink implements AudioSink { } } + @RequiresApi(29) @Override public void setOffloadMode(@OffloadMode int offloadMode) { Assertions.checkState(Util.SDK_INT >= 29); this.offloadMode = offloadMode; } + @RequiresApi(29) + @Override + public void setOffloadDelayPadding(int delayInFrames, int paddingInFrames) { + if (audioTrack != null + && isOffloadedPlayback(audioTrack) + && configuration != null + && configuration.enableOffloadGapless) { + audioTrack.setOffloadDelayPadding(delayInFrames, paddingInFrames); + } + } + @Override public void setVolume(float volume) { if (this.volume != volume) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java index 1935047fe4..e8472b47b5 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java @@ -173,10 +173,17 @@ public class ForwardingAudioSink implements AudioSink { } @Override + @RequiresApi(29) public void setOffloadMode(@OffloadMode int offloadMode) { sink.setOffloadMode(offloadMode); } + @Override + @RequiresApi(29) + public void setOffloadDelayPadding(int delayInFrames, int paddingInFrames) { + sink.setOffloadDelayPadding(delayInFrames, paddingInFrames); + } + @Override public void setVolume(float volume) { sink.setVolume(volume); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java index c70734b9d4..51c651991f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java @@ -45,6 +45,7 @@ import androidx.media3.common.util.Log; import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; +import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.DecoderReuseEvaluation; import androidx.media3.exoplayer.DecoderReuseEvaluation.DecoderDiscardReasons; import androidx.media3.exoplayer.ExoPlaybackException; @@ -64,7 +65,9 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.List; +import java.util.Objects; /** * Decodes and renders audio using {@link MediaCodec} and an {@link AudioSink}. @@ -820,6 +823,22 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } } + @Override + protected void handleInputBufferSupplementalData(DecoderInputBuffer buffer) { + if (Util.SDK_INT >= 29 + && buffer.format != null + && Objects.equals(buffer.format.sampleMimeType, MimeTypes.AUDIO_OPUS) + && isBypassEnabled()) { + ByteBuffer data = checkNotNull(buffer.supplementalData); + int preSkip = checkNotNull(buffer.format).encoderDelay; + if (data.remaining() == 8) { + int discardSamples = + (int) ((data.order(ByteOrder.LITTLE_ENDIAN).getLong() * 48_000L) / C.NANOS_PER_SECOND); + audioSink.setOffloadDelayPadding(preSkip, discardSamples); + } + } + } + /** * Returns a maximum input size suitable for configuring a codec for {@code format} in a way that * will allow possible adaptation to other compatible formats in {@code streamFormats}. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index dff880860a..aed5233206 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -77,6 +77,7 @@ import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.SampleStream; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; import androidx.media3.exoplayer.source.SampleStream.ReadFlags; +import androidx.media3.extractor.OpusUtil; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -85,6 +86,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayDeque; import java.util.List; +import java.util.Objects; /** An abstract renderer that uses {@link MediaCodec} to decode samples for rendering. */ // @@ -2334,16 +2336,28 @@ public abstract class MediaCodecRenderer extends BaseRenderer { if (waitingForFirstSampleInFormat) { // This is the first buffer in a new format, the output format must be updated. outputFormat = checkNotNull(inputFormat); + if (Objects.equals(outputFormat.sampleMimeType, MimeTypes.AUDIO_OPUS) + && !outputFormat.initializationData.isEmpty()) { + // Format mimetype is Opus so format should be updated with preSkip data. + // TODO(b/298634018): Adjust encoderDelay value based on starting position. + int numberPreSkipSamples = + OpusUtil.getPreSkipSamples(outputFormat.initializationData.get(0)); + outputFormat = outputFormat.buildUpon().setEncoderDelay(numberPreSkipSamples).build(); + } onOutputFormatChanged(outputFormat, /* mediaFormat= */ null); waitingForFirstSampleInFormat = false; } // Try to append the buffer to the batch buffer. bypassSampleBuffer.flip(); - if (inputFormat != null - && inputFormat.sampleMimeType != null - && inputFormat.sampleMimeType.equals(MimeTypes.AUDIO_OPUS)) { - oggOpusAudioPacketizer.packetize(bypassSampleBuffer, inputFormat.initializationData); + if (outputFormat != null + && Objects.equals(outputFormat.sampleMimeType, MimeTypes.AUDIO_OPUS)) { + if (bypassSampleBuffer.hasSupplementalData()) { + // Set format on sample buffer so that it contains the mimetype and encodingDelay. + bypassSampleBuffer.format = outputFormat; + handleInputBufferSupplementalData(bypassSampleBuffer); + } + oggOpusAudioPacketizer.packetize(bypassSampleBuffer, outputFormat.initializationData); } if (!haveBypassBatchBufferAndNewSampleSameDecodeOnlyState() || !bypassBatchBuffer.append(bypassSampleBuffer)) { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java index 728f1dd424..b70bacf266 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java @@ -18,6 +18,7 @@ package androidx.media3.exoplayer.audio; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.format; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.sample; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -50,6 +51,7 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecInfo; import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; import androidx.media3.exoplayer.upstream.DefaultAllocator; import androidx.media3.test.utils.FakeSampleStream; +import androidx.media3.test.utils.TestUtil; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -63,12 +65,21 @@ import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; +import org.robolectric.annotation.Config; /** Unit tests for {@link MediaCodecAudioRenderer} */ @RunWith(AndroidJUnit4.class) public class MediaCodecAudioRendererTest { @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + /** Payload for Ogg ID Header Page in accordance with RFC 7845. */ + private static final byte[] OGG_OPUS_ID_HEADER_PAYLOAD = + TestUtil.createByteArray( + 0x4F, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64, 0x1, 0x2, 0x38, 0x1, 0x80, 0xBB, 0x0, 0x0, + 0x0, 0x0, 0x0); + + private static final byte[] SAMPLE_ONE_BYTE_WITH_SUPPLEMENTAL_DATA = + TestUtil.createByteArray(0x0, 0x0, 0x0, 0x1, 0x64, 0xA, 0x2C, 0x2A, 0x0, 0x0, 0x0, 0x0, 0x0); private static final Format AUDIO_AAC = new Format.Builder() .setSampleMimeType(MimeTypes.AUDIO_AAC) @@ -78,11 +89,23 @@ public class MediaCodecAudioRendererTest { .setEncoderDelay(100) .setEncoderPadding(150) .build(); + private static final Format AUDIO_OPUS = + new Format.Builder() + .setSampleMimeType(MimeTypes.AUDIO_OPUS) + .setChannelCount(2) + .setSampleRate(48000) + .setInitializationData(ImmutableList.of(OGG_OPUS_ID_HEADER_PAYLOAD)) + .build(); private static final AudioOffloadSupport AUDIO_OFFLOAD_SUPPORTED_GAPLESS_NOT_SUPPORTED = new AudioOffloadSupport.Builder() .setIsFormatSupported(true) .setIsGaplessSupported(false) .build(); + private static final AudioOffloadSupport AUDIO_OFFLOAD_SUPPORTED_GAPLESS_SUPPORTED = + new AudioOffloadSupport.Builder() + .setIsFormatSupported(true) + .setIsGaplessSupported(true) + .build(); private static final RendererConfiguration RENDERER_CONFIGURATION_OFFLOAD_ENABLED_GAPLESS_REQUIRED = new RendererConfiguration( @@ -639,6 +662,46 @@ public class MediaCodecAudioRendererTest { verify(audioSink).setOffloadMode(AudioSink.OFFLOAD_MODE_DISABLED); } + @Test + @Config(minSdk = 30) + public void render_offloadedOpusWithPadding_callsSetOffloadDelayPadding() throws Exception { + when(audioSink.getFormatOffloadSupport(any())) + .thenReturn(AUDIO_OFFLOAD_SUPPORTED_GAPLESS_SUPPORTED); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ AUDIO_OPUS, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100, C.BUFFER_FLAG_KEY_FRAME), + sample( + /* timeUs= */ 150, + C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA, + SAMPLE_ONE_BYTE_WITH_SUPPLEMENTAL_DATA), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + + mediaCodecAudioRenderer.enable( + RENDERER_CONFIGURATION_OFFLOAD_ENABLED_GAPLESS_REQUIRED, + new Format[] {AUDIO_OPUS}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ false, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + mediaCodecAudioRenderer.setCurrentStreamFinal(); + while (!mediaCodecAudioRenderer.isEnded()) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + + verify(audioSink).setOffloadDelayPadding(/* delayInFrames= */ 312, /* paddingInFrames= */ 132); + } + private static Format getAudioSinkFormat(Format inputFormat) { return new Format.Builder() .setSampleMimeType(MimeTypes.AUDIO_RAW) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/OpusUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/OpusUtil.java index ca34bb5b80..086b628eff 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/OpusUtil.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/OpusUtil.java @@ -139,6 +139,16 @@ public class OpusUtil { return getPacketDurationUs(buffer[0], buffer.length > 1 ? buffer[1] : 0); } + /** + * Returns the number of pre-skip samples specified by the given Opus codec initialization data. + * + * @param header The Opus Identification header. + * @return The number of pre-skip samples. + */ + public static int getPreSkipSamples(byte[] header) { + return ((header[11] & 0xFF) << 8) | (header[10] & 0xFF); + } + private static long getPacketDurationUs(byte packetByte0, byte packetByte1) { // See RFC6716, Sections 3.1 and 3.2. int toc = packetByte0 & 0xFF; @@ -171,10 +181,6 @@ public class OpusUtil { return (long) frames * frameDurationUs; } - private static int getPreSkipSamples(byte[] header) { - return ((header[11] & 0xFF) << 8) | (header[10] & 0xFF); - } - private static byte[] buildNativeOrderByteArray(long value) { return ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(value).array(); }