From 5de56cd6189c0071eb9b28c2d69b35351e42b24b Mon Sep 17 00:00:00 2001 From: olly Date: Thu, 6 Aug 2020 12:09:17 +0100 Subject: [PATCH] Opus: Add utility for handling header and initialization data PiperOrigin-RevId: 325202386 --- .../ext/opus/LibopusAudioRenderer.java | 3 +- .../exoplayer2/ext/opus/OpusDecoder.java | 68 ++++------- .../android/exoplayer2/audio/OpusUtil.java | 112 ++++++++++++++++++ .../exoplayer2/audio/OpusUtilTest.java | 104 ++++++++++++++++ .../exoplayer2/extractor/ogg/OpusReader.java | 30 +---- 5 files changed, 248 insertions(+), 69 deletions(-) create mode 100644 library/common/src/main/java/com/google/android/exoplayer2/audio/OpusUtil.java create mode 100644 library/common/src/test/java/com/google/android/exoplayer2/audio/OpusUtilTest.java diff --git a/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java b/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java index 5a9c131c51..603241486c 100644 --- a/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java +++ b/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java @@ -24,6 +24,7 @@ import com.google.android.exoplayer2.audio.AudioRendererEventListener; import com.google.android.exoplayer2.audio.AudioSink; import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport; import com.google.android.exoplayer2.audio.DecoderAudioRenderer; +import com.google.android.exoplayer2.audio.OpusUtil; import com.google.android.exoplayer2.drm.ExoMediaCrypto; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.TraceUtil; @@ -125,6 +126,6 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer { protected Format getOutputFormat(OpusDecoder decoder) { @C.PcmEncoding int pcmEncoding = decoder.outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT; - return Util.getPcmFormat(pcmEncoding, decoder.channelCount, OpusDecoder.SAMPLE_RATE); + return Util.getPcmFormat(pcmEncoding, decoder.channelCount, OpusUtil.SAMPLE_RATE); } } diff --git a/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java b/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java index 23dd312116..6b96cc5e49 100644 --- a/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java +++ b/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java @@ -17,6 +17,7 @@ package com.google.android.exoplayer2.ext.opus; import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.audio.OpusUtil; import com.google.android.exoplayer2.decoder.CryptoInfo; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.decoder.SimpleDecoder; @@ -26,18 +27,12 @@ import com.google.android.exoplayer2.drm.ExoMediaCrypto; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Util; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.List; /** Opus decoder. */ /* package */ final class OpusDecoder extends SimpleDecoder { - private static final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840; - - /** Opus streams are always decoded at 48000 Hz. */ - public static final int SAMPLE_RATE = 48_000; - private static final int NO_ERROR = 0; private static final int DECODE_ERROR = -1; private static final int DRM_ERROR = -2; @@ -46,9 +41,8 @@ import java.util.List; public final int channelCount; @Nullable private final ExoMediaCrypto exoMediaCrypto; - - private final int headerSkipSamples; - private final int headerSeekPreRollSamples; + private final int preSkipSamples; + private final int seekPreRollSamples; private final long nativeDecoderContext; private int skipSamples; @@ -77,21 +71,31 @@ import java.util.List; throws OpusDecoderException { super(new DecoderInputBuffer[numInputBuffers], new SimpleOutputBuffer[numOutputBuffers]); if (!OpusLibrary.isAvailable()) { - throw new OpusDecoderException("Failed to load decoder native libraries."); + throw new OpusDecoderException("Failed to load decoder native libraries"); } this.exoMediaCrypto = exoMediaCrypto; if (exoMediaCrypto != null && !OpusLibrary.opusIsSecureDecodeSupported()) { - throw new OpusDecoderException("Opus decoder does not support secure decode."); + throw new OpusDecoderException("Opus decoder does not support secure decode"); } + int initializationDataSize = initializationData.size(); + if (initializationDataSize != 1 && initializationDataSize != 3) { + throw new OpusDecoderException("Invalid initialization data size"); + } + if (initializationDataSize == 3 + && (initializationData.get(1).length != 8 || initializationData.get(2).length != 8)) { + throw new OpusDecoderException("Invalid pre-skip or seek pre-roll"); + } + preSkipSamples = OpusUtil.getPreSkipSamples(initializationData); + seekPreRollSamples = OpusUtil.getSeekPreRollSamples(initializationData); + byte[] headerBytes = initializationData.get(0); if (headerBytes.length < 19) { - throw new OpusDecoderException("Header size is too small."); + throw new OpusDecoderException("Invalid header length"); } - channelCount = headerBytes[9] & 0xFF; + channelCount = OpusUtil.getChannelCount(headerBytes); if (channelCount > 8) { throw new OpusDecoderException("Invalid channel count: " + channelCount); } - int preskip = readUnsignedLittleEndian16(headerBytes, 10); int gain = readSignedLittleEndian16(headerBytes, 16); byte[] streamMap = new byte[8]; @@ -100,7 +104,7 @@ import java.util.List; if (headerBytes[18] == 0) { // Channel mapping // If there is no channel mapping, use the defaults. if (channelCount > 2) { // Maximum channel count with default layout. - throw new OpusDecoderException("Invalid Header, missing stream map."); + throw new OpusDecoderException("Invalid header, missing stream map"); } numStreams = 1; numCoupled = (channelCount == 2) ? 1 : 0; @@ -108,29 +112,15 @@ import java.util.List; streamMap[1] = 1; } else { if (headerBytes.length < 21 + channelCount) { - throw new OpusDecoderException("Header size is too small."); + throw new OpusDecoderException("Invalid header length"); } // Read the channel mapping. numStreams = headerBytes[19] & 0xFF; numCoupled = headerBytes[20] & 0xFF; System.arraycopy(headerBytes, 21, streamMap, 0, channelCount); } - if (initializationData.size() == 3) { - if (initializationData.get(1).length != 8 || initializationData.get(2).length != 8) { - throw new OpusDecoderException("Invalid Codec Delay or Seek Preroll"); - } - long codecDelayNs = - ByteBuffer.wrap(initializationData.get(1)).order(ByteOrder.nativeOrder()).getLong(); - long seekPreRollNs = - ByteBuffer.wrap(initializationData.get(2)).order(ByteOrder.nativeOrder()).getLong(); - headerSkipSamples = nsToSamples(codecDelayNs); - headerSeekPreRollSamples = nsToSamples(seekPreRollNs); - } else { - headerSkipSamples = preskip; - headerSeekPreRollSamples = DEFAULT_SEEK_PRE_ROLL_SAMPLES; - } nativeDecoderContext = - opusInit(SAMPLE_RATE, channelCount, numStreams, numCoupled, gain, streamMap); + opusInit(OpusUtil.SAMPLE_RATE, channelCount, numStreams, numCoupled, gain, streamMap); if (nativeDecoderContext == 0) { throw new OpusDecoderException("Failed to initialize decoder"); } @@ -170,7 +160,7 @@ import java.util.List; opusReset(nativeDecoderContext); // When seeking to 0, skip number of samples as specified in opus header. When seeking to // any other time, skip number of samples as specified by seek preroll. - skipSamples = (inputBuffer.timeUs == 0) ? headerSkipSamples : headerSeekPreRollSamples; + skipSamples = (inputBuffer.timeUs == 0) ? preSkipSamples : seekPreRollSamples; } ByteBuffer inputData = Util.castNonNull(inputBuffer.data); CryptoInfo cryptoInfo = inputBuffer.cryptoInfo; @@ -182,7 +172,7 @@ import java.util.List; inputData, inputData.limit(), outputBuffer, - SAMPLE_RATE, + OpusUtil.SAMPLE_RATE, exoMediaCrypto, cryptoInfo.mode, Assertions.checkNotNull(cryptoInfo.key), @@ -231,18 +221,10 @@ import java.util.List; opusClose(nativeDecoderContext); } - private static int nsToSamples(long ns) { - return (int) (ns * SAMPLE_RATE / 1000000000); - } - - private static int readUnsignedLittleEndian16(byte[] input, int offset) { + private static int readSignedLittleEndian16(byte[] input, int offset) { int value = input[offset] & 0xFF; value |= (input[offset + 1] & 0xFF) << 8; - return value; - } - - private static int readSignedLittleEndian16(byte[] input, int offset) { - return (short) readUnsignedLittleEndian16(input, offset); + return (short) value; } private native long opusInit( diff --git a/library/common/src/main/java/com/google/android/exoplayer2/audio/OpusUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/audio/OpusUtil.java new file mode 100644 index 0000000000..3e434bb7e8 --- /dev/null +++ b/library/common/src/main/java/com/google/android/exoplayer2/audio/OpusUtil.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.audio; + +import com.google.android.exoplayer2.C; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; + +/** Utility methods for handling Opus audio streams. */ +public class OpusUtil { + + /** Opus streams are always 48000 Hz. */ + public static final int SAMPLE_RATE = 48_000; + + private static final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840; + private static final int FULL_CODEC_INITIALIZATION_DATA_BUFFER_COUNT = 3; + + private OpusUtil() {} // Prevents instantiation. + + /** + * Parses the channel count from an Opus Identification Header. + * + * @param header An Opus Identification Header, as defined by RFC 7845. + * @return The parsed channel count. + */ + public static int getChannelCount(byte[] header) { + return header[9] & 0xFF; + } + + /** + * Builds codec initialization data from an Opus Identification Header. + * + * @param header An Opus Identification Header, as defined by RFC 7845. + * @return Codec initialization data suitable for an Opus MediaCodec. + */ + public static List buildInitializationData(byte[] header) { + int preSkipSamples = getPreSkipSamples(header); + long preSkipNanos = sampleCountToNanoseconds(preSkipSamples); + long seekPreRollNanos = sampleCountToNanoseconds(DEFAULT_SEEK_PRE_ROLL_SAMPLES); + + List initializationData = new ArrayList<>(FULL_CODEC_INITIALIZATION_DATA_BUFFER_COUNT); + initializationData.add(header); + initializationData.add(buildNativeOrderByteArray(preSkipNanos)); + initializationData.add(buildNativeOrderByteArray(seekPreRollNanos)); + return initializationData; + } + + /** + * Returns the number of pre-skip samples specified by the given Opus codec initialization data. + * + * @param initializationData The codec initialization data. + * @return The number of pre-skip samples. + */ + public static int getPreSkipSamples(List initializationData) { + if (initializationData.size() == FULL_CODEC_INITIALIZATION_DATA_BUFFER_COUNT) { + long codecDelayNs = + ByteBuffer.wrap(initializationData.get(1)).order(ByteOrder.nativeOrder()).getLong(); + return (int) nanosecondsToSampleCount(codecDelayNs); + } + // Fall back to parsing directly from the Opus Identification header. + return getPreSkipSamples(initializationData.get(0)); + } + + /** + * Returns the number of seek per-roll samples specified by the given Opus codec initialization + * data. + * + * @param initializationData The codec initialization data. + * @return The number of seek pre-roll samples. + */ + public static int getSeekPreRollSamples(List initializationData) { + if (initializationData.size() == FULL_CODEC_INITIALIZATION_DATA_BUFFER_COUNT) { + long seekPreRollNs = + ByteBuffer.wrap(initializationData.get(2)).order(ByteOrder.nativeOrder()).getLong(); + return (int) nanosecondsToSampleCount(seekPreRollNs); + } + // Fall back to returning the default seek pre-roll. + return DEFAULT_SEEK_PRE_ROLL_SAMPLES; + } + + private static int getPreSkipSamples(byte[] header) { + return ((header[11] & 0xFF) << 8) | (header[10] & 0xFF); + } + + private static byte[] buildNativeOrderByteArray(long value) { + return ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(value).array(); + } + + private static long sampleCountToNanoseconds(long sampleCount) { + return (sampleCount * C.NANOS_PER_SECOND) / SAMPLE_RATE; + } + + private static long nanosecondsToSampleCount(long nanoseconds) { + return (nanoseconds * SAMPLE_RATE) / C.NANOS_PER_SECOND; + } +} diff --git a/library/common/src/test/java/com/google/android/exoplayer2/audio/OpusUtilTest.java b/library/common/src/test/java/com/google/android/exoplayer2/audio/OpusUtilTest.java new file mode 100644 index 0000000000..4fe18aa4d0 --- /dev/null +++ b/library/common/src/test/java/com/google/android/exoplayer2/audio/OpusUtilTest.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.audio; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link OpusUtil}. */ +@RunWith(AndroidJUnit4.class) +public final class OpusUtilTest { + + private static final byte[] HEADER = + new byte[] {79, 112, 117, 115, 72, 101, 97, 100, 0, 2, 1, 56, 0, 0, -69, -128, 0, 0, 0}; + + private static final int HEADER_PRE_SKIP_SAMPLES = 14337; + private static final byte[] HEADER_PRE_SKIP_BYTES = + buildNativeOrderByteArray(sampleCountToNanoseconds(HEADER_PRE_SKIP_SAMPLES)); + + private static final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840; + private static final byte[] DEFAULT_SEEK_PRE_ROLL_BYTES = + buildNativeOrderByteArray(sampleCountToNanoseconds(DEFAULT_SEEK_PRE_ROLL_SAMPLES)); + + private static final ImmutableList HEADER_ONLY_INITIALIZATION_DATA = + ImmutableList.of(HEADER); + + private static final long CUSTOM_PRE_SKIP_SAMPLES = 28674; + private static final byte[] CUSTOM_PRE_SKIP_BYTES = + buildNativeOrderByteArray(sampleCountToNanoseconds(CUSTOM_PRE_SKIP_SAMPLES)); + + private static final long CUSTOM_SEEK_PRE_ROLL_SAMPLES = 7680; + private static final byte[] CUSTOM_SEEK_PRE_ROLL_BYTES = + buildNativeOrderByteArray(sampleCountToNanoseconds(CUSTOM_SEEK_PRE_ROLL_SAMPLES)); + + private static final ImmutableList FULL_INITIALIZATION_DATA = + ImmutableList.of(HEADER, CUSTOM_PRE_SKIP_BYTES, CUSTOM_SEEK_PRE_ROLL_BYTES); + + @Test + public void buildInitializationData() { + List initializationData = OpusUtil.buildInitializationData(HEADER); + assertThat(initializationData).hasSize(3); + assertThat(initializationData.get(0)).isEqualTo(HEADER); + assertThat(initializationData.get(1)).isEqualTo(HEADER_PRE_SKIP_BYTES); + assertThat(initializationData.get(2)).isEqualTo(DEFAULT_SEEK_PRE_ROLL_BYTES); + } + + @Test + public void getChannelCount() { + int channelCount = OpusUtil.getChannelCount(HEADER); + assertThat(channelCount).isEqualTo(2); + } + + @Test + public void getPreSkipSamples_fullInitializationData_returnsOverrideValue() { + int preSkipSamples = OpusUtil.getPreSkipSamples(FULL_INITIALIZATION_DATA); + assertThat(preSkipSamples).isEqualTo(CUSTOM_PRE_SKIP_SAMPLES); + } + + @Test + public void getPreSkipSamples_headerOnlyInitializationData_returnsHeaderValue() { + int preSkipSamples = OpusUtil.getPreSkipSamples(HEADER_ONLY_INITIALIZATION_DATA); + assertThat(preSkipSamples).isEqualTo(HEADER_PRE_SKIP_SAMPLES); + } + + @Test + public void getSeekPreRollSamples_fullInitializationData_returnsInitializationDataValue() { + int seekPreRollSamples = OpusUtil.getSeekPreRollSamples(FULL_INITIALIZATION_DATA); + assertThat(seekPreRollSamples).isEqualTo(CUSTOM_SEEK_PRE_ROLL_SAMPLES); + } + + @Test + public void getSeekPreRollSamples_headerOnlyInitializationData_returnsDefaultValue() { + int seekPreRollSamples = OpusUtil.getSeekPreRollSamples(HEADER_ONLY_INITIALIZATION_DATA); + assertThat(seekPreRollSamples).isEqualTo(DEFAULT_SEEK_PRE_ROLL_SAMPLES); + } + + private static long sampleCountToNanoseconds(long sampleCount) { + return (sampleCount * C.NANOS_PER_SECOND) / OpusUtil.SAMPLE_RATE; + } + + private static byte[] buildNativeOrderByteArray(long value) { + return ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(value).array(); + } +} diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ogg/OpusReader.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ogg/OpusReader.java index de03843b30..8144af7b66 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ogg/OpusReader.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/ogg/OpusReader.java @@ -15,13 +15,10 @@ */ package com.google.android.exoplayer2.extractor.ogg; -import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.audio.OpusUtil; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.ParsableByteArray; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -30,11 +27,6 @@ import java.util.List; */ /* package */ final class OpusReader extends StreamReader { - private static final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840; - - /** Opus streams are always decoded at 48000 Hz. */ - private static final int SAMPLE_RATE = 48_000; - private static final int OPUS_CODE = 0x4f707573; private static final byte[] OPUS_SIGNATURE = {'O', 'p', 'u', 's', 'H', 'e', 'a', 'd'}; @@ -65,20 +57,14 @@ import java.util.List; @Override protected boolean readHeaders(ParsableByteArray packet, long position, SetupData setupData) { if (!headerRead) { - byte[] metadata = Arrays.copyOf(packet.getData(), packet.limit()); - int channelCount = metadata[9] & 0xFF; - int preskip = ((metadata[11] & 0xFF) << 8) | (metadata[10] & 0xFF); - - List initializationData = new ArrayList<>(3); - initializationData.add(metadata); - putNativeOrderLong(initializationData, preskip); - putNativeOrderLong(initializationData, DEFAULT_SEEK_PRE_ROLL_SAMPLES); - + byte[] headerBytes = Arrays.copyOf(packet.getData(), packet.limit()); + int channelCount = OpusUtil.getChannelCount(headerBytes); + List initializationData = OpusUtil.buildInitializationData(headerBytes); setupData.format = new Format.Builder() .setSampleMimeType(MimeTypes.AUDIO_OPUS) .setChannelCount(channelCount) - .setSampleRate(SAMPLE_RATE) + .setSampleRate(OpusUtil.SAMPLE_RATE) .setInitializationData(initializationData) .build(); headerRead = true; @@ -90,12 +76,6 @@ import java.util.List; return true; } - private void putNativeOrderLong(List initializationData, int samples) { - long ns = (samples * C.NANOS_PER_SECOND) / SAMPLE_RATE; - byte[] array = ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).putLong(ns).array(); - initializationData.add(array); - } - /** * Returns the duration of the given audio packet. *