From c3b92f84562aa55dc79d764f6fa4cd2e827f39e7 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Mon, 20 Nov 2017 04:32:56 -0800 Subject: [PATCH] Add support for Dolby Atmos Issue: #2465 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=176341309 --- RELEASENOTES.md | 2 + .../android/exoplayer2/audio/Ac3Util.java | 170 +++++++++++++++++- .../exoplayer2/extractor/ts/Ac3Reader.java | 2 +- .../exoplayer2/mediacodec/MediaCodecUtil.java | 60 +++++-- .../android/exoplayer2/util/MimeTypes.java | 4 + .../dash/manifest/DashManifestParser.java | 22 ++- 6 files changed, 237 insertions(+), 23 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6683ee3f55..f772bc9f19 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -6,6 +6,8 @@ DashMediaSource, SingleSampleMediaSource. * Support 32-bit PCM float output from `DefaultAudioSink`, and add an option to use this with `FfmpegAudioRenderer`. +* Support extraction and decoding of Dolby Atmos + ([#2465](https://github.com/google/ExoPlayer/issues/2465)). ### 2.6.0 ### diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/Ac3Util.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/Ac3Util.java index e1a70e2579..e9ffab7ace 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/Ac3Util.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/Ac3Util.java @@ -15,6 +15,10 @@ */ package com.google.android.exoplayer2.audio; +import static com.google.android.exoplayer2.audio.Ac3Util.Ac3SyncFrameInfo.STREAM_TYPE_TYPE0; +import static com.google.android.exoplayer2.audio.Ac3Util.Ac3SyncFrameInfo.STREAM_TYPE_TYPE1; +import static com.google.android.exoplayer2.audio.Ac3Util.Ac3SyncFrameInfo.STREAM_TYPE_UNDEFINED; + import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.drm.DrmInitData; @@ -181,7 +185,14 @@ public final class Ac3Util { channelCount += 2; } } - return Format.createAudioSampleFormat(trackId, MimeTypes.AUDIO_E_AC3, null, Format.NO_VALUE, + String mimeType = MimeTypes.AUDIO_E_AC3; + if (data.bytesLeft() > 0) { + nextByte = data.readUnsignedByte(); + if ((nextByte & 0x01) != 0) { // flag_ec3_extension_type_a + mimeType = MimeTypes.AUDIO_ATMOS; + } + } + return Format.createAudioSampleFormat(trackId, mimeType, null, Format.NO_VALUE, Format.NO_VALUE, channelCount, sampleRate, null, drmInitData, 0, language); } @@ -198,29 +209,176 @@ public final class Ac3Util { boolean isEac3 = data.readBits(5) == 16; data.setPosition(initialPosition); String mimeType; - int streamType = Ac3SyncFrameInfo.STREAM_TYPE_UNDEFINED; + int streamType = STREAM_TYPE_UNDEFINED; int sampleRate; int acmod; int frameSize; int sampleCount; + boolean lfeon; + int channelCount; if (isEac3) { - mimeType = MimeTypes.AUDIO_E_AC3; + // Syntax from ETSI TS 102 366 V1.2.1 subsections E.1.2.1 and E.1.2.2. data.skipBits(16); // syncword streamType = data.readBits(2); data.skipBits(3); // substreamid frameSize = (data.readBits(11) + 1) * 2; int fscod = data.readBits(2); int audioBlocks; + int numblkscod; if (fscod == 3) { + numblkscod = 3; sampleRate = SAMPLE_RATE_BY_FSCOD2[data.readBits(2)]; audioBlocks = 6; } else { - int numblkscod = data.readBits(2); + numblkscod = data.readBits(2); audioBlocks = BLOCKS_PER_SYNCFRAME_BY_NUMBLKSCOD[numblkscod]; sampleRate = SAMPLE_RATE_BY_FSCOD[fscod]; } sampleCount = AUDIO_SAMPLES_PER_AUDIO_BLOCK * audioBlocks; acmod = data.readBits(3); + lfeon = data.readBit(); + channelCount = CHANNEL_COUNT_BY_ACMOD[acmod] + (lfeon ? 1 : 0); + data.skipBits(5 + 5); // bsid, dialnorm + if (data.readBit()) { // compre + data.skipBits(8); // compr + } + if (acmod == 0) { + data.skipBits(5); // dialnorm2 + if (data.readBit()) { // compr2e + data.skipBits(8); // compr2 + } + } + if (streamType == STREAM_TYPE_TYPE1 && data.readBit()) { // chanmape + data.skipBits(16); // chanmap + } + if (data.readBit()) { // mixmdate + if (acmod > 2) { + data.skipBits(2); // dmixmod + } + if ((acmod & 0x01) != 0 && acmod > 2) { + data.skipBits(3 + 3); // ltrtcmixlev, lorocmixlev + } + if ((acmod & 0x04) != 0) { + data.skipBits(6); // ltrtsurmixlev, lorosurmixlev + } + if (lfeon && data.readBit()) { // lfemixlevcode + data.skipBits(5); // lfemixlevcod + } + if (streamType == STREAM_TYPE_TYPE0) { + if (data.readBit()) { // pgmscle + data.skipBits(6); //pgmscl + } + if (acmod == 0 && data.readBit()) { // pgmscl2e + data.skipBits(6); // pgmscl2 + } + if (data.readBit()) { // extpgmscle + data.skipBits(6); // extpgmscl + } + int mixdef = data.readBits(2); + if (mixdef == 1) { + data.skipBits(1 + 1 + 3); // premixcmpsel, drcsrc, premixcmpscl + } else if (mixdef == 2) { + data.skipBits(12); // mixdata + } else if (mixdef == 3) { + int mixdeflen = data.readBits(5); + if (data.readBit()) { // mixdata2e + data.skipBits(1 + 1 + 3); // premixcmpsel, drcsrc, premixcmpscl + if (data.readBit()) { // extpgmlscle + data.skipBits(4); // extpgmlscl + } + if (data.readBit()) { // extpgmcscle + data.skipBits(4); // extpgmcscl + } + if (data.readBit()) { // extpgmrscle + data.skipBits(4); // extpgmrscl + } + if (data.readBit()) { // extpgmlsscle + data.skipBits(4); // extpgmlsscl + } + if (data.readBit()) { // extpgmrsscle + data.skipBits(4); // extpgmrsscl + } + if (data.readBit()) { // extpgmlfescle + data.skipBits(4); // extpgmlfescl + } + if (data.readBit()) { // dmixscle + data.skipBits(4); // dmixscl + } + if (data.readBit()) { // addche + if (data.readBit()) { // extpgmaux1scle + data.skipBits(4); // extpgmaux1scl + } + if (data.readBit()) { // extpgmaux2scle + data.skipBits(4); // extpgmaux2scl + } + } + } + if (data.readBit()) { // mixdata3e + data.skipBits(5); // spchdat + if (data.readBit()) { // addspchdate + data.skipBits(5 + 2); // spchdat1, spchan1att + if (data.readBit()) { // addspdat1e + data.skipBits(5 + 3); // spchdat2, spchan2att + } + } + } + data.skipBits(8 * (mixdeflen + 2)); // mixdata + data.byteAlign(); // mixdatafill + } + if (acmod < 2) { + if (data.readBit()) { // paninfoe + data.skipBits(8 + 6); // panmean, paninfo + } + if (acmod == 0) { + if (data.readBit()) { // paninfo2e + data.skipBits(8 + 6); // panmean2, paninfo2 + } + } + } + if (data.readBit()) { // frmmixcfginfoe + if (numblkscod == 0) { + data.skipBits(5); // blkmixcfginfo[0] + } else { + for (int blk = 0; blk < audioBlocks; blk++) { + if (data.readBit()) { // blkmixcfginfoe + data.skipBits(5); // blkmixcfginfo[blk] + } + } + } + } + } + } + if (data.readBit()) { // infomdate + data.skipBits(3 + 1 + 1); // bsmod, copyrightb, origbs + if (acmod == 2) { + data.skipBits(2 + 2); // dsurmod, dheadphonmod + } + if (acmod >= 6) { + data.skipBits(2); // dsurexmod + } + if (data.readBit()) { // audioprodie + data.skipBits(5 + 2 + 1); // mixlevel, roomtyp, adconvtyp + } + if (acmod == 0 && data.readBit()) { // audioprodi2e + data.skipBits(5 + 2 + 1); // mixlevel2, roomtyp2, adconvtyp2 + } + if (fscod < 3) { + data.skipBit(); // sourcefscod + } + } + if (streamType == 0 && numblkscod != 3) { + data.skipBit(); // convsync + } + if (streamType == 2 && (numblkscod == 3 || data.readBit())) { // blkid + data.skipBits(6); // frmsizecod + } + mimeType = MimeTypes.AUDIO_E_AC3; + if (data.readBit()) { // addbsie + int addbsil = data.readBits(6); + if (addbsil == 1 && data.readBits(8) == 1) { // addbsi + mimeType = MimeTypes.AUDIO_ATMOS; + } + } } else /* is AC-3 */ { mimeType = MimeTypes.AUDIO_AC3; data.skipBits(16 + 16); // syncword, crc1 @@ -240,9 +398,9 @@ public final class Ac3Util { } sampleRate = SAMPLE_RATE_BY_FSCOD[fscod]; sampleCount = AC3_SYNCFRAME_AUDIO_SAMPLE_COUNT; + lfeon = data.readBit(); + channelCount = CHANNEL_COUNT_BY_ACMOD[acmod] + (lfeon ? 1 : 0); } - boolean lfeon = data.readBit(); - int channelCount = CHANNEL_COUNT_BY_ACMOD[acmod] + (lfeon ? 1 : 0); return new Ac3SyncFrameInfo(mimeType, streamType, channelCount, sampleRate, frameSize, sampleCount); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Ac3Reader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Ac3Reader.java index 6a1c566faf..8383bfb8d2 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Ac3Reader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Ac3Reader.java @@ -39,7 +39,7 @@ public final class Ac3Reader implements ElementaryStreamReader { private static final int STATE_READING_HEADER = 1; private static final int STATE_READING_SAMPLE = 2; - private static final int HEADER_SIZE = 8; + private static final int HEADER_SIZE = 128; private final ParsableBitArray headerScratchBits; private final ParsableByteArray headerScratchBytes; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java index f75ce5a9e5..7ae8eb3cd4 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java @@ -20,6 +20,7 @@ import android.annotation.TargetApi; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecInfo.CodecProfileLevel; import android.media.MediaCodecList; +import android.support.annotation.Nullable; import android.text.TextUtils; import android.util.Log; import android.util.Pair; @@ -120,7 +121,7 @@ public final class MediaCodecUtil { * exists. * @throws DecoderQueryException If there was an error querying the available decoders. */ - public static MediaCodecInfo getDecoderInfo(String mimeType, boolean secure) + public static @Nullable MediaCodecInfo getDecoderInfo(String mimeType, boolean secure) throws DecoderQueryException { List decoderInfos = getDecoderInfos(mimeType, secure); return decoderInfos.isEmpty() ? null : decoderInfos.get(0); @@ -140,27 +141,34 @@ public final class MediaCodecUtil { public static synchronized List getDecoderInfos(String mimeType, boolean secure) throws DecoderQueryException { CodecKey key = new CodecKey(mimeType, secure); - List decoderInfos = decoderInfosCache.get(key); - if (decoderInfos != null) { - return decoderInfos; + List cachedDecoderInfos = decoderInfosCache.get(key); + if (cachedDecoderInfos != null) { + return cachedDecoderInfos; } MediaCodecListCompat mediaCodecList = Util.SDK_INT >= 21 ? new MediaCodecListCompatV21(secure) : new MediaCodecListCompatV16(); - decoderInfos = getDecoderInfosInternal(key, mediaCodecList); + ArrayList decoderInfos = getDecoderInfosInternal(key, mediaCodecList, mimeType); if (secure && decoderInfos.isEmpty() && 21 <= Util.SDK_INT && Util.SDK_INT <= 23) { // Some devices don't list secure decoders on API level 21 [Internal: b/18678462]. Try the // legacy path. We also try this path on API levels 22 and 23 as a defensive measure. mediaCodecList = new MediaCodecListCompatV16(); - decoderInfos = getDecoderInfosInternal(key, mediaCodecList); + decoderInfos = getDecoderInfosInternal(key, mediaCodecList, mimeType); if (!decoderInfos.isEmpty()) { Log.w(TAG, "MediaCodecList API didn't list secure decoder for: " + mimeType + ". Assuming: " + decoderInfos.get(0).name); } } + if (MimeTypes.AUDIO_ATMOS.equals(mimeType)) { + // E-AC3 decoders can decode Atmos streams, but in 2-D rather than 3-D. + CodecKey eac3Key = new CodecKey(MimeTypes.AUDIO_E_AC3, key.secure); + ArrayList eac3DecoderInfos = + getDecoderInfosInternal(eac3Key, mediaCodecList, mimeType); + decoderInfos.addAll(eac3DecoderInfos); + } applyWorkarounds(decoderInfos); - decoderInfos = Collections.unmodifiableList(decoderInfos); - decoderInfosCache.put(key, decoderInfos); - return decoderInfos; + List unmodifiableDecoderInfos = Collections.unmodifiableList(decoderInfos); + decoderInfosCache.put(key, unmodifiableDecoderInfos); + return unmodifiableDecoderInfos; } /** @@ -212,10 +220,21 @@ public final class MediaCodecUtil { // Internal methods. - private static List getDecoderInfosInternal( - CodecKey key, MediaCodecListCompat mediaCodecList) throws DecoderQueryException { + /** + * Returns {@link MediaCodecInfo}s for the given codec {@code key} in the order given by + * {@code mediaCodecList}. + * + * @param key The codec key. + * @param mediaCodecList The codec list. + * @param requestedMimeType The originally requested MIME type, which may differ from the codec + * key MIME type if the codec key is being considered as a fallback. + * @return The codec information for usable codecs matching the specified key. + * @throws DecoderQueryException If there was an error querying the available decoders. + */ + private static ArrayList getDecoderInfosInternal(CodecKey key, + MediaCodecListCompat mediaCodecList, String requestedMimeType) throws DecoderQueryException { try { - List decoderInfos = new ArrayList<>(); + ArrayList decoderInfos = new ArrayList<>(); String mimeType = key.mimeType; int numberOfCodecs = mediaCodecList.getCodecCount(); boolean secureDecodersExplicit = mediaCodecList.secureDecodersExplicit(); @@ -223,7 +242,7 @@ public final class MediaCodecUtil { for (int i = 0; i < numberOfCodecs; i++) { android.media.MediaCodecInfo codecInfo = mediaCodecList.getCodecInfoAt(i); String codecName = codecInfo.getName(); - if (isCodecUsableDecoder(codecInfo, codecName, secureDecodersExplicit)) { + if (isCodecUsableDecoder(codecInfo, codecName, secureDecodersExplicit, requestedMimeType)) { for (String supportedType : codecInfo.getSupportedTypes()) { if (supportedType.equalsIgnoreCase(mimeType)) { try { @@ -265,9 +284,16 @@ public final class MediaCodecUtil { /** * Returns whether the specified codec is usable for decoding on the current device. + * + * @param info The codec information. + * @param name The name of the codec + * @param secureDecodersExplicit Whether secure decoders were explicitly listed, if present. + * @param requestedMimeType The originally requested MIME type, which may differ from the codec + * key MIME type if the codec key is being considered as a fallback. + * @return Whether the specified codec is usable for decoding on the current device. */ private static boolean isCodecUsableDecoder(android.media.MediaCodecInfo info, String name, - boolean secureDecodersExplicit) { + boolean secureDecodersExplicit, String requestedMimeType) { if (info.isEncoder() || (!secureDecodersExplicit && name.endsWith(".secure"))) { return false; } @@ -356,6 +382,12 @@ public final class MediaCodecUtil { return false; } + // MTK E-AC3 decoder doesn't support decoding Atmos streams in 2-D. See [Internal: b/69400041]. + if (MimeTypes.AUDIO_ATMOS.equals(requestedMimeType) + && "OMX.MTK.AUDIO.DECODER.DSPAC3".equals(name)) { + return false; + } + return true; } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java b/library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java index c29a4c3717..a68e0142d6 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java @@ -51,6 +51,7 @@ public final class MimeTypes { public static final String AUDIO_MLAW = BASE_TYPE_AUDIO + "/g711-mlaw"; public static final String AUDIO_AC3 = BASE_TYPE_AUDIO + "/ac3"; public static final String AUDIO_E_AC3 = BASE_TYPE_AUDIO + "/eac3"; + public static final String AUDIO_ATMOS = BASE_TYPE_AUDIO + "/eac3-joc"; public static final String AUDIO_TRUEHD = BASE_TYPE_AUDIO + "/true-hd"; public static final String AUDIO_DTS = BASE_TYPE_AUDIO + "/vnd.dts"; public static final String AUDIO_DTS_HD = BASE_TYPE_AUDIO + "/vnd.dts.hd"; @@ -195,6 +196,8 @@ public final class MimeTypes { return MimeTypes.AUDIO_AC3; } else if (codec.startsWith("ec-3") || codec.startsWith("dec3")) { return MimeTypes.AUDIO_E_AC3; + } else if (codec.startsWith("ec+3")) { + return MimeTypes.AUDIO_ATMOS; } else if (codec.startsWith("dtsc") || codec.startsWith("dtse")) { return MimeTypes.AUDIO_DTS; } else if (codec.startsWith("dtsh") || codec.startsWith("dtsl")) { @@ -252,6 +255,7 @@ public final class MimeTypes { case MimeTypes.AUDIO_AC3: return C.ENCODING_AC3; case MimeTypes.AUDIO_E_AC3: + case MimeTypes.AUDIO_ATMOS: return C.ENCODING_E_AC3; case MimeTypes.AUDIO_DTS: return C.ENCODING_DTS; diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java index 137e29c5ab..aa4c6b1e30 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java @@ -452,6 +452,7 @@ public class DashManifestParser extends DefaultHandler String drmSchemeType = null; ArrayList drmSchemeDatas = new ArrayList<>(); ArrayList inbandEventStreams = new ArrayList<>(); + ArrayList supplementalProperties = new ArrayList<>(); boolean seenFirstBaseUrl = false; do { @@ -479,12 +480,14 @@ public class DashManifestParser extends DefaultHandler } } else if (XmlPullParserUtil.isStartTag(xpp, "InbandEventStream")) { inbandEventStreams.add(parseDescriptor(xpp, "InbandEventStream")); + } else if (XmlPullParserUtil.isStartTag(xpp, "SupplementalProperty")) { + supplementalProperties.add(parseDescriptor(xpp, "SupplementalProperty")); } } while (!XmlPullParserUtil.isEndTag(xpp, "Representation")); Format format = buildFormat(id, mimeType, width, height, frameRate, audioChannels, audioSamplingRate, bandwidth, adaptationSetLanguage, adaptationSetSelectionFlags, - adaptationSetAccessibilityDescriptors, codecs); + adaptationSetAccessibilityDescriptors, codecs, supplementalProperties); segmentBase = segmentBase != null ? segmentBase : new SingleSegmentBase(); return new RepresentationInfo(format, baseUrl, segmentBase, drmSchemeType, drmSchemeDatas, @@ -494,9 +497,12 @@ public class DashManifestParser extends DefaultHandler protected Format buildFormat(String id, String containerMimeType, int width, int height, float frameRate, int audioChannels, int audioSamplingRate, int bitrate, String language, @C.SelectionFlags int selectionFlags, List accessibilityDescriptors, - String codecs) { + String codecs, List supplementalProperties) { String sampleMimeType = getSampleMimeType(containerMimeType, codecs); if (sampleMimeType != null) { + if (MimeTypes.AUDIO_E_AC3.equals(sampleMimeType)) { + sampleMimeType = parseEac3SupplementalProperties(supplementalProperties); + } if (MimeTypes.isVideo(sampleMimeType)) { return Format.createVideoContainerFormat(id, containerMimeType, sampleMimeType, codecs, bitrate, width, height, frameRate, null, selectionFlags); @@ -900,6 +906,18 @@ public class DashManifestParser extends DefaultHandler return Format.NO_VALUE; } + protected static String parseEac3SupplementalProperties(List supplementalProperties) { + for (int i = 0; i < supplementalProperties.size(); i++) { + Descriptor descriptor = supplementalProperties.get(i); + String schemeIdUri = descriptor.schemeIdUri; + if ("tag:dolby.com,2014:dash:DolbyDigitalPlusExtensionType:2014".equals(schemeIdUri) + && "ec+3".equals(descriptor.value)) { + return MimeTypes.AUDIO_ATMOS; + } + } + return MimeTypes.AUDIO_E_AC3; + } + protected static float parseFrameRate(XmlPullParser xpp, float defaultValue) { float frameRate = defaultValue; String frameRateAttribute = xpp.getAttributeValue(null, "frameRate");