From 4efdd14c659c151d5fddadb0280c26f1e69c2800 Mon Sep 17 00:00:00 2001 From: olly Date: Tue, 24 Jan 2017 04:24:29 -0800 Subject: [PATCH] Allow FMP4 extractor to output CEA-608 Issue: #2362 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=145401668 --- .../extractor/mp4/FragmentedMp4Extractor.java | 60 +++++++--- .../exoplayer2/extractor/ts/SeiReader.java | 38 +------ .../exoplayer2/text/cea/Cea608Decoder.java | 31 ----- .../android/exoplayer2/text/cea/CeaUtil.java | 106 ++++++++++++++++++ 4 files changed, 155 insertions(+), 80 deletions(-) create mode 100644 library/src/main/java/com/google/android/exoplayer2/text/cea/CeaUtil.java diff --git a/library/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java b/library/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java index 45cb788a2b..7d687cc709 100644 --- a/library/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java +++ b/library/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java @@ -34,6 +34,7 @@ import com.google.android.exoplayer2.extractor.SeekMap; import com.google.android.exoplayer2.extractor.TrackOutput; import com.google.android.exoplayer2.extractor.mp4.Atom.ContainerAtom; import com.google.android.exoplayer2.extractor.mp4.Atom.LeafAtom; +import com.google.android.exoplayer2.text.cea.CeaUtil; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.NalUnitUtil; @@ -67,15 +68,13 @@ public final class FragmentedMp4Extractor implements Extractor { }; - private static final String TAG = "FragmentedMp4Extractor"; - private static final int SAMPLE_GROUP_TYPE_seig = Util.getIntegerCodeForString("seig"); - /** * Flags controlling the behavior of the extractor. */ @Retention(RetentionPolicy.SOURCE) @IntDef(flag = true, value = {FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME, - FLAG_WORKAROUND_IGNORE_TFDT_BOX, FLAG_ENABLE_EMSG_TRACK, FLAG_SIDELOADED}) + FLAG_WORKAROUND_IGNORE_TFDT_BOX, FLAG_ENABLE_EMSG_TRACK, FLAG_ENABLE_CEA608_TRACK, + FLAG_SIDELOADED}) public @interface Flags {} /** * Flag to work around an issue in some video streams where every frame is marked as a sync frame. @@ -94,12 +93,20 @@ public final class FragmentedMp4Extractor implements Extractor { * messages in the stream will be delivered as samples to this track. */ public static final int FLAG_ENABLE_EMSG_TRACK = 4; + /** + * Flag to indicate that the extractor should output a CEA-608 text track. Any CEA-608 messages + * contained within SEI NAL units in the stream will be delivered as samples to this track. + */ + public static final int FLAG_ENABLE_CEA608_TRACK = 8; /** * Flag to indicate that the {@link Track} was sideloaded, instead of being declared by the MP4 * container. */ - private static final int FLAG_SIDELOADED = 8; + private static final int FLAG_SIDELOADED = 16; + private static final String TAG = "FragmentedMp4Extractor"; + private static final int SAMPLE_GROUP_TYPE_seig = Util.getIntegerCodeForString("seig"); + private static final int NAL_UNIT_TYPE_SEI = 6; // Supplemental enhancement information private static final byte[] PIFF_SAMPLE_ENCRYPTION_BOX_EXTENDED_TYPE = new byte[] {-94, 57, 79, 82, 90, -101, 79, 20, -94, 68, 108, 66, 124, 100, -115, -12}; @@ -121,6 +128,7 @@ public final class FragmentedMp4Extractor implements Extractor { // Temporary arrays. private final ParsableByteArray nalStartCode; private final ParsableByteArray nalLength; + private final ParsableByteArray nalPayload; private final ParsableByteArray encryptionSignalByte; // Adjusts sample timestamps. @@ -150,6 +158,7 @@ public final class FragmentedMp4Extractor implements Extractor { // Extractor output. private ExtractorOutput extractorOutput; private TrackOutput eventMessageTrackOutput; + private TrackOutput cea608TrackOutput; // Whether extractorOutput.seekMap has been called. private boolean haveOutputSeekMap; @@ -180,6 +189,7 @@ public final class FragmentedMp4Extractor implements Extractor { atomHeader = new ParsableByteArray(Atom.LONG_HEADER_SIZE); nalStartCode = new ParsableByteArray(NalUnitUtil.NAL_START_CODE); nalLength = new ParsableByteArray(4); + nalPayload = new ParsableByteArray(1); encryptionSignalByte = new ParsableByteArray(1); extendedTypeScratch = new byte[16]; containerAtoms = new Stack<>(); @@ -202,7 +212,7 @@ public final class FragmentedMp4Extractor implements Extractor { TrackBundle bundle = new TrackBundle(output.track(0)); bundle.init(sideloadedTrack, new DefaultSampleValues(0, 0, 0, 0)); trackBundles.put(0, bundle); - maybeInitEventMessageTrack(); + maybeInitExtraTracks(); extractorOutput.endTracks(); } } @@ -413,7 +423,7 @@ public final class FragmentedMp4Extractor implements Extractor { trackBundles.put(track.id, new TrackBundle(extractorOutput.track(i))); durationUs = Math.max(durationUs, track.durationUs); } - maybeInitEventMessageTrack(); + maybeInitExtraTracks(); extractorOutput.endTracks(); } else { Assertions.checkState(trackBundles.size() == trackCount); @@ -437,13 +447,17 @@ public final class FragmentedMp4Extractor implements Extractor { } } - private void maybeInitEventMessageTrack() { - if ((flags & FLAG_ENABLE_EMSG_TRACK) == 0) { - return; + private void maybeInitExtraTracks() { + if ((flags & FLAG_ENABLE_EMSG_TRACK) != 0 && eventMessageTrackOutput == null) { + eventMessageTrackOutput = extractorOutput.track(trackBundles.size()); + eventMessageTrackOutput.format(Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG, + Format.OFFSET_SAMPLE_RELATIVE)); + } + if ((flags & FLAG_ENABLE_CEA608_TRACK) != 0 && cea608TrackOutput == null) { + cea608TrackOutput = extractorOutput.track(trackBundles.size()); + cea608TrackOutput.format(Format.createTextSampleFormat(null, MimeTypes.APPLICATION_CEA608, + null, Format.NO_VALUE, 0, null, null)); } - eventMessageTrackOutput = extractorOutput.track(trackBundles.size()); - eventMessageTrackOutput.format(Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG, - Format.OFFSET_SAMPLE_RELATIVE)); } /** @@ -1065,6 +1079,26 @@ public final class FragmentedMp4Extractor implements Extractor { output.sampleData(nalStartCode, 4); sampleBytesWritten += 4; sampleSize += nalUnitLengthFieldLengthDiff; + if (cea608TrackOutput != null) { + byte[] nalPayloadData = nalPayload.data; + // Peek the NAL unit type byte. + input.peekFully(nalPayloadData, 0, 1); + if ((nalPayloadData[0] & 0x1F) == NAL_UNIT_TYPE_SEI) { + // Read the whole SEI NAL unit into nalWrapper, including the NAL unit type byte. + nalPayload.reset(sampleCurrentNalBytesRemaining); + input.readFully(nalPayloadData, 0, sampleCurrentNalBytesRemaining); + // Write the SEI unit straight to the output. + output.sampleData(nalPayload, sampleCurrentNalBytesRemaining); + sampleBytesWritten += sampleCurrentNalBytesRemaining; + sampleCurrentNalBytesRemaining = 0; + // Unescape and process the SEI unit. + int unescapedLength = NalUnitUtil.unescapeStream(nalPayloadData, nalPayload.limit()); + nalPayload.setPosition(1); // Skip the NAL unit type byte. + nalPayload.setLimit(unescapedLength); + CeaUtil.consume(fragment.getSamplePresentationTime(sampleIndex) * 1000L, nalPayload, + cea608TrackOutput); + } + } } else { // Write the payload of the NAL unit. int writtenBytes = output.sampleData(input, sampleCurrentNalBytesRemaining, false); diff --git a/library/src/main/java/com/google/android/exoplayer2/extractor/ts/SeiReader.java b/library/src/main/java/com/google/android/exoplayer2/extractor/ts/SeiReader.java index a2791bcaae..6e2e42d8e2 100644 --- a/library/src/main/java/com/google/android/exoplayer2/extractor/ts/SeiReader.java +++ b/library/src/main/java/com/google/android/exoplayer2/extractor/ts/SeiReader.java @@ -15,10 +15,9 @@ */ package com.google.android.exoplayer2.extractor.ts; -import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.extractor.TrackOutput; -import com.google.android.exoplayer2.text.cea.Cea608Decoder; +import com.google.android.exoplayer2.text.cea.CeaUtil; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.ParsableByteArray; @@ -36,40 +35,7 @@ import com.google.android.exoplayer2.util.ParsableByteArray; } public void consume(long pesTimeUs, ParsableByteArray seiBuffer) { - int b; - while (seiBuffer.bytesLeft() > 1 /* last byte will be rbsp_trailing_bits */) { - // Parse payload type. - int payloadType = 0; - do { - b = seiBuffer.readUnsignedByte(); - payloadType += b; - } while (b == 0xFF); - // Parse payload size. - int payloadSize = 0; - do { - b = seiBuffer.readUnsignedByte(); - payloadSize += b; - } while (b == 0xFF); - // Process the payload. - if (Cea608Decoder.isSeiMessageCea608(payloadType, payloadSize, seiBuffer)) { - // Ignore country_code (1) + provider_code (2) + user_identifier (4) - // + user_data_type_code (1). - seiBuffer.skipBytes(8); - // Ignore first three bits: reserved (1) + process_cc_data_flag (1) + zero_bit (1). - int ccCount = seiBuffer.readUnsignedByte() & 0x1F; - // Ignore em_data (1) - seiBuffer.skipBytes(1); - // Each data packet consists of 24 bits: marker bits (5) + cc_valid (1) + cc_type (2) - // + cc_data_1 (8) + cc_data_2 (8). - int sampleLength = ccCount * 3; - output.sampleData(seiBuffer, sampleLength); - output.sampleMetadata(pesTimeUs, C.BUFFER_FLAG_KEY_FRAME, sampleLength, 0, null); - // Ignore trailing information in SEI, if any. - seiBuffer.skipBytes(payloadSize - (10 + ccCount * 3)); - } else { - seiBuffer.skipBytes(payloadSize); - } - } + CeaUtil.consume(pesTimeUs, seiBuffer, output); } } diff --git a/library/src/main/java/com/google/android/exoplayer2/text/cea/Cea608Decoder.java b/library/src/main/java/com/google/android/exoplayer2/text/cea/Cea608Decoder.java index 3ae8ded9ba..7324c94288 100644 --- a/library/src/main/java/com/google/android/exoplayer2/text/cea/Cea608Decoder.java +++ b/library/src/main/java/com/google/android/exoplayer2/text/cea/Cea608Decoder.java @@ -49,12 +49,6 @@ public final class Cea608Decoder extends CeaDecoder { private static final int NTSC_CC_FIELD_2 = 0x01; private static final int CC_VALID_608_ID = 0x04; - private static final int PAYLOAD_TYPE_CC = 4; - private static final int COUNTRY_CODE = 0xB5; - private static final int PROVIDER_CODE = 0x31; - private static final int USER_ID = 0x47413934; // "GA94" - private static final int USER_DATA_TYPE_CODE = 0x3; - private static final int CC_MODE_UNKNOWN = 0; private static final int CC_MODE_ROLL_UP = 1; private static final int CC_MODE_POP_ON = 2; @@ -573,31 +567,6 @@ public final class Cea608Decoder extends CeaDecoder { return (cc1 & 0xF0) == 0x10; } - /** - * Inspects an sei message to determine whether it contains CEA-608. - *

- * The position of {@code payload} is left unchanged. - * - * @param payloadType The payload type of the message. - * @param payloadLength The length of the payload. - * @param payload A {@link ParsableByteArray} containing the payload. - * @return Whether the sei message contains CEA-608. - */ - public static boolean isSeiMessageCea608(int payloadType, int payloadLength, - ParsableByteArray payload) { - if (payloadType != PAYLOAD_TYPE_CC || payloadLength < 8) { - return false; - } - int startPosition = payload.getPosition(); - int countryCode = payload.readUnsignedByte(); - int providerCode = payload.readUnsignedShort(); - int userIdentifier = payload.readInt(); - int userDataTypeCode = payload.readUnsignedByte(); - payload.setPosition(startPosition); - return countryCode == COUNTRY_CODE && providerCode == PROVIDER_CODE - && userIdentifier == USER_ID && userDataTypeCode == USER_DATA_TYPE_CODE; - } - private static class CueBuilder { private static final int POSITION_UNSET = -1; diff --git a/library/src/main/java/com/google/android/exoplayer2/text/cea/CeaUtil.java b/library/src/main/java/com/google/android/exoplayer2/text/cea/CeaUtil.java new file mode 100644 index 0000000000..3053debfcf --- /dev/null +++ b/library/src/main/java/com/google/android/exoplayer2/text/cea/CeaUtil.java @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.text.cea; + +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.extractor.TrackOutput; +import com.google.android.exoplayer2.util.ParsableByteArray; + +/** + * Utility methods for handling CEA-608/708 messages. + */ +public final class CeaUtil { + + private static final int PAYLOAD_TYPE_CC = 4; + private static final int COUNTRY_CODE = 0xB5; + private static final int PROVIDER_CODE = 0x31; + private static final int USER_ID = 0x47413934; // "GA94" + private static final int USER_DATA_TYPE_CODE = 0x3; + + /** + * Consumes the unescaped content of an SEI NAL unit, writing the content of any CEA-608 messages + * as samples to the provided output. + * + * @param presentationTimeUs The presentation time in microseconds for any samples. + * @param seiBuffer The unescaped SEI NAL unit data, excluding the NAL unit start code and type. + * @param output The output to which any samples should be written. + */ + public static void consume(long presentationTimeUs, ParsableByteArray seiBuffer, + TrackOutput output) { + int b; + while (seiBuffer.bytesLeft() > 1 /* last byte will be rbsp_trailing_bits */) { + // Parse payload type. + int payloadType = 0; + do { + b = seiBuffer.readUnsignedByte(); + payloadType += b; + } while (b == 0xFF); + // Parse payload size. + int payloadSize = 0; + do { + b = seiBuffer.readUnsignedByte(); + payloadSize += b; + } while (b == 0xFF); + // Process the payload. + if (isSeiMessageCea608(payloadType, payloadSize, seiBuffer)) { + // Ignore country_code (1) + provider_code (2) + user_identifier (4) + // + user_data_type_code (1). + seiBuffer.skipBytes(8); + // Ignore first three bits: reserved (1) + process_cc_data_flag (1) + zero_bit (1). + int ccCount = seiBuffer.readUnsignedByte() & 0x1F; + // Ignore em_data (1) + seiBuffer.skipBytes(1); + // Each data packet consists of 24 bits: marker bits (5) + cc_valid (1) + cc_type (2) + // + cc_data_1 (8) + cc_data_2 (8). + int sampleLength = ccCount * 3; + output.sampleData(seiBuffer, sampleLength); + output.sampleMetadata(presentationTimeUs, C.BUFFER_FLAG_KEY_FRAME, sampleLength, 0, null); + // Ignore trailing information in SEI, if any. + seiBuffer.skipBytes(payloadSize - (10 + ccCount * 3)); + } else { + seiBuffer.skipBytes(payloadSize); + } + } + } + + /** + * Inspects an sei message to determine whether it contains CEA-608. + *

+ * The position of {@code payload} is left unchanged. + * + * @param payloadType The payload type of the message. + * @param payloadLength The length of the payload. + * @param payload A {@link ParsableByteArray} containing the payload. + * @return Whether the sei message contains CEA-608. + */ + private static boolean isSeiMessageCea608(int payloadType, int payloadLength, + ParsableByteArray payload) { + if (payloadType != PAYLOAD_TYPE_CC || payloadLength < 8) { + return false; + } + int startPosition = payload.getPosition(); + int countryCode = payload.readUnsignedByte(); + int providerCode = payload.readUnsignedShort(); + int userIdentifier = payload.readInt(); + int userDataTypeCode = payload.readUnsignedByte(); + payload.setPosition(startPosition); + return countryCode == COUNTRY_CODE && providerCode == PROVIDER_CODE + && userIdentifier == USER_ID && userDataTypeCode == USER_DATA_TYPE_CODE; + } + + private CeaUtil() {} + +}