Allow FMP4 extractor to output CEA-608

Issue: #2362

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=145401668

library/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java

@@ -34,6 +34,7 @@ import com.google.android.exoplayer2.extractor.SeekMap;
 import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.extractor.mp4.Atom.ContainerAtom;
 import com.google.android.exoplayer2.extractor.mp4.Atom.LeafAtom;
+import com.google.android.exoplayer2.text.cea.CeaUtil;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.NalUnitUtil;
@@ -67,15 +68,13 @@ public final class FragmentedMp4Extractor implements Extractor {
 
   };
 
-  private static final String TAG = "FragmentedMp4Extractor";
-  private static final int SAMPLE_GROUP_TYPE_seig = Util.getIntegerCodeForString("seig");
-
   /**
    * Flags controlling the behavior of the extractor.
    */
   @Retention(RetentionPolicy.SOURCE)
   @IntDef(flag = true, value = {FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME,
-      FLAG_WORKAROUND_IGNORE_TFDT_BOX, FLAG_ENABLE_EMSG_TRACK, FLAG_SIDELOADED})
+      FLAG_WORKAROUND_IGNORE_TFDT_BOX, FLAG_ENABLE_EMSG_TRACK, FLAG_ENABLE_CEA608_TRACK,
+      FLAG_SIDELOADED})
   public @interface Flags {}
   /**
    * Flag to work around an issue in some video streams where every frame is marked as a sync frame.
@@ -94,12 +93,20 @@ public final class FragmentedMp4Extractor implements Extractor {
    * messages in the stream will be delivered as samples to this track.
    */
   public static final int FLAG_ENABLE_EMSG_TRACK = 4;
+  /**
+   * Flag to indicate that the extractor should output a CEA-608 text track. Any CEA-608 messages
+   * contained within SEI NAL units in the stream will be delivered as samples to this track.
+   */
+  public static final int FLAG_ENABLE_CEA608_TRACK = 8;
   /**
    * Flag to indicate that the {@link Track} was sideloaded, instead of being declared by the MP4
    * container.
    */
-  private static final int FLAG_SIDELOADED = 8;
+  private static final int FLAG_SIDELOADED = 16;
 
+  private static final String TAG = "FragmentedMp4Extractor";
+  private static final int SAMPLE_GROUP_TYPE_seig = Util.getIntegerCodeForString("seig");
+  private static final int NAL_UNIT_TYPE_SEI = 6; // Supplemental enhancement information
   private static final byte[] PIFF_SAMPLE_ENCRYPTION_BOX_EXTENDED_TYPE =
       new byte[] {-94, 57, 79, 82, 90, -101, 79, 20, -94, 68, 108, 66, 124, 100, -115, -12};
 
@@ -121,6 +128,7 @@ public final class FragmentedMp4Extractor implements Extractor {
   // Temporary arrays.
   private final ParsableByteArray nalStartCode;
   private final ParsableByteArray nalLength;
+  private final ParsableByteArray nalPayload;
   private final ParsableByteArray encryptionSignalByte;
 
   // Adjusts sample timestamps.
@@ -150,6 +158,7 @@ public final class FragmentedMp4Extractor implements Extractor {
   // Extractor output.
   private ExtractorOutput extractorOutput;
   private TrackOutput eventMessageTrackOutput;
+  private TrackOutput cea608TrackOutput;
 
   // Whether extractorOutput.seekMap has been called.
   private boolean haveOutputSeekMap;
@@ -180,6 +189,7 @@ public final class FragmentedMp4Extractor implements Extractor {
     atomHeader = new ParsableByteArray(Atom.LONG_HEADER_SIZE);
     nalStartCode = new ParsableByteArray(NalUnitUtil.NAL_START_CODE);
     nalLength = new ParsableByteArray(4);
+    nalPayload = new ParsableByteArray(1);
     encryptionSignalByte = new ParsableByteArray(1);
     extendedTypeScratch = new byte[16];
     containerAtoms = new Stack<>();
@@ -202,7 +212,7 @@ public final class FragmentedMp4Extractor implements Extractor {
       TrackBundle bundle = new TrackBundle(output.track(0));
       bundle.init(sideloadedTrack, new DefaultSampleValues(0, 0, 0, 0));
       trackBundles.put(0, bundle);
-      maybeInitEventMessageTrack();
+      maybeInitExtraTracks();
       extractorOutput.endTracks();
     }
   }
@@ -413,7 +423,7 @@ public final class FragmentedMp4Extractor implements Extractor {
         trackBundles.put(track.id, new TrackBundle(extractorOutput.track(i)));
         durationUs = Math.max(durationUs, track.durationUs);
       }
-      maybeInitEventMessageTrack();
+      maybeInitExtraTracks();
       extractorOutput.endTracks();
     } else {
       Assertions.checkState(trackBundles.size() == trackCount);
@@ -437,13 +447,17 @@ public final class FragmentedMp4Extractor implements Extractor {
     }
   }
 
-  private void maybeInitEventMessageTrack() {
+  private void maybeInitExtraTracks() {
-    if ((flags & FLAG_ENABLE_EMSG_TRACK) == 0) {
+    if ((flags & FLAG_ENABLE_EMSG_TRACK) != 0 && eventMessageTrackOutput == null) {
-      return;
+      eventMessageTrackOutput = extractorOutput.track(trackBundles.size());
+      eventMessageTrackOutput.format(Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG,
+          Format.OFFSET_SAMPLE_RELATIVE));
+    }
+    if ((flags & FLAG_ENABLE_CEA608_TRACK) != 0 && cea608TrackOutput == null) {
+      cea608TrackOutput = extractorOutput.track(trackBundles.size());
+      cea608TrackOutput.format(Format.createTextSampleFormat(null, MimeTypes.APPLICATION_CEA608,
+          null, Format.NO_VALUE, 0, null, null));
     }
-    eventMessageTrackOutput = extractorOutput.track(trackBundles.size());
-    eventMessageTrackOutput.format(Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG,
-        Format.OFFSET_SAMPLE_RELATIVE));
   }
 
   /**
@@ -1065,6 +1079,26 @@ public final class FragmentedMp4Extractor implements Extractor {
           output.sampleData(nalStartCode, 4);
           sampleBytesWritten += 4;
           sampleSize += nalUnitLengthFieldLengthDiff;
+          if (cea608TrackOutput != null) {
+            byte[] nalPayloadData = nalPayload.data;
+            // Peek the NAL unit type byte.
+            input.peekFully(nalPayloadData, 0, 1);
+            if ((nalPayloadData[0] & 0x1F) == NAL_UNIT_TYPE_SEI) {
+              // Read the whole SEI NAL unit into nalWrapper, including the NAL unit type byte.
+              nalPayload.reset(sampleCurrentNalBytesRemaining);
+              input.readFully(nalPayloadData, 0, sampleCurrentNalBytesRemaining);
+              // Write the SEI unit straight to the output.
+              output.sampleData(nalPayload, sampleCurrentNalBytesRemaining);
+              sampleBytesWritten += sampleCurrentNalBytesRemaining;
+              sampleCurrentNalBytesRemaining = 0;
+              // Unescape and process the SEI unit.
+              int unescapedLength = NalUnitUtil.unescapeStream(nalPayloadData, nalPayload.limit());
+              nalPayload.setPosition(1); // Skip the NAL unit type byte.
+              nalPayload.setLimit(unescapedLength);
+              CeaUtil.consume(fragment.getSamplePresentationTime(sampleIndex) * 1000L, nalPayload,
+                  cea608TrackOutput);
+            }
+          }
         } else {
           // Write the payload of the NAL unit.
           int writtenBytes = output.sampleData(input, sampleCurrentNalBytesRemaining, false);

library/src/main/java/com/google/android/exoplayer2/extractor/ts/SeiReader.java

@@ -15,10 +15,9 @@
  */
 package com.google.android.exoplayer2.extractor.ts;
 
-import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.extractor.TrackOutput;
-import com.google.android.exoplayer2.text.cea.Cea608Decoder;
+import com.google.android.exoplayer2.text.cea.CeaUtil;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.ParsableByteArray;
 
@@ -36,40 +35,7 @@ import com.google.android.exoplayer2.util.ParsableByteArray;
   }
 
   public void consume(long pesTimeUs, ParsableByteArray seiBuffer) {
-    int b;
+    CeaUtil.consume(pesTimeUs, seiBuffer, output);
-    while (seiBuffer.bytesLeft() > 1 /* last byte will be rbsp_trailing_bits */) {
-      // Parse payload type.
-      int payloadType = 0;
-      do {
-        b = seiBuffer.readUnsignedByte();
-        payloadType += b;
-      } while (b == 0xFF);
-      // Parse payload size.
-      int payloadSize = 0;
-      do {
-        b = seiBuffer.readUnsignedByte();
-        payloadSize += b;
-      } while (b == 0xFF);
-      // Process the payload.
-      if (Cea608Decoder.isSeiMessageCea608(payloadType, payloadSize, seiBuffer)) {
-        // Ignore country_code (1) + provider_code (2) + user_identifier (4)
-        // + user_data_type_code (1).
-        seiBuffer.skipBytes(8);
-        // Ignore first three bits: reserved (1) + process_cc_data_flag (1) + zero_bit (1).
-        int ccCount = seiBuffer.readUnsignedByte() & 0x1F;
-        // Ignore em_data (1)
-        seiBuffer.skipBytes(1);
-        // Each data packet consists of 24 bits: marker bits (5) + cc_valid (1) + cc_type (2)
-        // + cc_data_1 (8) + cc_data_2 (8).
-        int sampleLength = ccCount * 3;
-        output.sampleData(seiBuffer, sampleLength);
-        output.sampleMetadata(pesTimeUs, C.BUFFER_FLAG_KEY_FRAME, sampleLength, 0, null);
-        // Ignore trailing information in SEI, if any.
-        seiBuffer.skipBytes(payloadSize - (10 + ccCount * 3));
-      } else {
-        seiBuffer.skipBytes(payloadSize);
-      }
-    }
   }
 
 }

library/src/main/java/com/google/android/exoplayer2/text/cea/Cea608Decoder.java

@@ -49,12 +49,6 @@ public final class Cea608Decoder extends CeaDecoder {
   private static final int NTSC_CC_FIELD_2 = 0x01;
   private static final int CC_VALID_608_ID = 0x04;
 
-  private static final int PAYLOAD_TYPE_CC = 4;
-  private static final int COUNTRY_CODE = 0xB5;
-  private static final int PROVIDER_CODE = 0x31;
-  private static final int USER_ID = 0x47413934; // "GA94"
-  private static final int USER_DATA_TYPE_CODE = 0x3;
-
   private static final int CC_MODE_UNKNOWN = 0;
   private static final int CC_MODE_ROLL_UP = 1;
   private static final int CC_MODE_POP_ON = 2;
@@ -573,31 +567,6 @@ public final class Cea608Decoder extends CeaDecoder {
     return (cc1 & 0xF0) == 0x10;
   }
 
-  /**
-   * Inspects an sei message to determine whether it contains CEA-608.
-   * <p>
-   * The position of {@code payload} is left unchanged.
-   *
-   * @param payloadType The payload type of the message.
-   * @param payloadLength The length of the payload.
-   * @param payload A {@link ParsableByteArray} containing the payload.
-   * @return Whether the sei message contains CEA-608.
-   */
-  public static boolean isSeiMessageCea608(int payloadType, int payloadLength,
-      ParsableByteArray payload) {
-    if (payloadType != PAYLOAD_TYPE_CC || payloadLength < 8) {
-      return false;
-    }
-    int startPosition = payload.getPosition();
-    int countryCode = payload.readUnsignedByte();
-    int providerCode = payload.readUnsignedShort();
-    int userIdentifier = payload.readInt();
-    int userDataTypeCode = payload.readUnsignedByte();
-    payload.setPosition(startPosition);
-    return countryCode == COUNTRY_CODE && providerCode == PROVIDER_CODE
-        && userIdentifier == USER_ID && userDataTypeCode == USER_DATA_TYPE_CODE;
-  }
-
   private static class CueBuilder {
 
     private static final int POSITION_UNSET = -1;

library/src/main/java/com/google/android/exoplayer2/text/cea/CeaUtil.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.text.cea;
+
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.extractor.TrackOutput;
+import com.google.android.exoplayer2.util.ParsableByteArray;
+
+/**
+ * Utility methods for handling CEA-608/708 messages.
+ */
+public final class CeaUtil {
+
+  private static final int PAYLOAD_TYPE_CC = 4;
+  private static final int COUNTRY_CODE = 0xB5;
+  private static final int PROVIDER_CODE = 0x31;
+  private static final int USER_ID = 0x47413934; // "GA94"
+  private static final int USER_DATA_TYPE_CODE = 0x3;
+
+  /**
+   * Consumes the unescaped content of an SEI NAL unit, writing the content of any CEA-608 messages
+   * as samples to the provided output.
+   *
+   * @param presentationTimeUs The presentation time in microseconds for any samples.
+   * @param seiBuffer The unescaped SEI NAL unit data, excluding the NAL unit start code and type.
+   * @param output The output to which any samples should be written.
+   */
+  public static void consume(long presentationTimeUs, ParsableByteArray seiBuffer,
+      TrackOutput output) {
+    int b;
+    while (seiBuffer.bytesLeft() > 1 /* last byte will be rbsp_trailing_bits */) {
+      // Parse payload type.
+      int payloadType = 0;
+      do {
+        b = seiBuffer.readUnsignedByte();
+        payloadType += b;
+      } while (b == 0xFF);
+      // Parse payload size.
+      int payloadSize = 0;
+      do {
+        b = seiBuffer.readUnsignedByte();
+        payloadSize += b;
+      } while (b == 0xFF);
+      // Process the payload.
+      if (isSeiMessageCea608(payloadType, payloadSize, seiBuffer)) {
+        // Ignore country_code (1) + provider_code (2) + user_identifier (4)
+        // + user_data_type_code (1).
+        seiBuffer.skipBytes(8);
+        // Ignore first three bits: reserved (1) + process_cc_data_flag (1) + zero_bit (1).
+        int ccCount = seiBuffer.readUnsignedByte() & 0x1F;
+        // Ignore em_data (1)
+        seiBuffer.skipBytes(1);
+        // Each data packet consists of 24 bits: marker bits (5) + cc_valid (1) + cc_type (2)
+        // + cc_data_1 (8) + cc_data_2 (8).
+        int sampleLength = ccCount * 3;
+        output.sampleData(seiBuffer, sampleLength);
+        output.sampleMetadata(presentationTimeUs, C.BUFFER_FLAG_KEY_FRAME, sampleLength, 0, null);
+        // Ignore trailing information in SEI, if any.
+        seiBuffer.skipBytes(payloadSize - (10 + ccCount * 3));
+      } else {
+        seiBuffer.skipBytes(payloadSize);
+      }
+    }
+  }
+
+  /**
+   * Inspects an sei message to determine whether it contains CEA-608.
+   * <p>
+   * The position of {@code payload} is left unchanged.
+   *
+   * @param payloadType The payload type of the message.
+   * @param payloadLength The length of the payload.
+   * @param payload A {@link ParsableByteArray} containing the payload.
+   * @return Whether the sei message contains CEA-608.
+   */
+  private static boolean isSeiMessageCea608(int payloadType, int payloadLength,
+      ParsableByteArray payload) {
+    if (payloadType != PAYLOAD_TYPE_CC || payloadLength < 8) {
+      return false;
+    }
+    int startPosition = payload.getPosition();
+    int countryCode = payload.readUnsignedByte();
+    int providerCode = payload.readUnsignedShort();
+    int userIdentifier = payload.readInt();
+    int userDataTypeCode = payload.readUnsignedByte();
+    payload.setPosition(startPosition);
+    return countryCode == COUNTRY_CODE && providerCode == PROVIDER_CODE
+        && userIdentifier == USER_ID && userDataTypeCode == USER_DATA_TYPE_CODE;
+  }
+
+  private CeaUtil() {}
+
+}