Add support for 608/708 captions in HLS+fMP4

This also allows exposing multiple CC channels to any fMP4 extractor client.

Issue:#1661

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=174458725

library/core/src/androidTest/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4ExtractorTest.java

@@ -16,9 +16,13 @@
 package com.google.android.exoplayer2.extractor.mp4;
 
 import android.test.InstrumentationTestCase;
+import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.extractor.Extractor;
 import com.google.android.exoplayer2.testutil.ExtractorAsserts;
 import com.google.android.exoplayer2.testutil.ExtractorAsserts.ExtractorFactory;
+import com.google.android.exoplayer2.util.MimeTypes;
+import java.util.Collections;
+import java.util.List;
 
 /**
  * Unit test for {@link FragmentedMp4Extractor}.
@@ -26,26 +30,23 @@ import com.google.android.exoplayer2.testutil.ExtractorAsserts.ExtractorFactory;
 public final class FragmentedMp4ExtractorTest extends InstrumentationTestCase {
 
   public void testSample() throws Exception {
-    ExtractorAsserts.assertBehavior(getExtractorFactory(), "mp4/sample_fragmented.mp4",
+    ExtractorAsserts.assertBehavior(getExtractorFactory(Collections.<Format>emptyList()),
-        getInstrumentation());
+        "mp4/sample_fragmented.mp4", getInstrumentation());
   }
 
   public void testSampleWithSeiPayloadParsing() throws Exception {
     // Enabling the CEA-608 track enables SEI payload parsing.
-    ExtractorAsserts.assertBehavior(
+    ExtractorFactory extractorFactory = getExtractorFactory(Collections.singletonList(
-        getExtractorFactory(FragmentedMp4Extractor.FLAG_ENABLE_CEA608_TRACK),
+        Format.createTextSampleFormat(null, MimeTypes.APPLICATION_CEA608, 0, null)));
-        "mp4/sample_fragmented_sei.mp4", getInstrumentation());
+    ExtractorAsserts.assertBehavior(extractorFactory, "mp4/sample_fragmented_sei.mp4",
+        getInstrumentation());
   }
 
-  private static ExtractorFactory getExtractorFactory() {
+  private static ExtractorFactory getExtractorFactory(final List<Format> closedCaptionFormats) {
-    return getExtractorFactory(0);
-  }
-
-  private static ExtractorFactory getExtractorFactory(final int flags) {
     return new ExtractorFactory() {
       @Override
       public Extractor create() {
-        return new FragmentedMp4Extractor(flags, null);
+        return new FragmentedMp4Extractor(0, null, null, null, closedCaptionFormats);
       }
     };
   }

library/core/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java

@@ -46,6 +46,7 @@ import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Stack;
@@ -73,8 +74,8 @@ public final class FragmentedMp4Extractor implements Extractor {
    */
   @Retention(RetentionPolicy.SOURCE)
   @IntDef(flag = true, value = {FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME,
-      FLAG_WORKAROUND_IGNORE_TFDT_BOX, FLAG_ENABLE_EMSG_TRACK, FLAG_ENABLE_CEA608_TRACK,
+      FLAG_WORKAROUND_IGNORE_TFDT_BOX, FLAG_ENABLE_EMSG_TRACK, FLAG_SIDELOADED,
-      FLAG_SIDELOADED, FLAG_WORKAROUND_IGNORE_EDIT_LISTS})
+      FLAG_WORKAROUND_IGNORE_EDIT_LISTS})
   public @interface Flags {}
   /**
    * Flag to work around an issue in some video streams where every frame is marked as a sync frame.
@@ -93,20 +94,15 @@ public final class FragmentedMp4Extractor implements Extractor {
    * messages in the stream will be delivered as samples to this track.
    */
   public static final int FLAG_ENABLE_EMSG_TRACK = 4;
-  /**
-   * Flag to indicate that the extractor should output a CEA-608 text track. Any CEA-608 messages
-   * contained within SEI NAL units in the stream will be delivered as samples to this track.
-   */
-  public static final int FLAG_ENABLE_CEA608_TRACK = 8;
   /**
    * Flag to indicate that the {@link Track} was sideloaded, instead of being declared by the MP4
    * container.
    */
-  private static final int FLAG_SIDELOADED = 16;
+  private static final int FLAG_SIDELOADED = 8;
   /**
    * Flag to ignore any edit lists in the stream.
    */
-  public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 32;
+  public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 16;
 
   private static final String TAG = "FragmentedMp4Extractor";
   private static final int SAMPLE_GROUP_TYPE_seig = Util.getIntegerCodeForString("seig");
@@ -124,7 +120,8 @@ public final class FragmentedMp4Extractor implements Extractor {
   @Flags private final int flags;
   private final Track sideloadedTrack;
 
-  // Manifest DRM data.
+  // Sideloaded data.
+  private final List<Format> closedCaptionFormats;
   private final DrmInitData sideloadedDrmInitData;
 
   // Track-linked data bundle, accessible as a whole through trackID.
@@ -193,15 +190,33 @@ public final class FragmentedMp4Extractor implements Extractor {
    * @param flags Flags that control the extractor's behavior.
    * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed.
    * @param sideloadedTrack Sideloaded track information, in the case that the extractor
-   *     will not receive a moov box in the input data.
+   *     will not receive a moov box in the input data. Null if a moov box is expected.
-   * @param sideloadedDrmInitData The {@link DrmInitData} to use for encrypted tracks.
+   * @param sideloadedDrmInitData The {@link DrmInitData} to use for encrypted tracks. If null, the
+   *     pssh boxes (if present) will be used.
    */
   public FragmentedMp4Extractor(@Flags int flags, TimestampAdjuster timestampAdjuster,
       Track sideloadedTrack, DrmInitData sideloadedDrmInitData) {
+    this(flags, timestampAdjuster, sideloadedTrack, sideloadedDrmInitData,
+        Collections.<Format>emptyList());
+  }
+
+  /**
+   * @param flags Flags that control the extractor's behavior.
+   * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed.
+   * @param sideloadedTrack Sideloaded track information, in the case that the extractor
+   *     will not receive a moov box in the input data. Null if a moov box is expected.
+   * @param sideloadedDrmInitData The {@link DrmInitData} to use for encrypted tracks. If null, the
+   *     pssh boxes (if present) will be used.
+   * @param closedCaptionFormats For tracks that contain SEI messages, the formats of the closed
+   *     caption channels to expose.
+   */
+  public FragmentedMp4Extractor(@Flags int flags, TimestampAdjuster timestampAdjuster,
+      Track sideloadedTrack, DrmInitData sideloadedDrmInitData, List<Format> closedCaptionFormats) {
     this.flags = flags | (sideloadedTrack != null ? FLAG_SIDELOADED : 0);
     this.timestampAdjuster = timestampAdjuster;
     this.sideloadedTrack = sideloadedTrack;
     this.sideloadedDrmInitData = sideloadedDrmInitData;
+    this.closedCaptionFormats = Collections.unmodifiableList(closedCaptionFormats);
     atomHeader = new ParsableByteArray(Atom.LONG_HEADER_SIZE);
     nalStartCode = new ParsableByteArray(NalUnitUtil.NAL_START_CODE);
     nalPrefix = new ParsableByteArray(5);
@@ -483,12 +498,13 @@ public final class FragmentedMp4Extractor implements Extractor {
       eventMessageTrackOutput.format(Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG,
           Format.OFFSET_SAMPLE_RELATIVE));
     }
-    if ((flags & FLAG_ENABLE_CEA608_TRACK) != 0 && cea608TrackOutputs == null) {
+    if (cea608TrackOutputs == null) {
-      TrackOutput cea608TrackOutput = extractorOutput.track(trackBundles.size() + 1,
+      cea608TrackOutputs = new TrackOutput[closedCaptionFormats.size()];
-          C.TRACK_TYPE_TEXT);
+      for (int i = 0; i < cea608TrackOutputs.length; i++) {
-      cea608TrackOutput.format(Format.createTextSampleFormat(null, MimeTypes.APPLICATION_CEA608, 0,
+        TrackOutput output = extractorOutput.track(trackBundles.size() + 1 + i, C.TRACK_TYPE_TEXT);
-          null));
+        output.format(closedCaptionFormats.get(i));
-      cea608TrackOutputs = new TrackOutput[] {cea608TrackOutput};
+        cea608TrackOutputs[i] = output;
+      }
     }
   }
 
@@ -1123,7 +1139,7 @@ public final class FragmentedMp4Extractor implements Extractor {
           output.sampleData(nalStartCode, 4);
           // Write the NAL unit type byte.
           output.sampleData(nalPrefix, 1);
-          processSeiNalUnitPayload = cea608TrackOutputs != null
+          processSeiNalUnitPayload = cea608TrackOutputs.length > 0
               && NalUnitUtil.isNalUnitSei(track.format.sampleMimeType, nalPrefixData[4]);
           sampleBytesWritten += 5;
           sampleSize += nalUnitLengthFieldLengthDiff;

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java

@@ -47,6 +47,7 @@ import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.Util;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -424,10 +425,12 @@ public class DefaultDashChunkSource implements DashChunkSource {
           if (enableEventMessageTrack) {
             flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
           }
-          if (enableCea608Track) {
+          // TODO: Use caption format information from the manifest if available.
-            flags |= FragmentedMp4Extractor.FLAG_ENABLE_CEA608_TRACK;
+          List<Format> closedCaptionFormats = enableCea608Track
-          }
+              ? Collections.singletonList(
-          extractor = new FragmentedMp4Extractor(flags);
+                  Format.createTextSampleFormat(null, MimeTypes.APPLICATION_CEA608, 0, null))
+              : Collections.<Format>emptyList();
+          extractor = new FragmentedMp4Extractor(flags, null, null, null, closedCaptionFormats);
         }
         // Prefer drmInitData obtained from the manifest over drmInitData obtained from the stream,
         // as per DASH IF Interoperability Recommendations V3.0, 7.5.3.

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/DefaultHlsExtractorFactory.java

@@ -74,7 +74,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
       extractor = previousExtractor;
     } else if (lastPathSegment.endsWith(MP4_FILE_EXTENSION)
         || lastPathSegment.startsWith(M4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 4)) {
-      extractor = new FragmentedMp4Extractor(0, timestampAdjuster, null, drmInitData);
+      extractor = new FragmentedMp4Extractor(0, timestampAdjuster, null, drmInitData,
+          muxedCaptionFormats != null ? muxedCaptionFormats : Collections.<Format>emptyList());
     } else {
       // For any other file extension, we assume TS format.
       @DefaultTsPayloadReaderFactory.Flags