Plumb SubtitleParser.Factory into FragmentedMp4Extractor

We introduce SubtitleParser.Factory that supports no formats to be used FragmentedMp4Extractors that will not do any subtitle parsing on the extraction side. We also slowly move away from using SubtitleTranscodingExtractor to SubtitleTranscodingExtractorOutput (hence making it public).

This is required by individual Extractor impls so that they can start using SubtitleTranscodingExtractorOutput rather than be wrapped by SubtitleTranscodingExtractor. The latter is to be deprecated after all the subtitle related Extractors have achieved this migration.

PiperOrigin-RevId: 596942147

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java

@@ -105,19 +105,27 @@ public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtrac
       } else if (Objects.equals(containerMimeType, MimeTypes.IMAGE_PNG)) {
         extractor = new PngExtractor();
       } else {
-        int flags = 0;
+        @FragmentedMp4Extractor.Flags int flags = 0;
         if (enableEventMessageTrack) {
           flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
         }
+        if (subtitleParserFactory == null) {
+          flags |= FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA;
+        }
         extractor =
             new FragmentedMp4Extractor(
+                subtitleParserFactory != null
+                    ? subtitleParserFactory
+                    : SubtitleParser.Factory.UNSUPPORTED,
                 flags,
                 /* timestampAdjuster= */ null,
                 /* sideloadedTrack= */ null,
                 closedCaptionFormats,
                 playerEmsgTrackOutput);
       }
-      if (subtitleParserFactory != null && !MimeTypes.isText(containerMimeType)) {
+      if (subtitleParserFactory != null
+          && !MimeTypes.isText(containerMimeType)
+          && !(extractor.getUnderlyingImplementation() instanceof FragmentedMp4Extractor)) {
         extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
       }
       return new BundledChunkExtractor(extractor, primaryTrackType, representationFormat);

libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java

@@ -39,6 +39,7 @@ import androidx.media3.extractor.ts.Ac4Extractor;
 import androidx.media3.extractor.ts.AdtsExtractor;
 import androidx.media3.extractor.ts.DefaultTsPayloadReaderFactory;
 import androidx.media3.extractor.ts.TsExtractor;
+import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
 import java.io.EOFException;
 import java.io.IOException;
@@ -201,11 +202,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
       case FileTypes.MP3:
         return new Mp3Extractor(/* flags= */ 0, /* forcedFirstSampleTimestampUs= */ 0);
       case FileTypes.MP4:
-        Extractor mp4Extractor =
+        return createFragmentedMp4Extractor(
-            createFragmentedMp4Extractor(timestampAdjuster, format, muxedCaptionFormats);
+            subtitleParserFactory, timestampAdjuster, format, muxedCaptionFormats);
-        return subtitleParserFactory != null
-            ? new SubtitleTranscodingExtractor(mp4Extractor, subtitleParserFactory)
-            : mp4Extractor;
       case FileTypes.TS:
         Extractor tsExtractor =
             createTsExtractor(
@@ -264,16 +262,25 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
   }
 
   private static FragmentedMp4Extractor createFragmentedMp4Extractor(
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       TimestampAdjuster timestampAdjuster,
       Format format,
       @Nullable List<Format> muxedCaptionFormats) {
     // Only enable the EMSG TrackOutput if this is the 'variant' track (i.e. the main one) to avoid
     // creating a separate EMSG track for every audio track in a video stream.
+    @FragmentedMp4Extractor.Flags
+    int flags = isFmp4Variant(format) ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0;
+    if (subtitleParserFactory == null) {
+      subtitleParserFactory = SubtitleParser.Factory.UNSUPPORTED;
+      flags |= FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA;
+    }
     return new FragmentedMp4Extractor(
-        /* flags= */ isFmp4Variant(format) ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0,
+        subtitleParserFactory,
+        flags,
         timestampAdjuster,
         /* sideloadedTrack= */ null,
-        muxedCaptionFormats != null ? muxedCaptionFormats : Collections.emptyList());
+        muxedCaptionFormats != null ? muxedCaptionFormats : ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
   }
 
   /** Returns true if this {@code format} represents a 'variant' track (i.e. the main one). */

libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java

@@ -53,7 +53,7 @@ import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
 import androidx.media3.extractor.mp4.Track;
 import androidx.media3.extractor.mp4.TrackEncryptionBox;
 import androidx.media3.extractor.text.SubtitleParser;
-import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
+import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.util.List;
 
@@ -173,15 +173,22 @@ public class DefaultSsChunkSource implements SsChunkSource {
               nalUnitLengthFieldLength,
               null,
               null);
+      @FragmentedMp4Extractor.Flags
+      int flags =
+          FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME
+              | FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX;
       Extractor extractor =
           new FragmentedMp4Extractor(
-              FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME
+              subtitleParserFactory == null
-                  | FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX,
+                  ? SubtitleParser.Factory.UNSUPPORTED
+                  : subtitleParserFactory,
+              subtitleParserFactory == null
+                  ? flags | FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA
+                  : flags,
               /* timestampAdjuster= */ null,
-              track);
+              track,
-      if (subtitleParserFactory != null) {
+              /* closedCaptionFormats= */ ImmutableList.of(),
-        extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
+              /* additionalEmsgTrackOutput= */ null);
-      }
       chunkExtractors[i] = new BundledChunkExtractor(extractor, streamElement.type, format);
     }
   }

libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java

@@ -277,7 +277,7 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory {
   /**
    * Sets flags for {@link FragmentedMp4Extractor} instances created by the factory.
    *
-   * @see FragmentedMp4Extractor#FragmentedMp4Extractor(int)
+   * @see FragmentedMp4Extractor#FragmentedMp4Extractor(SubtitleParser.Factory, int)
    * @param flags The flags to use.
    * @return The factory, for convenience.
    */
@@ -436,10 +436,12 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory {
     }
     Extractor[] result = new Extractor[extractors.size()];
     for (int i = 0; i < extractors.size(); i++) {
+      Extractor extractor = extractors.get(i);
       result[i] =
           textTrackTranscodingEnabled
-              ? new SubtitleTranscodingExtractor(extractors.get(i), subtitleParserFactory)
+                  && !(extractor.getUnderlyingImplementation() instanceof FragmentedMp4Extractor)
-              : extractors.get(i);
+              ? new SubtitleTranscodingExtractor(extractor, subtitleParserFactory)
+              : extractor;
     }
     return result;
   }
@@ -500,7 +502,13 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory {
                         : 0)));
         break;
       case FileTypes.MP4:
-        extractors.add(new FragmentedMp4Extractor(fragmentedMp4Flags));
+        extractors.add(
+            new FragmentedMp4Extractor(
+                subtitleParserFactory,
+                fragmentedMp4Flags
+                    | (textTrackTranscodingEnabled
+                        ? 0
+                        : FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA)));
         extractors.add(new Mp4Extractor(mp4Flags));
         break;
       case FileTypes.OGG:

libraries/extractor/src/main/java/androidx/media3/extractor/Extractor.java

@@ -25,6 +25,7 @@ import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import org.checkerframework.dataflow.qual.SideEffectFree;
 
 /** Extracts media data from a container format. */
 @UnstableApi
@@ -130,6 +131,7 @@ public interface Extractor {
    * <p>{@code Extractor} implementations that operate by delegating to another {@code Extractor}
    * should override this method to return that delegate.
    */
+  @SideEffectFree
   default Extractor getUnderlyingImplementation() {
     return this;
   }

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java

@@ -54,6 +54,9 @@ import androidx.media3.extractor.metadata.emsg.EventMessage;
 import androidx.media3.extractor.metadata.emsg.EventMessageEncoder;
 import androidx.media3.extractor.mp4.Atom.ContainerAtom;
 import androidx.media3.extractor.mp4.Atom.LeafAtom;
+import androidx.media3.extractor.text.SubtitleParser;
+import androidx.media3.extractor.text.SubtitleTranscodingExtractorOutput;
+import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
@@ -71,10 +74,21 @@ import java.util.UUID;
 @UnstableApi
 public class FragmentedMp4Extractor implements Extractor {
 
-  /** Factory for {@link FragmentedMp4Extractor} instances. */
+  /**
+   * @deprecated Use {@link #newFactory(SubtitleParser.Factory)} instead.
+   */
+  @Deprecated
   public static final ExtractorsFactory FACTORY =
       () -> new Extractor[] {new FragmentedMp4Extractor()};
 
+  /**
+   * Creates a factory for {@link FragmentedMp4Extractor} instances with the provided {@link
+   * SubtitleParser.Factory}.
+   */
+  public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParserFactory) {
+    return () -> new Extractor[] {new FragmentedMp4Extractor(subtitleParserFactory)};
+  }
+
   /**
    * Flags controlling the behavior of the extractor. Possible flag values are {@link
    * #FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME}, {@link #FLAG_WORKAROUND_IGNORE_TFDT_BOX},
@@ -89,7 +103,8 @@ public class FragmentedMp4Extractor implements Extractor {
         FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME,
         FLAG_WORKAROUND_IGNORE_TFDT_BOX,
         FLAG_ENABLE_EMSG_TRACK,
-        FLAG_WORKAROUND_IGNORE_EDIT_LISTS
+        FLAG_WORKAROUND_IGNORE_EDIT_LISTS,
+        FLAG_EMIT_RAW_SUBTITLE_DATA
       })
   public @interface Flags {}
 
@@ -114,6 +129,12 @@ public class FragmentedMp4Extractor implements Extractor {
   /** Flag to ignore any edit lists in the stream. */
   public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 1 << 4; // 16
 
+  /**
+   * Flag to use the source subtitle formats without modification. If unset, subtitles will be
+   * transcoded to {@link MimeTypes#APPLICATION_MEDIA3_CUES} during extraction.
+   */
+  public static final int FLAG_EMIT_RAW_SUBTITLE_DATA = 1 << 5; // 32
+
   private static final String TAG = "FragmentedMp4Extractor";
 
   @SuppressWarnings("ConstantCaseForConstants")
@@ -134,7 +155,7 @@ public class FragmentedMp4Extractor implements Extractor {
   private static final int STATE_READING_SAMPLE_START = 3;
   private static final int STATE_READING_SAMPLE_CONTINUE = 4;
 
-  // Workarounds.
+  private final SubtitleParser.Factory subtitleParserFactory;
   private final @Flags int flags;
   @Nullable private final Track sideloadedTrack;
 
@@ -187,53 +208,113 @@ public class FragmentedMp4Extractor implements Extractor {
   // Whether extractorOutput.seekMap has been called.
   private boolean haveOutputSeekMap;
 
+  /**
+   * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory)} instead
+   */
+  @Deprecated
   public FragmentedMp4Extractor() {
-    this(0);
+    this(
+        SubtitleParser.Factory.UNSUPPORTED,
+        /* flags= */ FLAG_EMIT_RAW_SUBTITLE_DATA,
+        /* timestampAdjuster= */ null,
+        /* sideloadedTrack= */ null,
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
   }
 
   /**
-   * @param flags Flags that control the extractor's behavior.
+   * Constructs an instance.
+   *
+   * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
+   *     extraction.
    */
+  public FragmentedMp4Extractor(SubtitleParser.Factory subtitleParserFactory) {
+    this(
+        subtitleParserFactory,
+        /* flags= */ 0,
+        /* timestampAdjuster= */ null,
+        /* sideloadedTrack= */ null,
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
+  }
+
+  /**
+   * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int)} instead
+   */
+  @Deprecated
   public FragmentedMp4Extractor(@Flags int flags) {
-    this(flags, /* timestampAdjuster= */ null);
+    this(
+        SubtitleParser.Factory.UNSUPPORTED,
+        flags | FLAG_EMIT_RAW_SUBTITLE_DATA,
+        /* timestampAdjuster= */ null,
+        /* sideloadedTrack= */ null,
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
   }
 
   /**
+   * Constructs an instance.
+   *
+   * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
+   *     extraction.
    * @param flags Flags that control the extractor's behavior.
-   * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed.
    */
+  public FragmentedMp4Extractor(SubtitleParser.Factory subtitleParserFactory, @Flags int flags) {
+    this(
+        subtitleParserFactory,
+        flags,
+        /* timestampAdjuster= */ null,
+        /* sideloadedTrack= */ null,
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
+  }
+
+  /**
+   * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int, TimestampAdjuster,
+   *     Track, List, TrackOutput)} instead
+   */
+  @Deprecated
   public FragmentedMp4Extractor(@Flags int flags, @Nullable TimestampAdjuster timestampAdjuster) {
-    this(flags, timestampAdjuster, /* sideloadedTrack= */ null, Collections.emptyList());
+    this(
+        SubtitleParser.Factory.UNSUPPORTED,
+        flags | FLAG_EMIT_RAW_SUBTITLE_DATA,
+        timestampAdjuster,
+        /* sideloadedTrack= */ null,
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
   }
 
   /**
-   * @param flags Flags that control the extractor's behavior.
+   * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int, TimestampAdjuster,
-   * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed.
+   *     Track, List, TrackOutput)} instead
-   * @param sideloadedTrack Sideloaded track information, in the case that the extractor will not
-   *     receive a moov box in the input data. Null if a moov box is expected.
    */
+  @Deprecated
   public FragmentedMp4Extractor(
       @Flags int flags,
       @Nullable TimestampAdjuster timestampAdjuster,
       @Nullable Track sideloadedTrack) {
-    this(flags, timestampAdjuster, sideloadedTrack, Collections.emptyList());
+    this(
+        SubtitleParser.Factory.UNSUPPORTED,
+        flags | FLAG_EMIT_RAW_SUBTITLE_DATA,
+        timestampAdjuster,
+        sideloadedTrack,
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
   }
 
   /**
-   * @param flags Flags that control the extractor's behavior.
+   * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int, TimestampAdjuster,
-   * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed.
+   *     Track, List, TrackOutput)} instead
-   * @param sideloadedTrack Sideloaded track information, in the case that the extractor will not
-   *     receive a moov box in the input data. Null if a moov box is expected.
-   * @param closedCaptionFormats For tracks that contain SEI messages, the formats of the closed
-   *     caption channels to expose.
    */
+  @Deprecated
   public FragmentedMp4Extractor(
       @Flags int flags,
       @Nullable TimestampAdjuster timestampAdjuster,
       @Nullable Track sideloadedTrack,
       List<Format> closedCaptionFormats) {
     this(
-        flags,
+        SubtitleParser.Factory.UNSUPPORTED,
+        flags | FLAG_EMIT_RAW_SUBTITLE_DATA,
         timestampAdjuster,
         sideloadedTrack,
         closedCaptionFormats,
@@ -241,6 +322,30 @@ public class FragmentedMp4Extractor implements Extractor {
   }
 
   /**
+   * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int, TimestampAdjuster,
+   *     Track, List, TrackOutput)} instead
+   */
+  @Deprecated
+  public FragmentedMp4Extractor(
+      @Flags int flags,
+      @Nullable TimestampAdjuster timestampAdjuster,
+      @Nullable Track sideloadedTrack,
+      List<Format> closedCaptionFormats,
+      @Nullable TrackOutput additionalEmsgTrackOutput) {
+    this(
+        SubtitleParser.Factory.UNSUPPORTED,
+        flags | FLAG_EMIT_RAW_SUBTITLE_DATA,
+        timestampAdjuster,
+        sideloadedTrack,
+        closedCaptionFormats,
+        additionalEmsgTrackOutput);
+  }
+
+  /**
+   * Constructs an instance.
+   *
+   * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
+   *     extraction.
    * @param flags Flags that control the extractor's behavior.
    * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed.
    * @param sideloadedTrack Sideloaded track information, in the case that the extractor will not
@@ -252,11 +357,13 @@ public class FragmentedMp4Extractor implements Extractor {
    *     handling of emsg messages for players is not required.
    */
   public FragmentedMp4Extractor(
+      SubtitleParser.Factory subtitleParserFactory,
       @Flags int flags,
       @Nullable TimestampAdjuster timestampAdjuster,
       @Nullable Track sideloadedTrack,
       List<Format> closedCaptionFormats,
       @Nullable TrackOutput additionalEmsgTrackOutput) {
+    this.subtitleParserFactory = subtitleParserFactory;
     this.flags = flags;
     this.timestampAdjuster = timestampAdjuster;
     this.sideloadedTrack = sideloadedTrack;
@@ -287,7 +394,10 @@ public class FragmentedMp4Extractor implements Extractor {
 
   @Override
   public void init(ExtractorOutput output) {
-    extractorOutput = output;
+    extractorOutput =
+        (flags & FLAG_EMIT_RAW_SUBTITLE_DATA) == 0
+            ? new SubtitleTranscodingExtractorOutput(output, subtitleParserFactory)
+            : output;
     enterReadingAtomHeaderState();
     initExtraTracks();
     if (sideloadedTrack != null) {

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java

@@ -37,6 +37,26 @@ public interface SubtitleParser {
   /** Factory for {@link SubtitleParser} instances. */
   interface Factory {
 
+    /** A subtitle parser factory that supports no formats. */
+    public static final Factory UNSUPPORTED =
+        new Factory() {
+          @Override
+          public boolean supportsFormat(Format format) {
+            return false;
+          }
+
+          @Override
+          public @CueReplacementBehavior int getCueReplacementBehavior(Format format) {
+            return Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
+          }
+
+          @Override
+          public SubtitleParser create(Format format) {
+            throw new IllegalStateException(
+                "This SubtitleParser.Factory doesn't support any formats.");
+          }
+        };
+
     /**
      * Returns whether the factory is able to instantiate a {@link SubtitleParser} for the given
      * {@link Format}.

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractor.java

@@ -44,6 +44,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
  * SubtitleParser.Factory#supportsFormat(Format)} on the {@link SubtitleParser.Factory} passed to
  * the constructor of this class.
  */
+// TODO: b/318679808 - deprecate when all subtitle-related Extractors use
+// SubtitleTranscodingExtractorOutput instead.
 @UnstableApi
 public class SubtitleTranscodingExtractor implements Extractor {
 

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractorOutput.java

@@ -18,12 +18,29 @@ package androidx.media3.extractor.text;
 
 import android.util.SparseArray;
 import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.UnstableApi;
 import androidx.media3.extractor.ExtractorOutput;
 import androidx.media3.extractor.SeekMap;
 import androidx.media3.extractor.TrackOutput;
 
-/** A wrapping {@link ExtractorOutput} for use by {@link SubtitleTranscodingExtractor}. */
+/**
-/* package */ class SubtitleTranscodingExtractorOutput implements ExtractorOutput {
+ * A wrapping {@link ExtractorOutput} that transcodes {@linkplain C#TRACK_TYPE_TEXT text samples}
+ * from supported subtitle formats to {@link MimeTypes#APPLICATION_MEDIA3_CUES}.
+ *
+ * <p>The resulting {@link MimeTypes#APPLICATION_MEDIA3_CUES} samples are emitted to the underlying
+ * {@link TrackOutput}.
+ *
+ * <p>For non-text tracks (i.e. where {@link C.TrackType} is not {@code C.TRACK_TYPE_TEXT}), samples
+ * are passed through to the underlying {@link TrackOutput} without modification.
+ *
+ * <p>Support for subtitle formats is determined by {@link
+ * SubtitleParser.Factory#supportsFormat(Format)} on the {@link SubtitleParser.Factory} passed to
+ * the constructor of this class.
+ */
+@UnstableApi
+public final class SubtitleTranscodingExtractorOutput implements ExtractorOutput {
 
   private final ExtractorOutput delegate;
   private final SubtitleParser.Factory subtitleParserFactory;

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java

@@ -38,7 +38,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
  * MimeTypes#APPLICATION_SUBRIP} to ExoPlayer's internal binary cue representation ({@link
  * MimeTypes#APPLICATION_MEDIA3_CUES}).
  */
-/* package */ class SubtitleTranscodingTrackOutput implements TrackOutput {
+/* package */ final class SubtitleTranscodingTrackOutput implements TrackOutput {
 
   private final TrackOutput delegate;
   private final SubtitleParser.Factory subtitleParserFactory;

libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java

@@ -178,7 +178,7 @@ public final class DefaultExtractorsFactoryTest {
     assertThat(aviExtractor).isInstanceOf(SubtitleTranscodingExtractor.class);
     assertThat(matroskaExtractor).isInstanceOf(SubtitleTranscodingExtractor.class);
     assertThat(mp4Extractor).isInstanceOf(SubtitleTranscodingExtractor.class);
-    assertThat(fragmentedMp4Extractor).isInstanceOf(SubtitleTranscodingExtractor.class);
+    assertThat(fragmentedMp4Extractor).isNotInstanceOf(SubtitleTranscodingExtractor.class);
     assertThat(tsExtractor).isInstanceOf(SubtitleTranscodingExtractor.class);
   }
 

libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNoSniffingTest.java

@@ -15,10 +15,15 @@
  */
 package androidx.media3.extractor.mp4;
 
+import static androidx.media3.extractor.mp4.FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA;
+
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.MimeTypes;
+import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
+import androidx.media3.extractor.text.SubtitleParser;
 import androidx.media3.test.utils.ExtractorAsserts;
+import com.google.common.collect.ImmutableList;
 import java.util.List;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -32,6 +37,21 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
 @RunWith(ParameterizedRobolectricTestRunner.class)
 public class FragmentedMp4ExtractorNoSniffingTest {
 
+  private static final String FMP4_SIDELOADED = "media/mp4/sample_fragmented_sideloaded_track.mp4";
+  private static final Track SIDELOADED_TRACK =
+      new Track(
+          /* id= */ 1,
+          /* type= */ C.TRACK_TYPE_VIDEO,
+          /* timescale= */ 30_000,
+          /* movieTimescale= */ 1000,
+          /* durationUs= */ C.TIME_UNSET,
+          new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H264).build(),
+          /* sampleTransformation= */ Track.TRANSFORMATION_NONE,
+          /* sampleDescriptionEncryptionBoxes= */ null,
+          /* nalUnitLengthFieldLength= */ 4,
+          /* editListDurations= */ null,
+          /* editListMediaTimes= */ null);
+
   @Parameters(name = "{0}")
   public static List<Object[]> params() {
     return ExtractorAsserts.configsNoSniffing();
@@ -40,27 +60,35 @@ public class FragmentedMp4ExtractorNoSniffingTest {
   @Parameter public ExtractorAsserts.SimulationConfig simulationConfig;
 
   @Test
-  public void sampleWithSideLoadedTrack() throws Exception {
+  public void sampleWithSideLoadedTrack_subtitlesParsedDuringDecoding() throws Exception {
     // Sideloaded tracks are generally used in Smooth Streaming, where the MP4 files do not contain
     // any ftyp box and are not sniffed.
-    Track sideloadedTrack =
-        new Track(
-            /* id= */ 1,
-            /* type= */ C.TRACK_TYPE_VIDEO,
-            /* timescale= */ 30_000,
-            /* movieTimescale= */ 1000,
-            /* durationUs= */ C.TIME_UNSET,
-            new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H264).build(),
-            /* sampleTransformation= */ Track.TRANSFORMATION_NONE,
-            /* sampleDescriptionEncryptionBoxes= */ null,
-            /* nalUnitLengthFieldLength= */ 4,
-            /* editListDurations= */ null,
-            /* editListMediaTimes= */ null);
     ExtractorAsserts.assertBehavior(
         () ->
-            new FragmentedMp4Extractor(
+            createFragmentedMp4Extractor(
-                /* flags= */ 0, /* timestampAdjuster= */ null, sideloadedTrack),
+                SubtitleParser.Factory.UNSUPPORTED, FLAG_EMIT_RAW_SUBTITLE_DATA),
-        "media/mp4/sample_fragmented_sideloaded_track.mp4",
+        FMP4_SIDELOADED,
         simulationConfig);
   }
+
+  @Test
+  public void sampleWithSideLoadedTrack_subtitlesParsedDuringExtraction() throws Exception {
+    // Sideloaded tracks are generally used in Smooth Streaming, where the MP4 files do not contain
+    // any ftyp box and are not sniffed.
+    ExtractorAsserts.assertBehavior(
+        () -> createFragmentedMp4Extractor(new DefaultSubtitleParserFactory(), /* flags= */ 0),
+        FMP4_SIDELOADED,
+        simulationConfig);
+  }
+
+  private FragmentedMp4Extractor createFragmentedMp4Extractor(
+      SubtitleParser.Factory subtitleParserFactory, @FragmentedMp4Extractor.Flags int flags) {
+    return new FragmentedMp4Extractor(
+        subtitleParserFactory,
+        flags,
+        /* timestampAdjuster= */ null,
+        SIDELOADED_TRACK,
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
+  }
 }

libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorTest.java

@@ -15,11 +15,16 @@
  */
 package androidx.media3.extractor.mp4;
 
+import static androidx.media3.extractor.mp4.FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA;
+
 import androidx.media3.common.Format;
 import androidx.media3.common.MimeTypes;
+import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
+import androidx.media3.extractor.text.SubtitleParser;
 import androidx.media3.test.utils.ExtractorAsserts;
 import androidx.media3.test.utils.ExtractorAsserts.ExtractorFactory;
 import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import org.junit.Test;
@@ -32,17 +37,27 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
 @RunWith(ParameterizedRobolectricTestRunner.class)
 public final class FragmentedMp4ExtractorTest {
 
-  @Parameters(name = "{0}")
+  @Parameters(name = "{0},subtitlesParsedDuringExtraction={1}")
-  public static ImmutableList<ExtractorAsserts.SimulationConfig> params() {
+  public static List<Object[]> params() {
-    return ExtractorAsserts.configs();
+    List<Object[]> parameterList = new ArrayList<>();
+    for (ExtractorAsserts.SimulationConfig config : ExtractorAsserts.configs()) {
+      parameterList.add(new Object[] {config, /* subtitlesParsedDuringExtraction */ true});
+      parameterList.add(new Object[] {config, /* subtitlesParsedDuringExtraction */ false});
+    }
+    return parameterList;
   }
 
-  @Parameter public ExtractorAsserts.SimulationConfig simulationConfig;
+  @Parameter(0)
+  public ExtractorAsserts.SimulationConfig simulationConfig;
+
+  @Parameter(1)
+  public boolean subtitlesParsedDuringExtraction;
 
   @Test
   public void sample() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_fragmented.mp4",
         simulationConfig);
   }
@@ -50,7 +65,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleSeekable() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_fragmented_seekable.mp4",
         simulationConfig);
   }
@@ -58,18 +74,21 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithSeiPayloadParsing() throws Exception {
     // Enabling the CEA-608 track enables SEI payload parsing.
-    ExtractorFactory extractorFactory =
+    List<Format> closedCaptions =
-        getExtractorFactory(
+        Collections.singletonList(
-            Collections.singletonList(
+            new Format.Builder().setSampleMimeType(MimeTypes.APPLICATION_CEA608).build());
-                new Format.Builder().setSampleMimeType(MimeTypes.APPLICATION_CEA608).build()));
+
     ExtractorAsserts.assertBehavior(
-        extractorFactory, "media/mp4/sample_fragmented_sei.mp4", simulationConfig);
+        getExtractorFactory(closedCaptions, subtitlesParsedDuringExtraction),
+        "media/mp4/sample_fragmented_sei.mp4",
+        simulationConfig);
   }
 
   @Test
   public void sampleWithAc3Track() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_ac3_fragmented.mp4",
         simulationConfig);
   }
@@ -77,7 +96,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithAc4Track() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_ac4_fragmented.mp4",
         simulationConfig);
   }
@@ -85,7 +105,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithProtectedAc4Track() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_ac4_protected.mp4",
         simulationConfig);
   }
@@ -93,7 +114,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithEac3Track() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_eac3_fragmented.mp4",
         simulationConfig);
   }
@@ -101,7 +123,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithEac3jocTrack() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_eac3joc_fragmented.mp4",
         simulationConfig);
   }
@@ -109,7 +132,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithOpusTrack() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_opus_fragmented.mp4",
         simulationConfig);
   }
@@ -117,7 +141,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void samplePartiallyFragmented() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_partially_fragmented.mp4",
         simulationConfig);
   }
@@ -126,7 +151,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithLargeBitrates() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_fragmented_large_bitrates.mp4",
         simulationConfig);
   }
@@ -134,7 +160,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithMhm1BlCicp1Track() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_mhm1_bl_cicp1_fragmented.mp4",
         simulationConfig);
   }
@@ -142,7 +169,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithMhm1LcblCicp1Track() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_mhm1_lcbl_cicp1_fragmented.mp4",
         simulationConfig);
   }
@@ -150,7 +178,8 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithMhm1BlConfigChangeTrack() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_mhm1_bl_configchange_fragmented.mp4",
         simulationConfig);
   }
@@ -158,17 +187,31 @@ public final class FragmentedMp4ExtractorTest {
   @Test
   public void sampleWithMhm1LcblConfigChangeTrack() throws Exception {
     ExtractorAsserts.assertBehavior(
-        getExtractorFactory(ImmutableList.of()),
+        getExtractorFactory(
+            /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction),
         "media/mp4/sample_mhm1_lcbl_configchange_fragmented.mp4",
         simulationConfig);
   }
 
-  private static ExtractorFactory getExtractorFactory(final List<Format> closedCaptionFormats) {
+  private static ExtractorFactory getExtractorFactory(
+      List<Format> closedCaptionFormats, boolean subtitlesParsedDuringExtraction) {
+    SubtitleParser.Factory subtitleParserFactory;
+    @FragmentedMp4Extractor.Flags int flags;
+    if (subtitlesParsedDuringExtraction) {
+      subtitleParserFactory = new DefaultSubtitleParserFactory();
+      flags = 0;
+    } else {
+      subtitleParserFactory = SubtitleParser.Factory.UNSUPPORTED;
+      flags = FLAG_EMIT_RAW_SUBTITLE_DATA;
+    }
+
     return () ->
         new FragmentedMp4Extractor(
-            /* flags= */ 0,
+            subtitleParserFactory,
+            flags,
             /* timestampAdjuster= */ null,
             /* sideloadedTrack= */ null,
-            closedCaptionFormats);
+            closedCaptionFormats,
+            /* additionalEmsgTrackOutput= */ null);
   }
 }