Add experimental opt-in to parse SS subtitles during extraction

PiperOrigin-RevId: 586331888

RELEASENOTES.md

@@ -92,6 +92,9 @@
     *   Parse "f800" as channel count of 5 for Dolby in DASH manifest
         ([#688](https://github.com/androidx/media/issues/688)).
 *   Smooth Streaming Extension:
+    *   Add experimental support for parsing subtitles during extraction. You
+        can enable this using
+        `SsMediaSource.Factory.experimentalParseSubtitlesDuringExtraction()`.
 *   RTSP Extension:
 *   Decoder Extensions (FFmpeg, VP9, AV1, MIDI, etc.):
     *   MIDI decoder: Ignore SysEx event messages

libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java

@@ -47,9 +47,12 @@ import androidx.media3.exoplayer.upstream.CmcdData;
 import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
 import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy.FallbackSelection;
 import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
+import androidx.media3.extractor.Extractor;
 import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
 import androidx.media3.extractor.mp4.Track;
 import androidx.media3.extractor.mp4.TrackEncryptionBox;
+import androidx.media3.extractor.text.SubtitleParser;
+import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
 import java.io.IOException;
 import java.util.List;
 
@@ -60,11 +63,24 @@ public class DefaultSsChunkSource implements SsChunkSource {
   public static final class Factory implements SsChunkSource.Factory {
 
     private final DataSource.Factory dataSourceFactory;
+    @Nullable private SubtitleParser.Factory subtitleParserFactory;
 
     public Factory(DataSource.Factory dataSourceFactory) {
       this.dataSourceFactory = dataSourceFactory;
     }
 
+    /**
+     * Sets the {@link SubtitleParser.Factory} to be used for parsing subtitles during extraction,
+     * or {@code null} to parse subtitles during decoding.
+     *
+     * <p>This may only be used with {@link BundledChunkExtractor.Factory}.
+     */
+    /* package */ Factory setSubtitleParserFactory(
+        @Nullable SubtitleParser.Factory subtitleParserFactory) {
+      this.subtitleParserFactory = subtitleParserFactory;
+      return this;
+    }
+
     @Override
     public SsChunkSource createChunkSource(
         LoaderErrorThrower manifestLoaderErrorThrower,
@@ -83,7 +99,8 @@ public class DefaultSsChunkSource implements SsChunkSource {
           streamElementIndex,
           trackSelection,
           dataSource,
-          cmcdConfiguration);
+          cmcdConfiguration,
+          subtitleParserFactory);
     }
   }
 
@@ -112,6 +129,8 @@ public class DefaultSsChunkSource implements SsChunkSource {
    * @param trackSelection The track selection.
    * @param dataSource A {@link DataSource} suitable for loading the media data.
    * @param cmcdConfiguration The {@link CmcdConfiguration} for this chunk source.
+   * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
+   *     extraction.
    */
   public DefaultSsChunkSource(
       LoaderErrorThrower manifestLoaderErrorThrower,
@@ -119,7 +138,8 @@ public class DefaultSsChunkSource implements SsChunkSource {
       int streamElementIndex,
       ExoTrackSelection trackSelection,
       DataSource dataSource,
-      @Nullable CmcdConfiguration cmcdConfiguration) {
+      @Nullable CmcdConfiguration cmcdConfiguration,
+      @Nullable SubtitleParser.Factory subtitleParserFactory) {
     this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
     this.manifest = manifest;
     this.streamElementIndex = streamElementIndex;
@@ -152,12 +172,15 @@ public class DefaultSsChunkSource implements SsChunkSource {
               nalUnitLengthFieldLength,
               null,
               null);
-      FragmentedMp4Extractor extractor =
+      Extractor extractor =
           new FragmentedMp4Extractor(
               FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME
                   | FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX,
               /* timestampAdjuster= */ null,
               track);
+      if (subtitleParserFactory != null) {
+        extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
+      }
       chunkExtractors[i] = new BundledChunkExtractor(extractor, streamElement.type, format);
     }
   }

libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java

@@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
 import androidx.media3.common.StreamKey;
 import androidx.media3.common.TrackGroup;
 import androidx.media3.common.util.NullableType;
@@ -41,6 +42,7 @@ import androidx.media3.exoplayer.upstream.Allocator;
 import androidx.media3.exoplayer.upstream.CmcdConfiguration;
 import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
 import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
+import androidx.media3.extractor.text.SubtitleParser;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -77,7 +79,8 @@ import java.util.List;
       LoadErrorHandlingPolicy loadErrorHandlingPolicy,
       MediaSourceEventListener.EventDispatcher mediaSourceEventDispatcher,
       LoaderErrorThrower manifestLoaderErrorThrower,
-      Allocator allocator) {
+      Allocator allocator,
+      @Nullable SubtitleParser.Factory subtitleParserFactory) {
     this.manifest = manifest;
     this.chunkSourceFactory = chunkSourceFactory;
     this.transferListener = transferListener;
@@ -89,7 +92,7 @@ import java.util.List;
     this.mediaSourceEventDispatcher = mediaSourceEventDispatcher;
     this.allocator = allocator;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
-    trackGroups = buildTrackGroups(manifest, drmSessionManager);
+    trackGroups = buildTrackGroups(manifest, drmSessionManager, subtitleParserFactory);
     sampleStreams = newSampleStreamArray(0);
     compositeSequenceableLoader =
         compositeSequenceableLoaderFactory.createCompositeSequenceableLoader(sampleStreams);
@@ -265,15 +268,32 @@ import java.util.List;
   }
 
   private static TrackGroupArray buildTrackGroups(
-      SsManifest manifest, DrmSessionManager drmSessionManager) {
+      SsManifest manifest,
+      DrmSessionManager drmSessionManager,
+      @Nullable SubtitleParser.Factory subtitleParserFactory) {
     TrackGroup[] trackGroups = new TrackGroup[manifest.streamElements.length];
     for (int i = 0; i < manifest.streamElements.length; i++) {
       Format[] manifestFormats = manifest.streamElements[i].formats;
       Format[] exposedFormats = new Format[manifestFormats.length];
       for (int j = 0; j < manifestFormats.length; j++) {
         Format manifestFormat = manifestFormats[j];
-        exposedFormats[j] =
+        Format.Builder updatedFormat =
-            manifestFormat.copyWithCryptoType(drmSessionManager.getCryptoType(manifestFormat));
+            manifestFormat
+                .buildUpon()
+                .setCryptoType(drmSessionManager.getCryptoType(manifestFormat));
+        if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(manifestFormat)) {
+          updatedFormat
+              .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
+              .setCueReplacementBehavior(
+                  subtitleParserFactory.getCueReplacementBehavior(manifestFormat))
+              .setCodecs(
+                  manifestFormat.sampleMimeType
+                      + (manifestFormat.codecs != null ? " " + manifestFormat.codecs : ""))
+              // Reset this value to the default. All non-default timestamp adjustments are done
+              // by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding.
+              .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE);
+        }
+        exposedFormats[j] = updatedFormat.build();
       }
       trackGroups[i] = new TrackGroup(/* id= */ Integer.toString(i), exposedFormats);
     }

libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java

@@ -65,6 +65,8 @@ import androidx.media3.exoplayer.upstream.Loader;
 import androidx.media3.exoplayer.upstream.Loader.LoadErrorAction;
 import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
 import androidx.media3.exoplayer.upstream.ParsingLoadable;
+import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
+import androidx.media3.extractor.text.SubtitleParser;
 import com.google.common.collect.ImmutableList;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.io.IOException;
@@ -91,6 +93,7 @@ public final class SsMediaSource extends BaseMediaSource
     @Nullable private CmcdConfiguration.Factory cmcdConfigurationFactory;
     private DrmSessionManagerProvider drmSessionManagerProvider;
     private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
+    @Nullable private SubtitleParser.Factory subtitleParserFactory;
     private long livePresentationDelayMs;
     @Nullable private ParsingLoadable.Parser<? extends SsManifest> manifestParser;
 
@@ -152,6 +155,40 @@ public final class SsMediaSource extends BaseMediaSource
       return this;
     }
 
+    /**
+     * Sets whether subtitles should be parsed as part of extraction (before being added to the
+     * sample queue) or as part of rendering (when being taken from the sample queue). Defaults to
+     * {@code false} (i.e. subtitles will be parsed as part of rendering).
+     *
+     * <p>This method is experimental. Its default value may change, or it may be renamed or removed
+     * in a future release.
+     *
+     * <p>This method may only be used with {@link DefaultSsChunkSource.Factory}.
+     *
+     * @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or
+     *     rendering.
+     * @return This factory, for convenience.
+     */
+    // TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on
+    //  DefaultMediaSourceFactory via the MediaSource.Factory interface).
+    public Factory experimentalParseSubtitlesDuringExtraction(
+        boolean parseSubtitlesDuringExtraction) {
+      if (parseSubtitlesDuringExtraction) {
+        if (subtitleParserFactory == null) {
+          this.subtitleParserFactory = new DefaultSubtitleParserFactory();
+        }
+      } else {
+        this.subtitleParserFactory = null;
+      }
+      if (chunkSourceFactory instanceof DefaultSsChunkSource.Factory) {
+        ((DefaultSsChunkSource.Factory) chunkSourceFactory)
+            .setSubtitleParserFactory(subtitleParserFactory);
+      } else {
+        throw new IllegalStateException();
+      }
+      return this;
+    }
+
     /**
      * Sets the duration in milliseconds by which the default start position should precede the end
      * of the live window for live playbacks. The default value is {@link
@@ -273,6 +310,7 @@ public final class SsMediaSource extends BaseMediaSource
           cmcdConfiguration,
           drmSessionManagerProvider.get(mediaItem),
           loadErrorHandlingPolicy,
+          subtitleParserFactory,
           livePresentationDelayMs);
     }
 
@@ -310,6 +348,7 @@ public final class SsMediaSource extends BaseMediaSource
           cmcdConfiguration,
           drmSessionManagerProvider.get(mediaItem),
           loadErrorHandlingPolicy,
+          subtitleParserFactory,
           livePresentationDelayMs);
     }
 
@@ -353,6 +392,7 @@ public final class SsMediaSource extends BaseMediaSource
   private long manifestLoadStartTimestamp;
   private SsManifest manifest;
   private Handler manifestRefreshHandler;
+  @Nullable private final SubtitleParser.Factory subtitleParserFactory;
 
   @GuardedBy("this")
   private MediaItem mediaItem;
@@ -367,6 +407,7 @@ public final class SsMediaSource extends BaseMediaSource
       @Nullable CmcdConfiguration cmcdConfiguration,
       DrmSessionManager drmSessionManager,
       LoadErrorHandlingPolicy loadErrorHandlingPolicy,
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       long livePresentationDelayMs) {
     Assertions.checkState(manifest == null || !manifest.isLive);
     this.mediaItem = mediaItem;
@@ -383,6 +424,7 @@ public final class SsMediaSource extends BaseMediaSource
     this.cmcdConfiguration = cmcdConfiguration;
     this.drmSessionManager = drmSessionManager;
     this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
+    this.subtitleParserFactory = subtitleParserFactory;
     this.livePresentationDelayMs = livePresentationDelayMs;
     this.manifestEventDispatcher = createEventDispatcher(/* mediaPeriodId= */ null);
     sideloadedManifest = manifest != null;
@@ -450,7 +492,8 @@ public final class SsMediaSource extends BaseMediaSource
             loadErrorHandlingPolicy,
             mediaSourceEventDispatcher,
             manifestLoaderErrorThrower,
-            allocator);
+            allocator,
+            subtitleParserFactory);
     mediaPeriods.add(period);
     return period;
   }

libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSourceTest.java

@@ -297,6 +297,7 @@ public class DefaultSsChunkSourceTest {
         /* streamElementIndex= */ 0,
         adaptiveTrackSelection,
         new FakeDataSource(),
-        cmcdConfiguration);
+        cmcdConfiguration,
+        /* subtitleParserFactory= */ null);
   }
 }

libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java

@@ -17,6 +17,7 @@ package androidx.media3.exoplayer.smoothstreaming;
 
 import static androidx.media3.exoplayer.smoothstreaming.SsTestUtils.createSsManifest;
 import static androidx.media3.exoplayer.smoothstreaming.SsTestUtils.createStreamElement;
+import static com.google.common.truth.Truth.assertThat;
 import static org.mockito.Mockito.mock;
 
 import androidx.media3.common.C;
@@ -32,6 +33,9 @@ import androidx.media3.exoplayer.source.MediaSourceEventListener;
 import androidx.media3.exoplayer.upstream.Allocator;
 import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
 import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
+import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
+import androidx.media3.extractor.text.SubtitleParser;
+import androidx.media3.test.utils.FakeTimeline;
 import androidx.media3.test.utils.MediaPeriodAsserts;
 import androidx.media3.test.utils.MediaPeriodAsserts.FilterableManifestMediaPeriodFactory;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
@@ -75,13 +79,75 @@ public class SsMediaPeriodTest {
               new MediaSourceEventListener.EventDispatcher()
                   .withParameters(/* windowIndex= */ 0, mediaPeriodId),
               mock(LoaderErrorThrower.class),
-              mock(Allocator.class));
+              mock(Allocator.class),
+              /* subtitleParserFactory= */ null);
         };
 
     MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration(
         mediaPeriodFactory, testManifest);
   }
 
+  @Test
+  public void getTrackGroups_withSubtitleParserFactory_matchesFormat() {
+    SubtitleParser.Factory subtitleParserFactory = new DefaultSubtitleParserFactory();
+
+    Format originalSubtitleFormat =
+        new Format.Builder()
+            .setContainerMimeType(MimeTypes.APPLICATION_MP4)
+            .setSampleMimeType(MimeTypes.TEXT_VTT)
+            .setLanguage("eng")
+            .build();
+    Format expectedSubtitleFormat =
+        new Format.Builder()
+            .setContainerMimeType(originalSubtitleFormat.containerMimeType)
+            .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
+            .setCodecs(originalSubtitleFormat.sampleMimeType)
+            .setCueReplacementBehavior(
+                subtitleParserFactory.getCueReplacementBehavior(originalSubtitleFormat))
+            .setLanguage(originalSubtitleFormat.language)
+            .build();
+
+    SsManifest testManifest =
+        createSsManifest(
+            createStreamElement(
+                /* name= */ "video",
+                C.TRACK_TYPE_VIDEO,
+                createVideoFormat(/* bitrate= */ 200000),
+                createVideoFormat(/* bitrate= */ 400000),
+                createVideoFormat(/* bitrate= */ 800000)),
+            createStreamElement(
+                /* name= */ "audio",
+                C.TRACK_TYPE_AUDIO,
+                createAudioFormat(/* bitrate= */ 48000),
+                createAudioFormat(/* bitrate= */ 96000)),
+            createStreamElement(/* name= */ "text", C.TRACK_TYPE_TEXT, originalSubtitleFormat));
+
+    MediaPeriodId mediaPeriodId =
+        new MediaPeriodId(
+            new FakeTimeline(/* windowCount= */ 2).getUidOfPeriod(/* periodIndex= */ 0),
+            /* windowSequenceNumber= */ 0);
+
+    SsMediaPeriod period =
+        new SsMediaPeriod(
+            testManifest,
+            mock(SsChunkSource.Factory.class),
+            mock(TransferListener.class),
+            mock(CompositeSequenceableLoaderFactory.class),
+            /* cmcdConfiguration= */ null,
+            mock(DrmSessionManager.class),
+            new DrmSessionEventListener.EventDispatcher()
+                .withParameters(/* windowIndex= */ 0, mediaPeriodId),
+            mock(LoadErrorHandlingPolicy.class),
+            new MediaSourceEventListener.EventDispatcher()
+                .withParameters(/* windowIndex= */ 0, mediaPeriodId),
+            mock(LoaderErrorThrower.class),
+            mock(Allocator.class),
+            subtitleParserFactory);
+
+    Format subtitleFormat = period.getTrackGroups().get(2).getFormat(0);
+    assertThat(subtitleFormat).isEqualTo(expectedSubtitleFormat);
+  }
+
   private static Format createVideoFormat(int bitrate) {
     return new Format.Builder()
         .setContainerMimeType(MimeTypes.VIDEO_MP4)

libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSourceTest.java

@@ -16,16 +16,24 @@
 package androidx.media3.exoplayer.smoothstreaming;
 
 import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.fail;
 
+import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.StreamKey;
 import androidx.media3.common.Timeline;
 import androidx.media3.datasource.ByteArrayDataSource;
 import androidx.media3.datasource.DataSource;
+import androidx.media3.datasource.TransferListener;
 import androidx.media3.exoplayer.analytics.PlayerId;
+import androidx.media3.exoplayer.smoothstreaming.manifest.SsManifest;
 import androidx.media3.exoplayer.source.MediaSource;
+import androidx.media3.exoplayer.trackselection.ExoTrackSelection;
+import androidx.media3.exoplayer.upstream.CmcdConfiguration;
+import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
+import androidx.media3.test.utils.FakeDataSource;
 import androidx.media3.test.utils.TestUtil;
 import androidx.media3.test.utils.robolectric.RobolectricUtil;
 import androidx.test.core.app.ApplicationProvider;
@@ -94,6 +102,27 @@ public class SsMediaSourceTest {
     assertThat(canUpdateMediaItem).isFalse();
   }
 
+  @Test
+  public void
+      setExperimentalParseSubtitlesDuringExtraction_withNonDefaultChunkSourceFactory_setThrows() {
+    SsMediaSource.Factory ssMediaSourceFactory =
+        new SsMediaSource.Factory(
+            /* chunkSourceFactory= */ this::createSampleSsChunkSource,
+            /* manifestDataSourceFactory= */ () -> createSampleDataSource(SAMPLE_MANIFEST));
+    assertThrows(
+        IllegalStateException.class,
+        () -> ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(false));
+  }
+
+  @Test
+  public void
+      setExperimentalParseSubtitlesDuringExtraction_withDefaultChunkSourceFactory_setSucceeds() {
+    SsMediaSource.Factory ssMediaSourceFactory =
+        new SsMediaSource.Factory(() -> createSampleDataSource(SAMPLE_MANIFEST));
+    ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(false);
+    ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(true);
+  }
+
   @Test
   public void canUpdateMediaItem_withChangedStreamKeys_returnsFalse() {
     MediaItem initialMediaItem =
@@ -175,4 +204,21 @@ public class SsMediaSourceTest {
     }
     return new ByteArrayDataSource(manifestData);
   }
+
+  private SsChunkSource createSampleSsChunkSource(
+      LoaderErrorThrower manifestLoaderErrorThrower,
+      SsManifest manifest,
+      int streamElementIndex,
+      ExoTrackSelection trackSelection,
+      @Nullable TransferListener transferListener,
+      @Nullable CmcdConfiguration cmcdConfiguration) {
+    return new DefaultSsChunkSource(
+        manifestLoaderErrorThrower,
+        manifest,
+        streamElementIndex,
+        trackSelection,
+        new FakeDataSource(),
+        cmcdConfiguration,
+        /* subtitleParserFactory= */ null);
+  }
 }