Add experimental opt-in to parse DASH subtitles during extraction

This currently only applies to subtitles muxed into mp4 segments, and
not standalone text files linked directly from the manifest.

Issue: androidx/media#288

#minor-release

PiperOrigin-RevId: 572263764
(cherry picked from commit 66fa5919590789b384506a4e604fe02a5a5e0877)

RELEASENOTES.md

@@ -37,6 +37,13 @@ This release includes the following changes since the
         Android Auto.
 *   DASH Extension:
     *   Allow multiple of the same DASH identifier in segment template url.
+    *   Add experimental support for parsing subtitles during extraction. This
+        has better support for merging overlapping subtitles, including
+        resolving flickering when transitioning between subtitle segments. You
+        can enable this using
+        `DashMediaSource.Factory.experimentalParseSubtitlesDuringExtraction()`
+        ([#288](https://github.com/androidx/media/issues/288)).
+*   Smooth Streaming Extension:
 *   RTSP Extension:
     *   Use RTSP Setup Response timeout value in time interval of sending
         keep-alive RTSP Options requests

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java

@@ -26,6 +26,7 @@ import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.Assertions;
 import androidx.media3.common.util.ParsableByteArray;
 import androidx.media3.common.util.UnstableApi;
+import androidx.media3.exoplayer.analytics.PlayerId;
 import androidx.media3.extractor.ChunkIndex;
 import androidx.media3.extractor.DummyTrackOutput;
 import androidx.media3.extractor.Extractor;
@@ -36,7 +37,10 @@ import androidx.media3.extractor.SeekMap;
 import androidx.media3.extractor.TrackOutput;
 import androidx.media3.extractor.mkv.MatroskaExtractor;
 import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
+import androidx.media3.extractor.text.SubtitleParser;
+import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
 import java.io.IOException;
+import java.util.List;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
@@ -46,36 +50,68 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 @UnstableApi
 public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtractor {
 
-  /** {@link ChunkExtractor.Factory} for instances of this class. */
+  /** {@link ChunkExtractor.Factory} for {@link BundledChunkExtractor}. */
-  public static final ChunkExtractor.Factory FACTORY =
+  public static final class Factory implements ChunkExtractor.Factory {
-      (primaryTrackType,
+
-          format,
+    /** Non-null if subtitles should be parsed during extraction, null otherwise. */
-          enableEventMessageTrack,
+    @Nullable private SubtitleParser.Factory subtitleParserFactory;
-          closedCaptionFormats,
+
-          playerEmsgTrackOutput,
+    /**
-          playerId) -> {
+     * Sets the {@link SubtitleParser.Factory} to use for parsing subtitles during extraction, or
-        @Nullable String containerMimeType = format.containerMimeType;
+     * null to parse subtitles during decoding. The default is null (subtitles parsed after
-        Extractor extractor;
+     * decoding).
-        if (MimeTypes.isText(containerMimeType)) {
+     *
-          // Text types do not need an extractor.
+     * <p>This method is experimental. Its default value may change, or it may be renamed or removed
-          return null;
+     * in a future release.
-        } else if (MimeTypes.isMatroska(containerMimeType)) {
+     *
-          extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES);
+     * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
-        } else {
+     *     extraction.
-          int flags = 0;
+     * @return This factory, for convenience.
-          if (enableEventMessageTrack) {
+     */
-            flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
+    public Factory experimentalSetSubtitleParserFactory(
-          }
+        @Nullable SubtitleParser.Factory subtitleParserFactory) {
-          extractor =
+      this.subtitleParserFactory = subtitleParserFactory;
-              new FragmentedMp4Extractor(
+      return this;
-                  flags,
+    }
-                  /* timestampAdjuster= */ null,
+
-                  /* sideloadedTrack= */ null,
+    @Nullable
-                  closedCaptionFormats,
+    @Override
-                  playerEmsgTrackOutput);
+    public ChunkExtractor createProgressiveMediaExtractor(
+        @C.TrackType int primaryTrackType,
+        Format representationFormat,
+        boolean enableEventMessageTrack,
+        List<Format> closedCaptionFormats,
+        @Nullable TrackOutput playerEmsgTrackOutput,
+        PlayerId playerId) {
+      @Nullable String containerMimeType = representationFormat.containerMimeType;
+      Extractor extractor;
+      if (MimeTypes.isText(containerMimeType)) {
+        // Text types do not need an extractor.
+        return null;
+      } else if (MimeTypes.isMatroska(containerMimeType)) {
+        extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES);
+      } else {
+        int flags = 0;
+        if (enableEventMessageTrack) {
+          flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
         }
-        return new BundledChunkExtractor(extractor, primaryTrackType, format);
+        extractor =
-      };
+            new FragmentedMp4Extractor(
+                flags,
+                /* timestampAdjuster= */ null,
+                /* sideloadedTrack= */ null,
+                closedCaptionFormats,
+                playerEmsgTrackOutput);
+      }
+      if (subtitleParserFactory != null) {
+        extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
+      }
+      return new BundledChunkExtractor(extractor, primaryTrackType, representationFormat);
+    }
+  }
+
+  /** {@link Factory} for {@link BundledChunkExtractor}. */
+  public static final Factory FACTORY = new Factory();
 
   private static final PositionHolder POSITION_HOLDER = new PositionHolder();
 

libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java

@@ -58,6 +58,7 @@ import androidx.media3.exoplayer.upstream.Allocator;
 import androidx.media3.exoplayer.upstream.CmcdConfiguration;
 import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
 import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
+import androidx.media3.extractor.text.SubtitleParser;
 import com.google.common.collect.Maps;
 import com.google.common.primitives.Ints;
 import java.io.IOException;
@@ -130,7 +131,8 @@ import java.util.regex.Pattern;
       Allocator allocator,
       CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
       PlayerEmsgCallback playerEmsgCallback,
-      PlayerId playerId) {
+      PlayerId playerId,
+      @Nullable SubtitleParser.Factory subtitleParserFactory) {
     this.id = id;
     this.manifest = manifest;
     this.baseUrlExclusionList = baseUrlExclusionList;
@@ -156,7 +158,8 @@ import java.util.regex.Pattern;
     Period period = manifest.getPeriod(periodIndex);
     eventStreams = period.eventStreams;
     Pair<TrackGroupArray, TrackGroupInfo[]> result =
-        buildTrackGroups(drmSessionManager, period.adaptationSets, eventStreams);
+        buildTrackGroups(
+            drmSessionManager, subtitleParserFactory, period.adaptationSets, eventStreams);
     trackGroups = result.first;
     trackGroupInfos = result.second;
   }
@@ -501,6 +504,7 @@ import java.util.regex.Pattern;
 
   private static Pair<TrackGroupArray, TrackGroupInfo[]> buildTrackGroups(
       DrmSessionManager drmSessionManager,
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       List<AdaptationSet> adaptationSets,
       List<EventStream> eventStreams) {
     int[][] groupedAdaptationSetIndices = getGroupedAdaptationSetIndices(adaptationSets);
@@ -523,6 +527,7 @@ import java.util.regex.Pattern;
     int trackGroupCount =
         buildPrimaryAndEmbeddedTrackGroupInfos(
             drmSessionManager,
+            subtitleParserFactory,
             adaptationSets,
             groupedAdaptationSetIndices,
             primaryGroupCount,
@@ -662,6 +667,7 @@ import java.util.regex.Pattern;
 
   private static int buildPrimaryAndEmbeddedTrackGroupInfos(
       DrmSessionManager drmSessionManager,
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       List<AdaptationSet> adaptationSets,
       int[][] groupedAdaptationSetIndices,
       int primaryGroupCount,
@@ -678,8 +684,24 @@ import java.util.regex.Pattern;
       }
       Format[] formats = new Format[representations.size()];
       for (int j = 0; j < formats.length; j++) {
-        Format format = representations.get(j).format;
+        Format originalFormat = representations.get(j).format;
-        formats[j] = format.copyWithCryptoType(drmSessionManager.getCryptoType(format));
+        Format.Builder updatedFormat =
+            originalFormat
+                .buildUpon()
+                .setCryptoType(drmSessionManager.getCryptoType(originalFormat));
+        if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(originalFormat)) {
+          updatedFormat
+              .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
+              .setCueReplacementBehavior(
+                  subtitleParserFactory.getCueReplacementBehavior(originalFormat))
+              .setCodecs(
+                  originalFormat.sampleMimeType
+                      + (originalFormat.codecs != null ? " " + originalFormat.codecs : ""))
+              // Reset this value to the default. All non-default timestamp adjustments are done
+              // by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding.
+              .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE);
+        }
+        formats[j] = updatedFormat.build();
       }
 
       AdaptationSet firstAdaptationSet = adaptationSets.get(adaptationSetIndices[0]);

libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java

@@ -77,6 +77,8 @@ import androidx.media3.exoplayer.upstream.Loader.LoadErrorAction;
 import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
 import androidx.media3.exoplayer.upstream.ParsingLoadable;
 import androidx.media3.exoplayer.util.SntpClient;
+import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
+import androidx.media3.extractor.text.SubtitleParser;
 import com.google.common.base.Charsets;
 import com.google.common.math.LongMath;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
@@ -112,6 +114,7 @@ public final class DashMediaSource extends BaseMediaSource {
     private DrmSessionManagerProvider drmSessionManagerProvider;
     private CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
     private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
+    @Nullable private SubtitleParser.Factory subtitleParserFactory;
     private long fallbackTargetLiveOffsetMs;
     private long minLiveStartPositionUs;
     @Nullable private ParsingLoadable.Parser<? extends DashManifest> manifestParser;
@@ -196,6 +199,40 @@ public final class DashMediaSource extends BaseMediaSource {
       return this;
     }
 
+    /**
+     * Sets whether subtitles should be parsed as part of extraction (before the sample queue) or as
+     * part of rendering (after the sample queue). Defaults to false (i.e. subtitles will be parsed
+     * as part of rendering).
+     *
+     * <p>This method is experimental. Its default value may change, or it may be renamed or removed
+     * in a future release.
+     *
+     * <p>This method may only be used with {@link DefaultDashChunkSource.Factory}.
+     *
+     * @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or
+     *     rendering.
+     * @return This factory, for convenience.
+     */
+    // TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on
+    //  DefaultMediaSourceFactory via the MediaSource.Factory interface).
+    public Factory experimentalParseSubtitlesDuringExtraction(
+        boolean parseSubtitlesDuringExtraction) {
+      if (parseSubtitlesDuringExtraction) {
+        if (subtitleParserFactory == null) {
+          this.subtitleParserFactory = new DefaultSubtitleParserFactory();
+        }
+      } else {
+        this.subtitleParserFactory = null;
+      }
+      if (chunkSourceFactory instanceof DefaultDashChunkSource.Factory) {
+        ((DefaultDashChunkSource.Factory) chunkSourceFactory)
+            .setSubtitleParserFactory(subtitleParserFactory);
+      } else {
+        throw new IllegalStateException();
+      }
+      return this;
+    }
+
     /**
      * Sets the target {@link Player#getCurrentLiveOffset() offset for live streams} that is used if
      * no value is defined in the {@link MediaItem} or the manifest.
@@ -315,6 +352,7 @@ public final class DashMediaSource extends BaseMediaSource {
           cmcdConfiguration,
           drmSessionManagerProvider.get(mediaItem),
           loadErrorHandlingPolicy,
+          subtitleParserFactory,
           fallbackTargetLiveOffsetMs,
           minLiveStartPositionUs);
     }
@@ -353,6 +391,7 @@ public final class DashMediaSource extends BaseMediaSource {
           cmcdConfiguration,
           drmSessionManagerProvider.get(mediaItem),
           loadErrorHandlingPolicy,
+          subtitleParserFactory,
           fallbackTargetLiveOffsetMs,
           minLiveStartPositionUs);
     }
@@ -411,6 +450,7 @@ public final class DashMediaSource extends BaseMediaSource {
   private final Runnable simulateManifestRefreshRunnable;
   private final PlayerEmsgCallback playerEmsgCallback;
   private final LoaderErrorThrower manifestLoadErrorThrower;
+  @Nullable private final SubtitleParser.Factory subtitleParserFactory;
 
   private DataSource dataSource;
   private Loader loader;
@@ -446,6 +486,7 @@ public final class DashMediaSource extends BaseMediaSource {
       @Nullable CmcdConfiguration cmcdConfiguration,
       DrmSessionManager drmSessionManager,
       LoadErrorHandlingPolicy loadErrorHandlingPolicy,
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       long fallbackTargetLiveOffsetMs,
       long minLiveStartPositionUs) {
     this.mediaItem = mediaItem;
@@ -459,6 +500,7 @@ public final class DashMediaSource extends BaseMediaSource {
     this.cmcdConfiguration = cmcdConfiguration;
     this.drmSessionManager = drmSessionManager;
     this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
+    this.subtitleParserFactory = subtitleParserFactory;
     this.fallbackTargetLiveOffsetMs = fallbackTargetLiveOffsetMs;
     this.minLiveStartPositionUs = minLiveStartPositionUs;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
@@ -564,7 +606,8 @@ public final class DashMediaSource extends BaseMediaSource {
             allocator,
             compositeSequenceableLoaderFactory,
             playerEmsgCallback,
-            getPlayerId());
+            getPlayerId(),
+            subtitleParserFactory);
     periodsById.put(mediaPeriod.id, mediaPeriod);
     return mediaPeriod;
   }

libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java

@@ -60,6 +60,7 @@ import androidx.media3.exoplayer.upstream.CmcdData;
 import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
 import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
 import androidx.media3.extractor.ChunkIndex;
+import androidx.media3.extractor.text.SubtitleParser;
 import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -71,6 +72,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 @UnstableApi
 public class DefaultDashChunkSource implements DashChunkSource {
 
+  /** {@link DashChunkSource.Factory} for {@link DefaultDashChunkSource} instances. */
   public static final class Factory implements DashChunkSource.Factory {
 
     private final DataSource.Factory dataSourceFactory;
@@ -110,6 +112,21 @@ public class DefaultDashChunkSource implements DashChunkSource {
       this.maxSegmentsPerLoad = maxSegmentsPerLoad;
     }
 
+    /**
+     * Sets the {@link SubtitleParser.Factory} to be used for parsing subtitles during extraction,
+     * or null to parse subtitles during decoding.
+     *
+     * <p>This may only be used with {@link BundledChunkExtractor.Factory}.
+     */
+    /* package */ Factory setSubtitleParserFactory(
+        @Nullable SubtitleParser.Factory subtitleParserFactory) {
+      if (chunkExtractorFactory instanceof BundledChunkExtractor.Factory) {
+        ((BundledChunkExtractor.Factory) chunkExtractorFactory)
+            .experimentalSetSubtitleParserFactory(subtitleParserFactory);
+      }
+      return this;
+    }
+
     @Override
     public DashChunkSource createDashChunkSource(
         LoaderErrorThrower manifestLoaderErrorThrower,

libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java

@@ -224,7 +224,8 @@ public final class DashMediaPeriodTest {
         mock(Allocator.class),
         mock(CompositeSequenceableLoaderFactory.class),
         mock(PlayerEmsgCallback.class),
-        PlayerId.UNSET);
+        PlayerId.UNSET,
+        /* subtitleParserFactory= */ null);
   }
 
   private static DashManifest parseManifest(String fileName) throws IOException {

libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java

@@ -22,9 +22,11 @@ import android.graphics.SurfaceTexture;
 import android.view.Surface;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.Player;
+import androidx.media3.datasource.DefaultDataSource;
 import androidx.media3.exoplayer.ExoPlayer;
 import androidx.media3.exoplayer.Renderer;
 import androidx.media3.exoplayer.RenderersFactory;
+import androidx.media3.exoplayer.dash.DashMediaSource;
 import androidx.media3.exoplayer.metadata.MetadataDecoderFactory;
 import androidx.media3.exoplayer.metadata.MetadataRenderer;
 import androidx.media3.exoplayer.trackselection.DefaultTrackSelector;
@@ -79,14 +81,15 @@ public final class DashPlaybackTest {
 
   // https://github.com/google/ExoPlayer/issues/7985
   @Test
-  @Ignore(
-      "Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].")
   public void webvttInMp4() throws Exception {
     Context applicationContext = ApplicationProvider.getApplicationContext();
     CapturingRenderersFactory capturingRenderersFactory =
         new CapturingRenderersFactory(applicationContext);
     ExoPlayer player =
         new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
+            .setMediaSourceFactory(
+                new DashMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
+                    .experimentalParseSubtitlesDuringExtraction(true))
             .setClock(new FakeClock(/* isAutoAdvancing= */ true))
             .build();
     player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java

@@ -26,7 +26,6 @@ import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.DataReader;
 import androidx.media3.common.Format;
-import androidx.media3.common.Format.CueReplacementBehavior;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.ParsableByteArray;
 import androidx.media3.common.util.Util;
@@ -86,8 +85,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     if (currentSubtitleParser == null) {
       delegate.format(format);
     } else {
-      @CueReplacementBehavior
-      int nextCuesBehavior = currentSubtitleParser.getCueReplacementBehavior();
       delegate.format(
           format
               .buildUpon()
@@ -96,7 +93,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               // Reset this value to the default. All non-default timestamp adjustments are done
               // below in sampleMetadata() and there are no 'subsamples' after transcoding.
               .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE)
-              .setCueReplacementBehavior(nextCuesBehavior)
+              .setCueReplacementBehavior(subtitleParserFactory.getCueReplacementBehavior(format))
               .build());
     }
   }

libraries/test_data/src/test/assets/playbackdumps/dash/webvtt-in-mp4.dump

@@ -244,3 +244,6 @@ TextOutput:
       position = 0.5
       positionAnchor = 1
       size = 1.0
+  Subtitle[4]:
+    presentationTimeUs = 456000
+    Cues = []