Change SubtitleParser interface to support incremental output

This change introduces two new types of method to `SubtitleParser`:
1. `parse()` methods that take a `Consumer<CuesWithTiming>` and return `void`
2. `parseToLegacySubtitle` method that returns `Subtitle`

(1) ensures that in the new 'parse before SampleQueue' world we can
write cues to the `SampleQueue` as soon as they're ready - this is
especially important when parsing monolithic text files, e.g. for a
whole movie.

(2) ensures that during the transition, the legacy 'parse after
SampleQueue' behaviour doesn't see any regressions in 'time to first
cue being shown'. Previously we had a single implementation to convert
from `List<CuesWithTiming>` to `Subtitle`, but this relies on the
complete list of cues being available, which can take a long time for
large files in some formats (with ExoPlayer's current parsing logic).
By allowing implementations to customise the way they create a
`Subtitle`, we can directly re-use the existing logic, so that the
'time to first cue being shown' should stay the same.

This change migrates all **usages** to the new methods, but doesn't
migrate any **implementations**. I will migrate the implementations in
follow-up CLs before deleting the old list-returning `parse()` methods.

PiperOrigin-RevId: 565057945

libraries/common/src/main/java/androidx/media3/common/util/Util.java

@@ -171,6 +171,9 @@ public final class Util {
   /** An empty byte array. */
   @UnstableApi public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
 
+  /** An empty long array. */
+  @UnstableApi public static final long[] EMPTY_LONG_ARRAY = new long[0];
+
   private static final String TAG = "Util";
   private static final Pattern XS_DATE_TIME_PATTERN =
       Pattern.compile(

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/DelegatingSubtitleDecoder.java

@@ -17,20 +17,16 @@ package androidx.media3.exoplayer.text;
 
 import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
 
-import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
-import androidx.media3.extractor.text.CuesWithTiming;
 import androidx.media3.extractor.text.SimpleSubtitleDecoder;
 import androidx.media3.extractor.text.Subtitle;
 import androidx.media3.extractor.text.SubtitleParser;
-import com.google.common.collect.ImmutableList;
-import java.util.List;
 
 /**
  * Wrapper around a {@link SubtitleParser} that can be used instead of any current {@link
  * SimpleSubtitleDecoder} subclass. The main {@link #decode(byte[], int, boolean)} method will be
  * delegating the parsing of the data to the underlying {@link SubtitleParser} instance and its
- * {@link SubtitleParser#parse(byte[], int, int)} implementation.
+ * {@link SubtitleParser#parseToLegacySubtitle(byte[], int, int)} implementation.
  *
  * <p>Functionally, once each XXXDecoder class is refactored to be a XXXParser that implements
  * {@link SubtitleParser}, the following should be equivalent:
@@ -55,7 +51,6 @@ import java.util.List;
 @VisibleForTesting(otherwise = PACKAGE_PRIVATE)
 public final class DelegatingSubtitleDecoder extends SimpleSubtitleDecoder {
 
-  private static final Subtitle EMPTY_SUBTITLE = new CuesWithTimingSubtitle(ImmutableList.of());
   private final SubtitleParser subtitleParser;
 
   public DelegatingSubtitleDecoder(String name, SubtitleParser subtitleParser) {
@@ -68,11 +63,6 @@ public final class DelegatingSubtitleDecoder extends SimpleSubtitleDecoder {
     if (reset) {
       subtitleParser.reset();
     }
-    @Nullable
+    return subtitleParser.parseToLegacySubtitle(data, /* offset= */ 0, length);
-    List<CuesWithTiming> cuesWithTiming = subtitleParser.parse(data, /* offset= */ 0, length);
-    if (cuesWithTiming == null) {
-      return EMPTY_SUBTITLE;
-    }
-    return new CuesWithTimingSubtitle(cuesWithTiming);
   }
 }

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java

@@ -26,6 +26,7 @@ import androidx.media3.common.C;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.extractor.text.CueDecoder;
+import androidx.media3.extractor.text.CuesWithTimingSubtitle;
 import androidx.media3.extractor.text.Subtitle;
 import androidx.media3.extractor.text.SubtitleDecoder;
 import androidx.media3.extractor.text.SubtitleDecoderException;

libraries/extractor/src/main/java/androidx/media3/extractor/text/CuesWithTimingSubtitle.java

@@ -13,16 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package androidx.media3.exoplayer.text;
+package androidx.media3.extractor.text;
 
 import static androidx.media3.common.util.Assertions.checkArgument;
 
 import androidx.media3.common.C;
 import androidx.media3.common.text.Cue;
 import androidx.media3.common.util.Log;
+import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
-import androidx.media3.extractor.text.CuesWithTiming;
-import androidx.media3.extractor.text.Subtitle;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Ordering;
@@ -31,7 +30,9 @@ import java.util.Arrays;
 import java.util.List;
 
 /** A {@link Subtitle} backed by a list of {@link CuesWithTiming} instances. */
-/* package */ final class CuesWithTimingSubtitle implements Subtitle {
+// TODO(b/181312195): Make this package-private when ExoplayerCuesDecoder is deleted.
+@UnstableApi
+public final class CuesWithTimingSubtitle implements Subtitle {
 
   private static final String TAG = "CuesWithTimingSubtitle";
 

libraries/extractor/src/main/java/androidx/media3/extractor/text/Subtitle.java

@@ -59,6 +59,8 @@ public interface Subtitle {
   List<Cue> getCues(long timeUs);
 
   /** Converts the current instance to a list of {@link CuesWithTiming} representing it. */
+  // TODO(b/181312195): Remove this when TtmlDecoder has been migrated to TtmlParser (and in-line it
+  //  in DelegatingSubtitleDecoderTtmlParserTest).
   default ImmutableList<CuesWithTiming> toCuesWithTimingList() {
     ImmutableList.Builder<CuesWithTiming> allCues = ImmutableList.builder();
     for (int i = 0; i < getEventTimeCount(); i++) {

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java

@@ -15,7 +15,6 @@
  */
 package androidx.media3.extractor.text;
 
-import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
 import static java.lang.annotation.ElementType.TYPE_USE;
@@ -42,6 +41,7 @@ import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
@@ -84,14 +84,14 @@ public class SubtitleExtractor implements Extractor {
   private final SubtitleParser subtitleParser;
   private final CueEncoder cueEncoder;
   private final Format format;
-  private final List<Long> timestamps;
+  private final List<Sample> samples;
-  private final List<byte[]> samples;
   private final ParsableByteArray scratchSampleArray;
 
   private byte[] subtitleData;
   private @MonotonicNonNull TrackOutput trackOutput;
   private int bytesRead;
   private @State int state;
+  private long[] timestamps;
   private long seekTimeUs;
 
   /**
@@ -112,9 +112,9 @@ public class SubtitleExtractor implements Extractor {
             .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
             .setCodecs(format.sampleMimeType)
             .build();
-    timestamps = new ArrayList<>();
     samples = new ArrayList<>();
     state = STATE_CREATED;
+    timestamps = Util.EMPTY_LONG_ARRAY;
     seekTimeUs = C.TIME_UNSET;
   }
 
@@ -157,8 +157,7 @@ public class SubtitleExtractor implements Extractor {
     if (state == STATE_EXTRACTING) {
       boolean inputFinished = readFromInput(input);
       if (inputFinished) {
-        parse();
+        parseAndWriteToOutput();
-        writeToOutput();
         state = STATE_FINISHED;
       }
     }
@@ -223,41 +222,78 @@ public class SubtitleExtractor implements Extractor {
         || readResult == C.RESULT_END_OF_INPUT;
   }
 
-  /** Parses the subtitle data and stores the samples in the memory of the extractor. */
+  /**
-  private void parse() throws IOException {
+   * Parses the subtitle data and writes the samples to the output, and stores them in {@link
+   * #timestamps} and {@link #samples} to speed up any subsequent seeks.
+   *
+   * <p>Also reassigns {@link #subtitleData} to an empty array once parsing is complete.
+   */
+  private void parseAndWriteToOutput() throws IOException {
     try {
-      List<CuesWithTiming> cuesWithTimingList = checkNotNull(subtitleParser.parse(subtitleData));
+      SubtitleParser.OutputOptions outputOptions =
-      for (int i = 0; i < cuesWithTimingList.size(); i++) {
+          seekTimeUs != C.TIME_UNSET
-        CuesWithTiming cuesWithTiming = cuesWithTimingList.get(i);
+              ? SubtitleParser.OutputOptions.cuesAfterThenRemainingCuesBefore(seekTimeUs)
-        long eventTimeUs = cuesWithTiming.startTimeUs;
+              : SubtitleParser.OutputOptions.allCues();
-        byte[] cuesSample = cueEncoder.encode(cuesWithTiming.cues, cuesWithTiming.durationUs);
+      subtitleParser.parse(
-        timestamps.add(eventTimeUs);
+          subtitleData,
-        samples.add(cuesSample);
+          outputOptions,
+          cuesWithTiming -> {
+            Sample sample =
+                new Sample(
+                    cuesWithTiming.startTimeUs,
+                    cueEncoder.encode(cuesWithTiming.cues, cuesWithTiming.durationUs));
+            samples.add(sample);
+            if (seekTimeUs == C.TIME_UNSET || cuesWithTiming.startTimeUs >= seekTimeUs) {
+              writeToOutput(sample);
+            }
+          });
+      Collections.sort(samples);
+      timestamps = new long[samples.size()];
+      for (int i = 0; i < samples.size(); i++) {
+        timestamps[i] = samples.get(i).timeUs;
       }
+      subtitleData = Util.EMPTY_BYTE_ARRAY;
     } catch (RuntimeException e) {
       throw ParserException.createForMalformedContainer("SubtitleParser failed.", e);
     }
   }
 
   private void writeToOutput() {
-    checkStateNotNull(this.trackOutput);
-    checkState(timestamps.size() == samples.size());
     int index =
         seekTimeUs == C.TIME_UNSET
             ? 0
             : Util.binarySearchFloor(
                 timestamps, seekTimeUs, /* inclusive= */ true, /* stayInBounds= */ true);
     for (int i = index; i < samples.size(); i++) {
-      byte[] sample = samples.get(i);
+      writeToOutput(samples.get(i));
-      int size = sample.length;
+    }
-      scratchSampleArray.reset(sample);
+  }
-      trackOutput.sampleData(scratchSampleArray, size);
+
-      trackOutput.sampleMetadata(
+  private void writeToOutput(Sample sample) {
-          /* timeUs= */ timestamps.get(i),
+    checkStateNotNull(this.trackOutput);
-          /* flags= */ C.BUFFER_FLAG_KEY_FRAME,
+    int size = sample.data.length;
-          /* size= */ size,
+    scratchSampleArray.reset(sample.data);
-          /* offset= */ 0,
+    trackOutput.sampleData(scratchSampleArray, size);
-          /* cryptoData= */ null);
+    trackOutput.sampleMetadata(
+        sample.timeUs,
+        /* flags= */ C.BUFFER_FLAG_KEY_FRAME,
+        /* size= */ size,
+        /* offset= */ 0,
+        /* cryptoData= */ null);
+  }
+
+  private static class Sample implements Comparable<Sample> {
+    private final long timeUs;
+    private final byte[] data;
+
+    private Sample(long timeUs, byte[] data) {
+      this.timeUs = timeUs;
+      this.data = data;
+    }
+
+    @Override
+    public int compareTo(Sample sample) {
+      return Long.compare(timeUs, sample.timeUs);
     }
   }
 }

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java

@@ -22,7 +22,10 @@ import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.Format.CueReplacementBehavior;
+import androidx.media3.common.util.Consumer;
 import androidx.media3.common.util.UnstableApi;
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -52,6 +55,50 @@ public interface SubtitleParser {
     SubtitleParser create(Format format);
   }
 
+  /**
+   * Options to control the output behavior of {@link SubtitleParser} methods that emit their output
+   * incrementally using a {@link Consumer} provided by the caller.
+   */
+  class OutputOptions {
+
+    private static final OutputOptions ALL =
+        new OutputOptions(C.TIME_UNSET, /* outputAllCues= */ false);
+
+    public final long startTimeUs;
+    public final boolean outputAllCues;
+
+    private OutputOptions(long startTimeUs, boolean outputAllCues) {
+      this.startTimeUs = startTimeUs;
+      this.outputAllCues = outputAllCues;
+    }
+
+    /** Output all {@link CuesWithTiming} instances. */
+    public static OutputOptions allCues() {
+      return ALL;
+    }
+
+    /**
+     * Only output {@link CuesWithTiming} instances where {@link CuesWithTiming#startTimeUs} is at
+     * least {@code startTimeUs}.
+     *
+     * <p>The order in which {@link CuesWithTiming} instances are emitted is not defined.
+     */
+    public static OutputOptions onlyCuesAfter(long startTimeUs) {
+      return new OutputOptions(startTimeUs, /* outputAllCues= */ false);
+    }
+
+    /**
+     * Output {@link CuesWithTiming} where {@link CuesWithTiming#startTimeUs} is at least {@code
+     * startTimeUs}, followed by the remaining {@link CuesWithTiming} instances.
+     *
+     * <p>Beyond this, the order in which {@link CuesWithTiming} instances are emitted is not
+     * defined.
+     */
+    public static OutputOptions cuesAfterThenRemainingCuesBefore(long startTimeUs) {
+      return new OutputOptions(startTimeUs, /* outputAllCues= */ true);
+    }
+  }
+
   /**
    * Parses {@code data} (and any data stored from previous invocations) and returns any resulting
    * complete {@link CuesWithTiming} instances.
@@ -63,6 +110,17 @@ public interface SubtitleParser {
     return parse(data, /* offset= */ 0, data.length);
   }
 
+  /**
+   * Parses {@code data} (and any data stored from previous invocations) and emits resulting {@link
+   * CuesWithTiming} instances.
+   *
+   * <p>Equivalent to {@link #parse(byte[], int, int, OutputOptions, Consumer) parse(data, 0,
+   * data.length, outputOptions, output)}.
+   */
+  default void parse(byte[] data, OutputOptions outputOptions, Consumer<CuesWithTiming> output) {
+    parse(data, /* offset= */ 0, data.length, outputOptions, output);
+  }
+
   /**
    * Parses {@code data} (and any data stored from previous invocations) and returns any resulting
    * complete {@link CuesWithTiming} instances.
@@ -90,6 +148,82 @@ public interface SubtitleParser {
   @Nullable
   List<CuesWithTiming> parse(byte[] data, int offset, int length);
 
+  /**
+   * Parses {@code data} (and any data stored from previous invocations) and emits any resulting
+   * complete {@link CuesWithTiming} instances via {@code output}.
+   *
+   * <p>Any samples not used from {@code data} will be persisted and used during subsequent calls to
+   * this method.
+   *
+   * <p>{@link CuesWithTiming#startTimeUs} in an emitted instance is derived only from the provided
+   * sample data, so has to be considered together with any relevant {@link
+   * Format#subsampleOffsetUs}. If the provided sample doesn't contain any timing information then
+   * at most one {@link CuesWithTiming} instance will be emitted, with {@link
+   * CuesWithTiming#startTimeUs} set to {@link C#TIME_UNSET}, in which case {@link
+   * Format#subsampleOffsetUs} <b>must</b> be {@link Format#OFFSET_SAMPLE_RELATIVE}.
+   *
+   * @param data The subtitle data to parse. This must contain only complete samples. For subtitles
+   *     muxed inside a media container, a sample is usually defined by the container. For subtitles
+   *     read from a text file, a sample is usually the entire contents of the text file.
+   * @param offset The index in {@code data} to start reading from (inclusive).
+   * @param length The number of bytes to read from {@code data}.
+   * @param outputOptions Options to control how instances are emitted to {@code output}.
+   * @param output A consumer for {@link CuesWithTiming} instances emitted by this method. All calls
+   *     will be made on the thread that called this method, and will be completed before this
+   *     method returns.
+   */
+  default void parse(
+      byte[] data,
+      int offset,
+      int length,
+      OutputOptions outputOptions,
+      Consumer<CuesWithTiming> output) {
+    List<CuesWithTiming> cuesWithTimingList = parse(data, offset, length);
+    if (cuesWithTimingList == null) {
+      return;
+    }
+    @Nullable
+    List<CuesWithTiming> cuesWithTimingBeforeRequestedStartTimeUs =
+        outputOptions.startTimeUs != C.TIME_UNSET && outputOptions.outputAllCues
+            ? new ArrayList<>()
+            : null;
+    for (CuesWithTiming cuesWithTiming : cuesWithTimingList) {
+      if (outputOptions.startTimeUs == C.TIME_UNSET
+          || cuesWithTiming.startTimeUs >= outputOptions.startTimeUs) {
+        output.accept(cuesWithTiming);
+      } else if (cuesWithTimingBeforeRequestedStartTimeUs != null) {
+        cuesWithTimingBeforeRequestedStartTimeUs.add(cuesWithTiming);
+      }
+    }
+    if (cuesWithTimingBeforeRequestedStartTimeUs != null) {
+      for (CuesWithTiming cuesWithTiming : cuesWithTimingBeforeRequestedStartTimeUs) {
+        output.accept(cuesWithTiming);
+      }
+    }
+  }
+
+  /**
+   * Parses {@code data} to a legacy {@link Subtitle} instance.
+   *
+   * <p>This method only exists temporarily to support the transition away from {@link
+   * SubtitleDecoder} and {@link Subtitle}. It will be removed in a future release.
+   *
+   * <p>The default implementation delegates to {@link #parse(byte[], int, int, OutputOptions,
+   * Consumer)}. Implementations can override this to provide a more efficient implementation if
+   * desired.
+   *
+   * @param data The subtitle data to parse. This must contain only complete samples. For subtitles
+   *     muxed inside a media container, a sample is usually defined by the container. For subtitles
+   *     read from a text file, a sample is usually the entire contents of the text file.
+   * @param offset The index in {@code data} to start reading from (inclusive).
+   * @param length The number of bytes to read from {@code data}.
+   */
+  default Subtitle parseToLegacySubtitle(byte[] data, int offset, int length) {
+    ImmutableList.Builder<CuesWithTiming> cuesWithTimingList = ImmutableList.builder();
+    parse(data, offset, length, OutputOptions.ALL, cuesWithTimingList::add);
+    return new CuesWithTimingSubtitle(cuesWithTimingList.build());
+  }
+
   /**
    * Clears any data stored inside this parser from previous {@link #parse(byte[])} calls.
    *

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java

@@ -33,7 +33,6 @@ import androidx.media3.common.util.Util;
 import androidx.media3.extractor.TrackOutput;
 import java.io.EOFException;
 import java.io.IOException;
-import java.util.List;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
@@ -145,41 +144,41 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       delegate.sampleMetadata(timeUs, flags, size, offset, cryptoData);
       return;
     }
-    checkStateNotNull(currentFormat); // format() must be called before sampleMetadata()
     checkArgument(cryptoData == null, "DRM on subtitles is not supported");
 
     int sampleStart = sampleDataEnd - offset - size;
-    @Nullable
+    currentSubtitleParser.parse(
-    List<CuesWithTiming> cuesWithTimingList =
+        sampleData,
-        currentSubtitleParser.parse(sampleData, /* offset= */ sampleStart, /* length= */ size);
+        sampleStart,
+        size,
+        SubtitleParser.OutputOptions.allCues(),
+        cuesWithTiming -> outputSample(cuesWithTiming, timeUs, flags));
     sampleDataStart = sampleStart + size;
-    if (cuesWithTimingList != null) {
+  }
-      for (int i = 0; i < cuesWithTimingList.size(); i++) {
-        CuesWithTiming cuesWithTiming = cuesWithTimingList.get(i);
-        byte[] cuesWithDurationBytes =
-            cueEncoder.encode(cuesWithTiming.cues, cuesWithTiming.durationUs);
 
-        parsableScratch.reset(cuesWithDurationBytes);
+  private void outputSample(CuesWithTiming cuesWithTiming, long timeUs, int flags) {
-        delegate.sampleData(parsableScratch, cuesWithDurationBytes.length);
+    checkStateNotNull(currentFormat); // format() must be called before sampleMetadata()
-        // Clear FLAG_DECODE_ONLY if it is set.
+    byte[] cuesWithDurationBytes =
-        flags &= ~C.BUFFER_FLAG_DECODE_ONLY;
+        cueEncoder.encode(cuesWithTiming.cues, cuesWithTiming.durationUs);
-        long outputSampleTimeUs;
+    parsableScratch.reset(cuesWithDurationBytes);
-        if (cuesWithTiming.startTimeUs == C.TIME_UNSET) {
+    delegate.sampleData(parsableScratch, cuesWithDurationBytes.length);
-          checkState(currentFormat.subsampleOffsetUs == Format.OFFSET_SAMPLE_RELATIVE);
+    // Clear FLAG_DECODE_ONLY if it is set.
-          outputSampleTimeUs = timeUs;
+    flags &= ~C.BUFFER_FLAG_DECODE_ONLY;
-        } else if (currentFormat.subsampleOffsetUs == Format.OFFSET_SAMPLE_RELATIVE) {
+    long outputSampleTimeUs;
-          outputSampleTimeUs = timeUs + cuesWithTiming.startTimeUs;
+    if (cuesWithTiming.startTimeUs == C.TIME_UNSET) {
-        } else {
+      checkState(currentFormat.subsampleOffsetUs == Format.OFFSET_SAMPLE_RELATIVE);
-          outputSampleTimeUs = cuesWithTiming.startTimeUs + currentFormat.subsampleOffsetUs;
+      outputSampleTimeUs = timeUs;
-        }
+    } else if (currentFormat.subsampleOffsetUs == Format.OFFSET_SAMPLE_RELATIVE) {
-        delegate.sampleMetadata(
+      outputSampleTimeUs = timeUs + cuesWithTiming.startTimeUs;
-            outputSampleTimeUs,
+    } else {
-            flags,
+      outputSampleTimeUs = cuesWithTiming.startTimeUs + currentFormat.subsampleOffsetUs;
-            cuesWithDurationBytes.length,
-            /* offset= */ 0,
-            /* cryptoData= */ null);
-      }
     }
+    delegate.sampleMetadata(
+        outputSampleTimeUs,
+        flags,
+        cuesWithDurationBytes.length,
+        /* offset= */ 0,
+        /* cryptoData= */ null);
   }
 
   /**