diff --git a/libraries/common/src/main/java/androidx/media3/common/util/Util.java b/libraries/common/src/main/java/androidx/media3/common/util/Util.java index 19c53791bb..a77b09985d 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/Util.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/Util.java @@ -171,6 +171,9 @@ public final class Util { /** An empty byte array. */ @UnstableApi public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + /** An empty long array. */ + @UnstableApi public static final long[] EMPTY_LONG_ARRAY = new long[0]; + private static final String TAG = "Util"; private static final Pattern XS_DATE_TIME_PATTERN = Pattern.compile( diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/DelegatingSubtitleDecoder.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/DelegatingSubtitleDecoder.java index d812bd11e5..a50aade64b 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/DelegatingSubtitleDecoder.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/DelegatingSubtitleDecoder.java @@ -17,20 +17,16 @@ package androidx.media3.exoplayer.text; import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE; -import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; -import androidx.media3.extractor.text.CuesWithTiming; import androidx.media3.extractor.text.SimpleSubtitleDecoder; import androidx.media3.extractor.text.Subtitle; import androidx.media3.extractor.text.SubtitleParser; -import com.google.common.collect.ImmutableList; -import java.util.List; /** * Wrapper around a {@link SubtitleParser} that can be used instead of any current {@link * SimpleSubtitleDecoder} subclass. The main {@link #decode(byte[], int, boolean)} method will be * delegating the parsing of the data to the underlying {@link SubtitleParser} instance and its - * {@link SubtitleParser#parse(byte[], int, int)} implementation. + * {@link SubtitleParser#parseToLegacySubtitle(byte[], int, int)} implementation. * *

Functionally, once each XXXDecoder class is refactored to be a XXXParser that implements * {@link SubtitleParser}, the following should be equivalent: @@ -55,7 +51,6 @@ import java.util.List; @VisibleForTesting(otherwise = PACKAGE_PRIVATE) public final class DelegatingSubtitleDecoder extends SimpleSubtitleDecoder { - private static final Subtitle EMPTY_SUBTITLE = new CuesWithTimingSubtitle(ImmutableList.of()); private final SubtitleParser subtitleParser; public DelegatingSubtitleDecoder(String name, SubtitleParser subtitleParser) { @@ -68,11 +63,6 @@ public final class DelegatingSubtitleDecoder extends SimpleSubtitleDecoder { if (reset) { subtitleParser.reset(); } - @Nullable - List cuesWithTiming = subtitleParser.parse(data, /* offset= */ 0, length); - if (cuesWithTiming == null) { - return EMPTY_SUBTITLE; - } - return new CuesWithTimingSubtitle(cuesWithTiming); + return subtitleParser.parseToLegacySubtitle(data, /* offset= */ 0, length); } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java index 7397ff56e7..49c0b925f4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java @@ -26,6 +26,7 @@ import androidx.media3.common.C; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.UnstableApi; import androidx.media3.extractor.text.CueDecoder; +import androidx.media3.extractor.text.CuesWithTimingSubtitle; import androidx.media3.extractor.text.Subtitle; import androidx.media3.extractor.text.SubtitleDecoder; import androidx.media3.extractor.text.SubtitleDecoderException; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/CuesWithTimingSubtitle.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/CuesWithTimingSubtitle.java similarity index 95% rename from libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/CuesWithTimingSubtitle.java rename to libraries/extractor/src/main/java/androidx/media3/extractor/text/CuesWithTimingSubtitle.java index b5e1ced914..4d3d599f42 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/CuesWithTimingSubtitle.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/CuesWithTimingSubtitle.java @@ -13,16 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.exoplayer.text; +package androidx.media3.extractor.text; import static androidx.media3.common.util.Assertions.checkArgument; import androidx.media3.common.C; import androidx.media3.common.text.Cue; import androidx.media3.common.util.Log; +import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; -import androidx.media3.extractor.text.CuesWithTiming; -import androidx.media3.extractor.text.Subtitle; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; @@ -31,7 +30,9 @@ import java.util.Arrays; import java.util.List; /** A {@link Subtitle} backed by a list of {@link CuesWithTiming} instances. */ -/* package */ final class CuesWithTimingSubtitle implements Subtitle { +// TODO(b/181312195): Make this package-private when ExoplayerCuesDecoder is deleted. +@UnstableApi +public final class CuesWithTimingSubtitle implements Subtitle { private static final String TAG = "CuesWithTimingSubtitle"; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/Subtitle.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/Subtitle.java index b82caf7b11..595fe6b36f 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/Subtitle.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/Subtitle.java @@ -59,6 +59,8 @@ public interface Subtitle { List getCues(long timeUs); /** Converts the current instance to a list of {@link CuesWithTiming} representing it. */ + // TODO(b/181312195): Remove this when TtmlDecoder has been migrated to TtmlParser (and in-line it + // in DelegatingSubtitleDecoderTtmlParserTest). default ImmutableList toCuesWithTimingList() { ImmutableList.Builder allCues = ImmutableList.builder(); for (int i = 0; i < getEventTimeCount(); i++) { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java index 6978000f04..8de2e12635 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java @@ -15,7 +15,6 @@ */ package androidx.media3.extractor.text; -import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static java.lang.annotation.ElementType.TYPE_USE; @@ -42,6 +41,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -84,14 +84,14 @@ public class SubtitleExtractor implements Extractor { private final SubtitleParser subtitleParser; private final CueEncoder cueEncoder; private final Format format; - private final List timestamps; - private final List samples; + private final List samples; private final ParsableByteArray scratchSampleArray; private byte[] subtitleData; private @MonotonicNonNull TrackOutput trackOutput; private int bytesRead; private @State int state; + private long[] timestamps; private long seekTimeUs; /** @@ -112,9 +112,9 @@ public class SubtitleExtractor implements Extractor { .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) .setCodecs(format.sampleMimeType) .build(); - timestamps = new ArrayList<>(); samples = new ArrayList<>(); state = STATE_CREATED; + timestamps = Util.EMPTY_LONG_ARRAY; seekTimeUs = C.TIME_UNSET; } @@ -157,8 +157,7 @@ public class SubtitleExtractor implements Extractor { if (state == STATE_EXTRACTING) { boolean inputFinished = readFromInput(input); if (inputFinished) { - parse(); - writeToOutput(); + parseAndWriteToOutput(); state = STATE_FINISHED; } } @@ -223,41 +222,78 @@ public class SubtitleExtractor implements Extractor { || readResult == C.RESULT_END_OF_INPUT; } - /** Parses the subtitle data and stores the samples in the memory of the extractor. */ - private void parse() throws IOException { + /** + * Parses the subtitle data and writes the samples to the output, and stores them in {@link + * #timestamps} and {@link #samples} to speed up any subsequent seeks. + * + *

Also reassigns {@link #subtitleData} to an empty array once parsing is complete. + */ + private void parseAndWriteToOutput() throws IOException { try { - List cuesWithTimingList = checkNotNull(subtitleParser.parse(subtitleData)); - for (int i = 0; i < cuesWithTimingList.size(); i++) { - CuesWithTiming cuesWithTiming = cuesWithTimingList.get(i); - long eventTimeUs = cuesWithTiming.startTimeUs; - byte[] cuesSample = cueEncoder.encode(cuesWithTiming.cues, cuesWithTiming.durationUs); - timestamps.add(eventTimeUs); - samples.add(cuesSample); + SubtitleParser.OutputOptions outputOptions = + seekTimeUs != C.TIME_UNSET + ? SubtitleParser.OutputOptions.cuesAfterThenRemainingCuesBefore(seekTimeUs) + : SubtitleParser.OutputOptions.allCues(); + subtitleParser.parse( + subtitleData, + outputOptions, + cuesWithTiming -> { + Sample sample = + new Sample( + cuesWithTiming.startTimeUs, + cueEncoder.encode(cuesWithTiming.cues, cuesWithTiming.durationUs)); + samples.add(sample); + if (seekTimeUs == C.TIME_UNSET || cuesWithTiming.startTimeUs >= seekTimeUs) { + writeToOutput(sample); + } + }); + Collections.sort(samples); + timestamps = new long[samples.size()]; + for (int i = 0; i < samples.size(); i++) { + timestamps[i] = samples.get(i).timeUs; } + subtitleData = Util.EMPTY_BYTE_ARRAY; } catch (RuntimeException e) { throw ParserException.createForMalformedContainer("SubtitleParser failed.", e); } } private void writeToOutput() { - checkStateNotNull(this.trackOutput); - checkState(timestamps.size() == samples.size()); int index = seekTimeUs == C.TIME_UNSET ? 0 : Util.binarySearchFloor( timestamps, seekTimeUs, /* inclusive= */ true, /* stayInBounds= */ true); for (int i = index; i < samples.size(); i++) { - byte[] sample = samples.get(i); - int size = sample.length; - scratchSampleArray.reset(sample); - trackOutput.sampleData(scratchSampleArray, size); - trackOutput.sampleMetadata( - /* timeUs= */ timestamps.get(i), - /* flags= */ C.BUFFER_FLAG_KEY_FRAME, - /* size= */ size, - /* offset= */ 0, - /* cryptoData= */ null); + writeToOutput(samples.get(i)); + } + } + + private void writeToOutput(Sample sample) { + checkStateNotNull(this.trackOutput); + int size = sample.data.length; + scratchSampleArray.reset(sample.data); + trackOutput.sampleData(scratchSampleArray, size); + trackOutput.sampleMetadata( + sample.timeUs, + /* flags= */ C.BUFFER_FLAG_KEY_FRAME, + /* size= */ size, + /* offset= */ 0, + /* cryptoData= */ null); + } + + private static class Sample implements Comparable { + private final long timeUs; + private final byte[] data; + + private Sample(long timeUs, byte[] data) { + this.timeUs = timeUs; + this.data = data; + } + + @Override + public int compareTo(Sample sample) { + return Long.compare(timeUs, sample.timeUs); } } } diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java index 928824514d..34a7083164 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java @@ -22,7 +22,10 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.Format.CueReplacementBehavior; +import androidx.media3.common.util.Consumer; import androidx.media3.common.util.UnstableApi; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; /** @@ -52,6 +55,50 @@ public interface SubtitleParser { SubtitleParser create(Format format); } + /** + * Options to control the output behavior of {@link SubtitleParser} methods that emit their output + * incrementally using a {@link Consumer} provided by the caller. + */ + class OutputOptions { + + private static final OutputOptions ALL = + new OutputOptions(C.TIME_UNSET, /* outputAllCues= */ false); + + public final long startTimeUs; + public final boolean outputAllCues; + + private OutputOptions(long startTimeUs, boolean outputAllCues) { + this.startTimeUs = startTimeUs; + this.outputAllCues = outputAllCues; + } + + /** Output all {@link CuesWithTiming} instances. */ + public static OutputOptions allCues() { + return ALL; + } + + /** + * Only output {@link CuesWithTiming} instances where {@link CuesWithTiming#startTimeUs} is at + * least {@code startTimeUs}. + * + *

The order in which {@link CuesWithTiming} instances are emitted is not defined. + */ + public static OutputOptions onlyCuesAfter(long startTimeUs) { + return new OutputOptions(startTimeUs, /* outputAllCues= */ false); + } + + /** + * Output {@link CuesWithTiming} where {@link CuesWithTiming#startTimeUs} is at least {@code + * startTimeUs}, followed by the remaining {@link CuesWithTiming} instances. + * + *

Beyond this, the order in which {@link CuesWithTiming} instances are emitted is not + * defined. + */ + public static OutputOptions cuesAfterThenRemainingCuesBefore(long startTimeUs) { + return new OutputOptions(startTimeUs, /* outputAllCues= */ true); + } + } + /** * Parses {@code data} (and any data stored from previous invocations) and returns any resulting * complete {@link CuesWithTiming} instances. @@ -63,6 +110,17 @@ public interface SubtitleParser { return parse(data, /* offset= */ 0, data.length); } + /** + * Parses {@code data} (and any data stored from previous invocations) and emits resulting {@link + * CuesWithTiming} instances. + * + *

Equivalent to {@link #parse(byte[], int, int, OutputOptions, Consumer) parse(data, 0, + * data.length, outputOptions, output)}. + */ + default void parse(byte[] data, OutputOptions outputOptions, Consumer output) { + parse(data, /* offset= */ 0, data.length, outputOptions, output); + } + /** * Parses {@code data} (and any data stored from previous invocations) and returns any resulting * complete {@link CuesWithTiming} instances. @@ -90,6 +148,82 @@ public interface SubtitleParser { @Nullable List parse(byte[] data, int offset, int length); + /** + * Parses {@code data} (and any data stored from previous invocations) and emits any resulting + * complete {@link CuesWithTiming} instances via {@code output}. + * + *

Any samples not used from {@code data} will be persisted and used during subsequent calls to + * this method. + * + *

{@link CuesWithTiming#startTimeUs} in an emitted instance is derived only from the provided + * sample data, so has to be considered together with any relevant {@link + * Format#subsampleOffsetUs}. If the provided sample doesn't contain any timing information then + * at most one {@link CuesWithTiming} instance will be emitted, with {@link + * CuesWithTiming#startTimeUs} set to {@link C#TIME_UNSET}, in which case {@link + * Format#subsampleOffsetUs} must be {@link Format#OFFSET_SAMPLE_RELATIVE}. + * + * @param data The subtitle data to parse. This must contain only complete samples. For subtitles + * muxed inside a media container, a sample is usually defined by the container. For subtitles + * read from a text file, a sample is usually the entire contents of the text file. + * @param offset The index in {@code data} to start reading from (inclusive). + * @param length The number of bytes to read from {@code data}. + * @param outputOptions Options to control how instances are emitted to {@code output}. + * @param output A consumer for {@link CuesWithTiming} instances emitted by this method. All calls + * will be made on the thread that called this method, and will be completed before this + * method returns. + */ + default void parse( + byte[] data, + int offset, + int length, + OutputOptions outputOptions, + Consumer output) { + List cuesWithTimingList = parse(data, offset, length); + if (cuesWithTimingList == null) { + return; + } + @Nullable + List cuesWithTimingBeforeRequestedStartTimeUs = + outputOptions.startTimeUs != C.TIME_UNSET && outputOptions.outputAllCues + ? new ArrayList<>() + : null; + for (CuesWithTiming cuesWithTiming : cuesWithTimingList) { + if (outputOptions.startTimeUs == C.TIME_UNSET + || cuesWithTiming.startTimeUs >= outputOptions.startTimeUs) { + output.accept(cuesWithTiming); + } else if (cuesWithTimingBeforeRequestedStartTimeUs != null) { + cuesWithTimingBeforeRequestedStartTimeUs.add(cuesWithTiming); + } + } + if (cuesWithTimingBeforeRequestedStartTimeUs != null) { + for (CuesWithTiming cuesWithTiming : cuesWithTimingBeforeRequestedStartTimeUs) { + output.accept(cuesWithTiming); + } + } + } + + /** + * Parses {@code data} to a legacy {@link Subtitle} instance. + * + *

This method only exists temporarily to support the transition away from {@link + * SubtitleDecoder} and {@link Subtitle}. It will be removed in a future release. + * + *

The default implementation delegates to {@link #parse(byte[], int, int, OutputOptions, + * Consumer)}. Implementations can override this to provide a more efficient implementation if + * desired. + * + * @param data The subtitle data to parse. This must contain only complete samples. For subtitles + * muxed inside a media container, a sample is usually defined by the container. For subtitles + * read from a text file, a sample is usually the entire contents of the text file. + * @param offset The index in {@code data} to start reading from (inclusive). + * @param length The number of bytes to read from {@code data}. + */ + default Subtitle parseToLegacySubtitle(byte[] data, int offset, int length) { + ImmutableList.Builder cuesWithTimingList = ImmutableList.builder(); + parse(data, offset, length, OutputOptions.ALL, cuesWithTimingList::add); + return new CuesWithTimingSubtitle(cuesWithTimingList.build()); + } + /** * Clears any data stored inside this parser from previous {@link #parse(byte[])} calls. * diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java index d5c9b01c6b..52f01a5e97 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java @@ -33,7 +33,6 @@ import androidx.media3.common.util.Util; import androidx.media3.extractor.TrackOutput; import java.io.EOFException; import java.io.IOException; -import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** @@ -145,41 +144,41 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; delegate.sampleMetadata(timeUs, flags, size, offset, cryptoData); return; } - checkStateNotNull(currentFormat); // format() must be called before sampleMetadata() checkArgument(cryptoData == null, "DRM on subtitles is not supported"); int sampleStart = sampleDataEnd - offset - size; - @Nullable - List cuesWithTimingList = - currentSubtitleParser.parse(sampleData, /* offset= */ sampleStart, /* length= */ size); + currentSubtitleParser.parse( + sampleData, + sampleStart, + size, + SubtitleParser.OutputOptions.allCues(), + cuesWithTiming -> outputSample(cuesWithTiming, timeUs, flags)); sampleDataStart = sampleStart + size; - if (cuesWithTimingList != null) { - for (int i = 0; i < cuesWithTimingList.size(); i++) { - CuesWithTiming cuesWithTiming = cuesWithTimingList.get(i); - byte[] cuesWithDurationBytes = - cueEncoder.encode(cuesWithTiming.cues, cuesWithTiming.durationUs); + } - parsableScratch.reset(cuesWithDurationBytes); - delegate.sampleData(parsableScratch, cuesWithDurationBytes.length); - // Clear FLAG_DECODE_ONLY if it is set. - flags &= ~C.BUFFER_FLAG_DECODE_ONLY; - long outputSampleTimeUs; - if (cuesWithTiming.startTimeUs == C.TIME_UNSET) { - checkState(currentFormat.subsampleOffsetUs == Format.OFFSET_SAMPLE_RELATIVE); - outputSampleTimeUs = timeUs; - } else if (currentFormat.subsampleOffsetUs == Format.OFFSET_SAMPLE_RELATIVE) { - outputSampleTimeUs = timeUs + cuesWithTiming.startTimeUs; - } else { - outputSampleTimeUs = cuesWithTiming.startTimeUs + currentFormat.subsampleOffsetUs; - } - delegate.sampleMetadata( - outputSampleTimeUs, - flags, - cuesWithDurationBytes.length, - /* offset= */ 0, - /* cryptoData= */ null); - } + private void outputSample(CuesWithTiming cuesWithTiming, long timeUs, int flags) { + checkStateNotNull(currentFormat); // format() must be called before sampleMetadata() + byte[] cuesWithDurationBytes = + cueEncoder.encode(cuesWithTiming.cues, cuesWithTiming.durationUs); + parsableScratch.reset(cuesWithDurationBytes); + delegate.sampleData(parsableScratch, cuesWithDurationBytes.length); + // Clear FLAG_DECODE_ONLY if it is set. + flags &= ~C.BUFFER_FLAG_DECODE_ONLY; + long outputSampleTimeUs; + if (cuesWithTiming.startTimeUs == C.TIME_UNSET) { + checkState(currentFormat.subsampleOffsetUs == Format.OFFSET_SAMPLE_RELATIVE); + outputSampleTimeUs = timeUs; + } else if (currentFormat.subsampleOffsetUs == Format.OFFSET_SAMPLE_RELATIVE) { + outputSampleTimeUs = timeUs + cuesWithTiming.startTimeUs; + } else { + outputSampleTimeUs = cuesWithTiming.startTimeUs + currentFormat.subsampleOffsetUs; } + delegate.sampleMetadata( + outputSampleTimeUs, + flags, + cuesWithDurationBytes.length, + /* offset= */ 0, + /* cryptoData= */ null); } /**