diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java index bac9a777d2..9c187fd88a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java @@ -105,19 +105,27 @@ public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtrac } else if (Objects.equals(containerMimeType, MimeTypes.IMAGE_PNG)) { extractor = new PngExtractor(); } else { - int flags = 0; + @FragmentedMp4Extractor.Flags int flags = 0; if (enableEventMessageTrack) { flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK; } + if (subtitleParserFactory == null) { + flags |= FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA; + } extractor = new FragmentedMp4Extractor( + subtitleParserFactory != null + ? subtitleParserFactory + : SubtitleParser.Factory.UNSUPPORTED, flags, /* timestampAdjuster= */ null, /* sideloadedTrack= */ null, closedCaptionFormats, playerEmsgTrackOutput); } - if (subtitleParserFactory != null && !MimeTypes.isText(containerMimeType)) { + if (subtitleParserFactory != null + && !MimeTypes.isText(containerMimeType) + && !(extractor.getUnderlyingImplementation() instanceof FragmentedMp4Extractor)) { extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory); } return new BundledChunkExtractor(extractor, primaryTrackType, representationFormat); diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java index 9522197ff8..8ecd1d667e 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java @@ -39,6 +39,7 @@ import androidx.media3.extractor.ts.Ac4Extractor; import androidx.media3.extractor.ts.AdtsExtractor; import androidx.media3.extractor.ts.DefaultTsPayloadReaderFactory; import androidx.media3.extractor.ts.TsExtractor; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; import java.io.EOFException; import java.io.IOException; @@ -201,11 +202,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory { case FileTypes.MP3: return new Mp3Extractor(/* flags= */ 0, /* forcedFirstSampleTimestampUs= */ 0); case FileTypes.MP4: - Extractor mp4Extractor = - createFragmentedMp4Extractor(timestampAdjuster, format, muxedCaptionFormats); - return subtitleParserFactory != null - ? new SubtitleTranscodingExtractor(mp4Extractor, subtitleParserFactory) - : mp4Extractor; + return createFragmentedMp4Extractor( + subtitleParserFactory, timestampAdjuster, format, muxedCaptionFormats); case FileTypes.TS: Extractor tsExtractor = createTsExtractor( @@ -264,16 +262,25 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory { } private static FragmentedMp4Extractor createFragmentedMp4Extractor( + @Nullable SubtitleParser.Factory subtitleParserFactory, TimestampAdjuster timestampAdjuster, Format format, @Nullable List muxedCaptionFormats) { // Only enable the EMSG TrackOutput if this is the 'variant' track (i.e. the main one) to avoid // creating a separate EMSG track for every audio track in a video stream. + @FragmentedMp4Extractor.Flags + int flags = isFmp4Variant(format) ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0; + if (subtitleParserFactory == null) { + subtitleParserFactory = SubtitleParser.Factory.UNSUPPORTED; + flags |= FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA; + } return new FragmentedMp4Extractor( - /* flags= */ isFmp4Variant(format) ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0, + subtitleParserFactory, + flags, timestampAdjuster, /* sideloadedTrack= */ null, - muxedCaptionFormats != null ? muxedCaptionFormats : Collections.emptyList()); + muxedCaptionFormats != null ? muxedCaptionFormats : ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); } /** Returns true if this {@code format} represents a 'variant' track (i.e. the main one). */ diff --git a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java index ff376b58f6..c0ba6d26af 100644 --- a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java +++ b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java @@ -53,7 +53,7 @@ import androidx.media3.extractor.mp4.FragmentedMp4Extractor; import androidx.media3.extractor.mp4.Track; import androidx.media3.extractor.mp4.TrackEncryptionBox; import androidx.media3.extractor.text.SubtitleParser; -import androidx.media3.extractor.text.SubtitleTranscodingExtractor; +import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.List; @@ -173,15 +173,22 @@ public class DefaultSsChunkSource implements SsChunkSource { nalUnitLengthFieldLength, null, null); + @FragmentedMp4Extractor.Flags + int flags = + FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME + | FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX; Extractor extractor = new FragmentedMp4Extractor( - FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME - | FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX, + subtitleParserFactory == null + ? SubtitleParser.Factory.UNSUPPORTED + : subtitleParserFactory, + subtitleParserFactory == null + ? flags | FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA + : flags, /* timestampAdjuster= */ null, - track); - if (subtitleParserFactory != null) { - extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory); - } + track, + /* closedCaptionFormats= */ ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); chunkExtractors[i] = new BundledChunkExtractor(extractor, streamElement.type, format); } } diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java b/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java index d1f6b46bce..5400bbe23f 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java @@ -277,7 +277,7 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory { /** * Sets flags for {@link FragmentedMp4Extractor} instances created by the factory. * - * @see FragmentedMp4Extractor#FragmentedMp4Extractor(int) + * @see FragmentedMp4Extractor#FragmentedMp4Extractor(SubtitleParser.Factory, int) * @param flags The flags to use. * @return The factory, for convenience. */ @@ -436,10 +436,12 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory { } Extractor[] result = new Extractor[extractors.size()]; for (int i = 0; i < extractors.size(); i++) { + Extractor extractor = extractors.get(i); result[i] = textTrackTranscodingEnabled - ? new SubtitleTranscodingExtractor(extractors.get(i), subtitleParserFactory) - : extractors.get(i); + && !(extractor.getUnderlyingImplementation() instanceof FragmentedMp4Extractor) + ? new SubtitleTranscodingExtractor(extractor, subtitleParserFactory) + : extractor; } return result; } @@ -500,7 +502,13 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory { : 0))); break; case FileTypes.MP4: - extractors.add(new FragmentedMp4Extractor(fragmentedMp4Flags)); + extractors.add( + new FragmentedMp4Extractor( + subtitleParserFactory, + fragmentedMp4Flags + | (textTrackTranscodingEnabled + ? 0 + : FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA))); extractors.add(new Mp4Extractor(mp4Flags)); break; case FileTypes.OGG: diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/Extractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/Extractor.java index aa89ba46c6..3322cf69ff 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/Extractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/Extractor.java @@ -25,6 +25,7 @@ import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import org.checkerframework.dataflow.qual.SideEffectFree; /** Extracts media data from a container format. */ @UnstableApi @@ -130,6 +131,7 @@ public interface Extractor { *

{@code Extractor} implementations that operate by delegating to another {@code Extractor} * should override this method to return that delegate. */ + @SideEffectFree default Extractor getUnderlyingImplementation() { return this; } diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java index 9c56828a20..cde4f8ab2e 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java @@ -54,6 +54,9 @@ import androidx.media3.extractor.metadata.emsg.EventMessage; import androidx.media3.extractor.metadata.emsg.EventMessageEncoder; import androidx.media3.extractor.mp4.Atom.ContainerAtom; import androidx.media3.extractor.mp4.Atom.LeafAtom; +import androidx.media3.extractor.text.SubtitleParser; +import androidx.media3.extractor.text.SubtitleTranscodingExtractorOutput; +import com.google.common.collect.ImmutableList; import java.io.IOException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; @@ -71,10 +74,21 @@ import java.util.UUID; @UnstableApi public class FragmentedMp4Extractor implements Extractor { - /** Factory for {@link FragmentedMp4Extractor} instances. */ + /** + * @deprecated Use {@link #newFactory(SubtitleParser.Factory)} instead. + */ + @Deprecated public static final ExtractorsFactory FACTORY = () -> new Extractor[] {new FragmentedMp4Extractor()}; + /** + * Creates a factory for {@link FragmentedMp4Extractor} instances with the provided {@link + * SubtitleParser.Factory}. + */ + public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParserFactory) { + return () -> new Extractor[] {new FragmentedMp4Extractor(subtitleParserFactory)}; + } + /** * Flags controlling the behavior of the extractor. Possible flag values are {@link * #FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME}, {@link #FLAG_WORKAROUND_IGNORE_TFDT_BOX}, @@ -89,7 +103,8 @@ public class FragmentedMp4Extractor implements Extractor { FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME, FLAG_WORKAROUND_IGNORE_TFDT_BOX, FLAG_ENABLE_EMSG_TRACK, - FLAG_WORKAROUND_IGNORE_EDIT_LISTS + FLAG_WORKAROUND_IGNORE_EDIT_LISTS, + FLAG_EMIT_RAW_SUBTITLE_DATA }) public @interface Flags {} @@ -114,6 +129,12 @@ public class FragmentedMp4Extractor implements Extractor { /** Flag to ignore any edit lists in the stream. */ public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 1 << 4; // 16 + /** + * Flag to use the source subtitle formats without modification. If unset, subtitles will be + * transcoded to {@link MimeTypes#APPLICATION_MEDIA3_CUES} during extraction. + */ + public static final int FLAG_EMIT_RAW_SUBTITLE_DATA = 1 << 5; // 32 + private static final String TAG = "FragmentedMp4Extractor"; @SuppressWarnings("ConstantCaseForConstants") @@ -134,7 +155,7 @@ public class FragmentedMp4Extractor implements Extractor { private static final int STATE_READING_SAMPLE_START = 3; private static final int STATE_READING_SAMPLE_CONTINUE = 4; - // Workarounds. + private final SubtitleParser.Factory subtitleParserFactory; private final @Flags int flags; @Nullable private final Track sideloadedTrack; @@ -187,53 +208,113 @@ public class FragmentedMp4Extractor implements Extractor { // Whether extractorOutput.seekMap has been called. private boolean haveOutputSeekMap; + /** + * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory)} instead + */ + @Deprecated public FragmentedMp4Extractor() { - this(0); + this( + SubtitleParser.Factory.UNSUPPORTED, + /* flags= */ FLAG_EMIT_RAW_SUBTITLE_DATA, + /* timestampAdjuster= */ null, + /* sideloadedTrack= */ null, + /* closedCaptionFormats= */ ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); } /** - * @param flags Flags that control the extractor's behavior. + * Constructs an instance. + * + * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during + * extraction. */ + public FragmentedMp4Extractor(SubtitleParser.Factory subtitleParserFactory) { + this( + subtitleParserFactory, + /* flags= */ 0, + /* timestampAdjuster= */ null, + /* sideloadedTrack= */ null, + /* closedCaptionFormats= */ ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); + } + + /** + * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int)} instead + */ + @Deprecated public FragmentedMp4Extractor(@Flags int flags) { - this(flags, /* timestampAdjuster= */ null); + this( + SubtitleParser.Factory.UNSUPPORTED, + flags | FLAG_EMIT_RAW_SUBTITLE_DATA, + /* timestampAdjuster= */ null, + /* sideloadedTrack= */ null, + /* closedCaptionFormats= */ ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); } /** + * Constructs an instance. + * + * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during + * extraction. * @param flags Flags that control the extractor's behavior. - * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed. */ + public FragmentedMp4Extractor(SubtitleParser.Factory subtitleParserFactory, @Flags int flags) { + this( + subtitleParserFactory, + flags, + /* timestampAdjuster= */ null, + /* sideloadedTrack= */ null, + /* closedCaptionFormats= */ ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); + } + + /** + * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int, TimestampAdjuster, + * Track, List, TrackOutput)} instead + */ + @Deprecated public FragmentedMp4Extractor(@Flags int flags, @Nullable TimestampAdjuster timestampAdjuster) { - this(flags, timestampAdjuster, /* sideloadedTrack= */ null, Collections.emptyList()); + this( + SubtitleParser.Factory.UNSUPPORTED, + flags | FLAG_EMIT_RAW_SUBTITLE_DATA, + timestampAdjuster, + /* sideloadedTrack= */ null, + /* closedCaptionFormats= */ ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); } /** - * @param flags Flags that control the extractor's behavior. - * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed. - * @param sideloadedTrack Sideloaded track information, in the case that the extractor will not - * receive a moov box in the input data. Null if a moov box is expected. + * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int, TimestampAdjuster, + * Track, List, TrackOutput)} instead */ + @Deprecated public FragmentedMp4Extractor( @Flags int flags, @Nullable TimestampAdjuster timestampAdjuster, @Nullable Track sideloadedTrack) { - this(flags, timestampAdjuster, sideloadedTrack, Collections.emptyList()); + this( + SubtitleParser.Factory.UNSUPPORTED, + flags | FLAG_EMIT_RAW_SUBTITLE_DATA, + timestampAdjuster, + sideloadedTrack, + /* closedCaptionFormats= */ ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); } /** - * @param flags Flags that control the extractor's behavior. - * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed. - * @param sideloadedTrack Sideloaded track information, in the case that the extractor will not - * receive a moov box in the input data. Null if a moov box is expected. - * @param closedCaptionFormats For tracks that contain SEI messages, the formats of the closed - * caption channels to expose. + * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int, TimestampAdjuster, + * Track, List, TrackOutput)} instead */ + @Deprecated public FragmentedMp4Extractor( @Flags int flags, @Nullable TimestampAdjuster timestampAdjuster, @Nullable Track sideloadedTrack, List closedCaptionFormats) { this( - flags, + SubtitleParser.Factory.UNSUPPORTED, + flags | FLAG_EMIT_RAW_SUBTITLE_DATA, timestampAdjuster, sideloadedTrack, closedCaptionFormats, @@ -241,6 +322,30 @@ public class FragmentedMp4Extractor implements Extractor { } /** + * @deprecated Use {@link #FragmentedMp4Extractor(SubtitleParser.Factory, int, TimestampAdjuster, + * Track, List, TrackOutput)} instead + */ + @Deprecated + public FragmentedMp4Extractor( + @Flags int flags, + @Nullable TimestampAdjuster timestampAdjuster, + @Nullable Track sideloadedTrack, + List closedCaptionFormats, + @Nullable TrackOutput additionalEmsgTrackOutput) { + this( + SubtitleParser.Factory.UNSUPPORTED, + flags | FLAG_EMIT_RAW_SUBTITLE_DATA, + timestampAdjuster, + sideloadedTrack, + closedCaptionFormats, + additionalEmsgTrackOutput); + } + + /** + * Constructs an instance. + * + * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during + * extraction. * @param flags Flags that control the extractor's behavior. * @param timestampAdjuster Adjusts sample timestamps. May be null if no adjustment is needed. * @param sideloadedTrack Sideloaded track information, in the case that the extractor will not @@ -252,11 +357,13 @@ public class FragmentedMp4Extractor implements Extractor { * handling of emsg messages for players is not required. */ public FragmentedMp4Extractor( + SubtitleParser.Factory subtitleParserFactory, @Flags int flags, @Nullable TimestampAdjuster timestampAdjuster, @Nullable Track sideloadedTrack, List closedCaptionFormats, @Nullable TrackOutput additionalEmsgTrackOutput) { + this.subtitleParserFactory = subtitleParserFactory; this.flags = flags; this.timestampAdjuster = timestampAdjuster; this.sideloadedTrack = sideloadedTrack; @@ -287,7 +394,10 @@ public class FragmentedMp4Extractor implements Extractor { @Override public void init(ExtractorOutput output) { - extractorOutput = output; + extractorOutput = + (flags & FLAG_EMIT_RAW_SUBTITLE_DATA) == 0 + ? new SubtitleTranscodingExtractorOutput(output, subtitleParserFactory) + : output; enterReadingAtomHeaderState(); initExtraTracks(); if (sideloadedTrack != null) { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java index 82a7262325..d040408e2e 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java @@ -37,6 +37,26 @@ public interface SubtitleParser { /** Factory for {@link SubtitleParser} instances. */ interface Factory { + /** A subtitle parser factory that supports no formats. */ + public static final Factory UNSUPPORTED = + new Factory() { + @Override + public boolean supportsFormat(Format format) { + return false; + } + + @Override + public @CueReplacementBehavior int getCueReplacementBehavior(Format format) { + return Format.CUE_REPLACEMENT_BEHAVIOR_MERGE; + } + + @Override + public SubtitleParser create(Format format) { + throw new IllegalStateException( + "This SubtitleParser.Factory doesn't support any formats."); + } + }; + /** * Returns whether the factory is able to instantiate a {@link SubtitleParser} for the given * {@link Format}. diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractor.java index 4157a9e442..fc21c56fe8 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractor.java @@ -44,6 +44,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * SubtitleParser.Factory#supportsFormat(Format)} on the {@link SubtitleParser.Factory} passed to * the constructor of this class. */ +// TODO: b/318679808 - deprecate when all subtitle-related Extractors use +// SubtitleTranscodingExtractorOutput instead. @UnstableApi public class SubtitleTranscodingExtractor implements Extractor { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractorOutput.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractorOutput.java index ae0cc9394c..faff23b5e3 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractorOutput.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingExtractorOutput.java @@ -18,12 +18,29 @@ package androidx.media3.extractor.text; import android.util.SparseArray; import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.UnstableApi; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.SeekMap; import androidx.media3.extractor.TrackOutput; -/** A wrapping {@link ExtractorOutput} for use by {@link SubtitleTranscodingExtractor}. */ -/* package */ class SubtitleTranscodingExtractorOutput implements ExtractorOutput { +/** + * A wrapping {@link ExtractorOutput} that transcodes {@linkplain C#TRACK_TYPE_TEXT text samples} + * from supported subtitle formats to {@link MimeTypes#APPLICATION_MEDIA3_CUES}. + * + *

The resulting {@link MimeTypes#APPLICATION_MEDIA3_CUES} samples are emitted to the underlying + * {@link TrackOutput}. + * + *

For non-text tracks (i.e. where {@link C.TrackType} is not {@code C.TRACK_TYPE_TEXT}), samples + * are passed through to the underlying {@link TrackOutput} without modification. + * + *

Support for subtitle formats is determined by {@link + * SubtitleParser.Factory#supportsFormat(Format)} on the {@link SubtitleParser.Factory} passed to + * the constructor of this class. + */ +@UnstableApi +public final class SubtitleTranscodingExtractorOutput implements ExtractorOutput { private final ExtractorOutput delegate; private final SubtitleParser.Factory subtitleParserFactory; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java index a61d274a5c..4122261cda 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java @@ -38,7 +38,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * MimeTypes#APPLICATION_SUBRIP} to ExoPlayer's internal binary cue representation ({@link * MimeTypes#APPLICATION_MEDIA3_CUES}). */ -/* package */ class SubtitleTranscodingTrackOutput implements TrackOutput { +/* package */ final class SubtitleTranscodingTrackOutput implements TrackOutput { private final TrackOutput delegate; private final SubtitleParser.Factory subtitleParserFactory; diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java index 3564027dff..205698f9bd 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java @@ -178,7 +178,7 @@ public final class DefaultExtractorsFactoryTest { assertThat(aviExtractor).isInstanceOf(SubtitleTranscodingExtractor.class); assertThat(matroskaExtractor).isInstanceOf(SubtitleTranscodingExtractor.class); assertThat(mp4Extractor).isInstanceOf(SubtitleTranscodingExtractor.class); - assertThat(fragmentedMp4Extractor).isInstanceOf(SubtitleTranscodingExtractor.class); + assertThat(fragmentedMp4Extractor).isNotInstanceOf(SubtitleTranscodingExtractor.class); assertThat(tsExtractor).isInstanceOf(SubtitleTranscodingExtractor.class); } diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNoSniffingTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNoSniffingTest.java index fac30af44f..fc93fa76fa 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNoSniffingTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorNoSniffingTest.java @@ -15,10 +15,15 @@ */ package androidx.media3.extractor.mp4; +import static androidx.media3.extractor.mp4.FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA; + import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.extractor.text.DefaultSubtitleParserFactory; +import androidx.media3.extractor.text.SubtitleParser; import androidx.media3.test.utils.ExtractorAsserts; +import com.google.common.collect.ImmutableList; import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; @@ -32,6 +37,21 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; @RunWith(ParameterizedRobolectricTestRunner.class) public class FragmentedMp4ExtractorNoSniffingTest { + private static final String FMP4_SIDELOADED = "media/mp4/sample_fragmented_sideloaded_track.mp4"; + private static final Track SIDELOADED_TRACK = + new Track( + /* id= */ 1, + /* type= */ C.TRACK_TYPE_VIDEO, + /* timescale= */ 30_000, + /* movieTimescale= */ 1000, + /* durationUs= */ C.TIME_UNSET, + new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H264).build(), + /* sampleTransformation= */ Track.TRANSFORMATION_NONE, + /* sampleDescriptionEncryptionBoxes= */ null, + /* nalUnitLengthFieldLength= */ 4, + /* editListDurations= */ null, + /* editListMediaTimes= */ null); + @Parameters(name = "{0}") public static List params() { return ExtractorAsserts.configsNoSniffing(); @@ -40,27 +60,35 @@ public class FragmentedMp4ExtractorNoSniffingTest { @Parameter public ExtractorAsserts.SimulationConfig simulationConfig; @Test - public void sampleWithSideLoadedTrack() throws Exception { + public void sampleWithSideLoadedTrack_subtitlesParsedDuringDecoding() throws Exception { // Sideloaded tracks are generally used in Smooth Streaming, where the MP4 files do not contain // any ftyp box and are not sniffed. - Track sideloadedTrack = - new Track( - /* id= */ 1, - /* type= */ C.TRACK_TYPE_VIDEO, - /* timescale= */ 30_000, - /* movieTimescale= */ 1000, - /* durationUs= */ C.TIME_UNSET, - new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H264).build(), - /* sampleTransformation= */ Track.TRANSFORMATION_NONE, - /* sampleDescriptionEncryptionBoxes= */ null, - /* nalUnitLengthFieldLength= */ 4, - /* editListDurations= */ null, - /* editListMediaTimes= */ null); ExtractorAsserts.assertBehavior( () -> - new FragmentedMp4Extractor( - /* flags= */ 0, /* timestampAdjuster= */ null, sideloadedTrack), - "media/mp4/sample_fragmented_sideloaded_track.mp4", + createFragmentedMp4Extractor( + SubtitleParser.Factory.UNSUPPORTED, FLAG_EMIT_RAW_SUBTITLE_DATA), + FMP4_SIDELOADED, simulationConfig); } + + @Test + public void sampleWithSideLoadedTrack_subtitlesParsedDuringExtraction() throws Exception { + // Sideloaded tracks are generally used in Smooth Streaming, where the MP4 files do not contain + // any ftyp box and are not sniffed. + ExtractorAsserts.assertBehavior( + () -> createFragmentedMp4Extractor(new DefaultSubtitleParserFactory(), /* flags= */ 0), + FMP4_SIDELOADED, + simulationConfig); + } + + private FragmentedMp4Extractor createFragmentedMp4Extractor( + SubtitleParser.Factory subtitleParserFactory, @FragmentedMp4Extractor.Flags int flags) { + return new FragmentedMp4Extractor( + subtitleParserFactory, + flags, + /* timestampAdjuster= */ null, + SIDELOADED_TRACK, + /* closedCaptionFormats= */ ImmutableList.of(), + /* additionalEmsgTrackOutput= */ null); + } } diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorTest.java index ca42f0b1cc..8bbf75ba00 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorTest.java @@ -15,11 +15,16 @@ */ package androidx.media3.extractor.mp4; +import static androidx.media3.extractor.mp4.FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA; + import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.extractor.text.DefaultSubtitleParserFactory; +import androidx.media3.extractor.text.SubtitleParser; import androidx.media3.test.utils.ExtractorAsserts; import androidx.media3.test.utils.ExtractorAsserts.ExtractorFactory; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.junit.Test; @@ -32,17 +37,27 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; @RunWith(ParameterizedRobolectricTestRunner.class) public final class FragmentedMp4ExtractorTest { - @Parameters(name = "{0}") - public static ImmutableList params() { - return ExtractorAsserts.configs(); + @Parameters(name = "{0},subtitlesParsedDuringExtraction={1}") + public static List params() { + List parameterList = new ArrayList<>(); + for (ExtractorAsserts.SimulationConfig config : ExtractorAsserts.configs()) { + parameterList.add(new Object[] {config, /* subtitlesParsedDuringExtraction */ true}); + parameterList.add(new Object[] {config, /* subtitlesParsedDuringExtraction */ false}); + } + return parameterList; } - @Parameter public ExtractorAsserts.SimulationConfig simulationConfig; + @Parameter(0) + public ExtractorAsserts.SimulationConfig simulationConfig; + + @Parameter(1) + public boolean subtitlesParsedDuringExtraction; @Test public void sample() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_fragmented.mp4", simulationConfig); } @@ -50,7 +65,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleSeekable() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_fragmented_seekable.mp4", simulationConfig); } @@ -58,18 +74,21 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithSeiPayloadParsing() throws Exception { // Enabling the CEA-608 track enables SEI payload parsing. - ExtractorFactory extractorFactory = - getExtractorFactory( - Collections.singletonList( - new Format.Builder().setSampleMimeType(MimeTypes.APPLICATION_CEA608).build())); + List closedCaptions = + Collections.singletonList( + new Format.Builder().setSampleMimeType(MimeTypes.APPLICATION_CEA608).build()); + ExtractorAsserts.assertBehavior( - extractorFactory, "media/mp4/sample_fragmented_sei.mp4", simulationConfig); + getExtractorFactory(closedCaptions, subtitlesParsedDuringExtraction), + "media/mp4/sample_fragmented_sei.mp4", + simulationConfig); } @Test public void sampleWithAc3Track() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_ac3_fragmented.mp4", simulationConfig); } @@ -77,7 +96,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithAc4Track() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_ac4_fragmented.mp4", simulationConfig); } @@ -85,7 +105,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithProtectedAc4Track() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_ac4_protected.mp4", simulationConfig); } @@ -93,7 +114,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithEac3Track() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_eac3_fragmented.mp4", simulationConfig); } @@ -101,7 +123,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithEac3jocTrack() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_eac3joc_fragmented.mp4", simulationConfig); } @@ -109,7 +132,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithOpusTrack() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_opus_fragmented.mp4", simulationConfig); } @@ -117,7 +141,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void samplePartiallyFragmented() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_partially_fragmented.mp4", simulationConfig); } @@ -126,7 +151,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithLargeBitrates() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_fragmented_large_bitrates.mp4", simulationConfig); } @@ -134,7 +160,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithMhm1BlCicp1Track() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_mhm1_bl_cicp1_fragmented.mp4", simulationConfig); } @@ -142,7 +169,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithMhm1LcblCicp1Track() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_mhm1_lcbl_cicp1_fragmented.mp4", simulationConfig); } @@ -150,7 +178,8 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithMhm1BlConfigChangeTrack() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_mhm1_bl_configchange_fragmented.mp4", simulationConfig); } @@ -158,17 +187,31 @@ public final class FragmentedMp4ExtractorTest { @Test public void sampleWithMhm1LcblConfigChangeTrack() throws Exception { ExtractorAsserts.assertBehavior( - getExtractorFactory(ImmutableList.of()), + getExtractorFactory( + /* closedCaptionFormats= */ ImmutableList.of(), subtitlesParsedDuringExtraction), "media/mp4/sample_mhm1_lcbl_configchange_fragmented.mp4", simulationConfig); } - private static ExtractorFactory getExtractorFactory(final List closedCaptionFormats) { + private static ExtractorFactory getExtractorFactory( + List closedCaptionFormats, boolean subtitlesParsedDuringExtraction) { + SubtitleParser.Factory subtitleParserFactory; + @FragmentedMp4Extractor.Flags int flags; + if (subtitlesParsedDuringExtraction) { + subtitleParserFactory = new DefaultSubtitleParserFactory(); + flags = 0; + } else { + subtitleParserFactory = SubtitleParser.Factory.UNSUPPORTED; + flags = FLAG_EMIT_RAW_SUBTITLE_DATA; + } + return () -> new FragmentedMp4Extractor( - /* flags= */ 0, + subtitleParserFactory, + flags, /* timestampAdjuster= */ null, /* sideloadedTrack= */ null, - closedCaptionFormats); + closedCaptionFormats, + /* additionalEmsgTrackOutput= */ null); } }