This may only be used with {@link BundledChunkExtractor.Factory}. + */ + /* package */ Factory setSubtitleParserFactory( + @Nullable SubtitleParser.Factory subtitleParserFactory) { + this.subtitleParserFactory = subtitleParserFactory; + return this; + } + @Override public SsChunkSource createChunkSource( LoaderErrorThrower manifestLoaderErrorThrower, @@ -83,7 +99,8 @@ public class DefaultSsChunkSource implements SsChunkSource { streamElementIndex, trackSelection, dataSource, - cmcdConfiguration); + cmcdConfiguration, + subtitleParserFactory); } } @@ -112,6 +129,8 @@ public class DefaultSsChunkSource implements SsChunkSource { * @param trackSelection The track selection. * @param dataSource A {@link DataSource} suitable for loading the media data. * @param cmcdConfiguration The {@link CmcdConfiguration} for this chunk source. + * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during + * extraction. */ public DefaultSsChunkSource( LoaderErrorThrower manifestLoaderErrorThrower, @@ -119,7 +138,8 @@ public class DefaultSsChunkSource implements SsChunkSource { int streamElementIndex, ExoTrackSelection trackSelection, DataSource dataSource, - @Nullable CmcdConfiguration cmcdConfiguration) { + @Nullable CmcdConfiguration cmcdConfiguration, + @Nullable SubtitleParser.Factory subtitleParserFactory) { this.manifestLoaderErrorThrower = manifestLoaderErrorThrower; this.manifest = manifest; this.streamElementIndex = streamElementIndex; @@ -152,12 +172,15 @@ public class DefaultSsChunkSource implements SsChunkSource { nalUnitLengthFieldLength, null, null); - FragmentedMp4Extractor extractor = + Extractor extractor = new FragmentedMp4Extractor( FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME | FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX, /* timestampAdjuster= */ null, track); + if (subtitleParserFactory != null) { + extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory); + } chunkExtractors[i] = new BundledChunkExtractor(extractor, streamElement.type, format); } } diff --git a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java index 07d8f90d91..759e9e86e2 100644 --- a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java +++ b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.common.StreamKey; import androidx.media3.common.TrackGroup; import androidx.media3.common.util.NullableType; @@ -41,6 +42,7 @@ import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; +import androidx.media3.extractor.text.SubtitleParser; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -77,7 +79,8 @@ import java.util.List; LoadErrorHandlingPolicy loadErrorHandlingPolicy, MediaSourceEventListener.EventDispatcher mediaSourceEventDispatcher, LoaderErrorThrower manifestLoaderErrorThrower, - Allocator allocator) { + Allocator allocator, + @Nullable SubtitleParser.Factory subtitleParserFactory) { this.manifest = manifest; this.chunkSourceFactory = chunkSourceFactory; this.transferListener = transferListener; @@ -89,7 +92,7 @@ import java.util.List; this.mediaSourceEventDispatcher = mediaSourceEventDispatcher; this.allocator = allocator; this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory; - trackGroups = buildTrackGroups(manifest, drmSessionManager); + trackGroups = buildTrackGroups(manifest, drmSessionManager, subtitleParserFactory); sampleStreams = newSampleStreamArray(0); compositeSequenceableLoader = compositeSequenceableLoaderFactory.createCompositeSequenceableLoader(sampleStreams); @@ -265,15 +268,32 @@ import java.util.List; } private static TrackGroupArray buildTrackGroups( - SsManifest manifest, DrmSessionManager drmSessionManager) { + SsManifest manifest, + DrmSessionManager drmSessionManager, + @Nullable SubtitleParser.Factory subtitleParserFactory) { TrackGroup[] trackGroups = new TrackGroup[manifest.streamElements.length]; for (int i = 0; i < manifest.streamElements.length; i++) { Format[] manifestFormats = manifest.streamElements[i].formats; Format[] exposedFormats = new Format[manifestFormats.length]; for (int j = 0; j < manifestFormats.length; j++) { Format manifestFormat = manifestFormats[j]; - exposedFormats[j] = - manifestFormat.copyWithCryptoType(drmSessionManager.getCryptoType(manifestFormat)); + Format.Builder updatedFormat = + manifestFormat + .buildUpon() + .setCryptoType(drmSessionManager.getCryptoType(manifestFormat)); + if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(manifestFormat)) { + updatedFormat + .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) + .setCueReplacementBehavior( + subtitleParserFactory.getCueReplacementBehavior(manifestFormat)) + .setCodecs( + manifestFormat.sampleMimeType + + (manifestFormat.codecs != null ? " " + manifestFormat.codecs : "")) + // Reset this value to the default. All non-default timestamp adjustments are done + // by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding. + .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE); + } + exposedFormats[j] = updatedFormat.build(); } trackGroups[i] = new TrackGroup(/* id= */ Integer.toString(i), exposedFormats); } diff --git a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java index 5aa8069030..5b51c185a0 100644 --- a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java +++ b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java @@ -65,6 +65,8 @@ import androidx.media3.exoplayer.upstream.Loader; import androidx.media3.exoplayer.upstream.Loader.LoadErrorAction; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; import androidx.media3.exoplayer.upstream.ParsingLoadable; +import androidx.media3.extractor.text.DefaultSubtitleParserFactory; +import androidx.media3.extractor.text.SubtitleParser; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; @@ -91,6 +93,7 @@ public final class SsMediaSource extends BaseMediaSource @Nullable private CmcdConfiguration.Factory cmcdConfigurationFactory; private DrmSessionManagerProvider drmSessionManagerProvider; private LoadErrorHandlingPolicy loadErrorHandlingPolicy; + @Nullable private SubtitleParser.Factory subtitleParserFactory; private long livePresentationDelayMs; @Nullable private ParsingLoadable.Parser manifestParser; @@ -152,6 +155,40 @@ public final class SsMediaSource extends BaseMediaSource return this; } + /** + * Sets whether subtitles should be parsed as part of extraction (before being added to the + * sample queue) or as part of rendering (when being taken from the sample queue). Defaults to + * {@code false} (i.e. subtitles will be parsed as part of rendering). + * + *
This method is experimental. Its default value may change, or it may be renamed or removed + * in a future release. + * + *
This method may only be used with {@link DefaultSsChunkSource.Factory}. + * + * @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or + * rendering. + * @return This factory, for convenience. + */ + // TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on + // DefaultMediaSourceFactory via the MediaSource.Factory interface). + public Factory experimentalParseSubtitlesDuringExtraction( + boolean parseSubtitlesDuringExtraction) { + if (parseSubtitlesDuringExtraction) { + if (subtitleParserFactory == null) { + this.subtitleParserFactory = new DefaultSubtitleParserFactory(); + } + } else { + this.subtitleParserFactory = null; + } + if (chunkSourceFactory instanceof DefaultSsChunkSource.Factory) { + ((DefaultSsChunkSource.Factory) chunkSourceFactory) + .setSubtitleParserFactory(subtitleParserFactory); + } else { + throw new IllegalStateException(); + } + return this; + } + /** * Sets the duration in milliseconds by which the default start position should precede the end * of the live window for live playbacks. The default value is {@link @@ -273,6 +310,7 @@ public final class SsMediaSource extends BaseMediaSource cmcdConfiguration, drmSessionManagerProvider.get(mediaItem), loadErrorHandlingPolicy, + subtitleParserFactory, livePresentationDelayMs); } @@ -310,6 +348,7 @@ public final class SsMediaSource extends BaseMediaSource cmcdConfiguration, drmSessionManagerProvider.get(mediaItem), loadErrorHandlingPolicy, + subtitleParserFactory, livePresentationDelayMs); } @@ -353,6 +392,7 @@ public final class SsMediaSource extends BaseMediaSource private long manifestLoadStartTimestamp; private SsManifest manifest; private Handler manifestRefreshHandler; + @Nullable private final SubtitleParser.Factory subtitleParserFactory; @GuardedBy("this") private MediaItem mediaItem; @@ -367,6 +407,7 @@ public final class SsMediaSource extends BaseMediaSource @Nullable CmcdConfiguration cmcdConfiguration, DrmSessionManager drmSessionManager, LoadErrorHandlingPolicy loadErrorHandlingPolicy, + @Nullable SubtitleParser.Factory subtitleParserFactory, long livePresentationDelayMs) { Assertions.checkState(manifest == null || !manifest.isLive); this.mediaItem = mediaItem; @@ -383,6 +424,7 @@ public final class SsMediaSource extends BaseMediaSource this.cmcdConfiguration = cmcdConfiguration; this.drmSessionManager = drmSessionManager; this.loadErrorHandlingPolicy = loadErrorHandlingPolicy; + this.subtitleParserFactory = subtitleParserFactory; this.livePresentationDelayMs = livePresentationDelayMs; this.manifestEventDispatcher = createEventDispatcher(/* mediaPeriodId= */ null); sideloadedManifest = manifest != null; @@ -450,7 +492,8 @@ public final class SsMediaSource extends BaseMediaSource loadErrorHandlingPolicy, mediaSourceEventDispatcher, manifestLoaderErrorThrower, - allocator); + allocator, + subtitleParserFactory); mediaPeriods.add(period); return period; } diff --git a/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSourceTest.java b/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSourceTest.java index 70ada1a5d2..e7015aaa17 100644 --- a/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSourceTest.java +++ b/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSourceTest.java @@ -297,6 +297,7 @@ public class DefaultSsChunkSourceTest { /* streamElementIndex= */ 0, adaptiveTrackSelection, new FakeDataSource(), - cmcdConfiguration); + cmcdConfiguration, + /* subtitleParserFactory= */ null); } } diff --git a/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java b/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java index 5123231ae8..865732ff01 100644 --- a/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java +++ b/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java @@ -17,6 +17,7 @@ package androidx.media3.exoplayer.smoothstreaming; import static androidx.media3.exoplayer.smoothstreaming.SsTestUtils.createSsManifest; import static androidx.media3.exoplayer.smoothstreaming.SsTestUtils.createStreamElement; +import static com.google.common.truth.Truth.assertThat; import static org.mockito.Mockito.mock; import androidx.media3.common.C; @@ -32,6 +33,9 @@ import androidx.media3.exoplayer.source.MediaSourceEventListener; import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; +import androidx.media3.extractor.text.DefaultSubtitleParserFactory; +import androidx.media3.extractor.text.SubtitleParser; +import androidx.media3.test.utils.FakeTimeline; import androidx.media3.test.utils.MediaPeriodAsserts; import androidx.media3.test.utils.MediaPeriodAsserts.FilterableManifestMediaPeriodFactory; import androidx.test.ext.junit.runners.AndroidJUnit4; @@ -75,13 +79,75 @@ public class SsMediaPeriodTest { new MediaSourceEventListener.EventDispatcher() .withParameters(/* windowIndex= */ 0, mediaPeriodId), mock(LoaderErrorThrower.class), - mock(Allocator.class)); + mock(Allocator.class), + /* subtitleParserFactory= */ null); }; MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration( mediaPeriodFactory, testManifest); } + @Test + public void getTrackGroups_withSubtitleParserFactory_matchesFormat() { + SubtitleParser.Factory subtitleParserFactory = new DefaultSubtitleParserFactory(); + + Format originalSubtitleFormat = + new Format.Builder() + .setContainerMimeType(MimeTypes.APPLICATION_MP4) + .setSampleMimeType(MimeTypes.TEXT_VTT) + .setLanguage("eng") + .build(); + Format expectedSubtitleFormat = + new Format.Builder() + .setContainerMimeType(originalSubtitleFormat.containerMimeType) + .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) + .setCodecs(originalSubtitleFormat.sampleMimeType) + .setCueReplacementBehavior( + subtitleParserFactory.getCueReplacementBehavior(originalSubtitleFormat)) + .setLanguage(originalSubtitleFormat.language) + .build(); + + SsManifest testManifest = + createSsManifest( + createStreamElement( + /* name= */ "video", + C.TRACK_TYPE_VIDEO, + createVideoFormat(/* bitrate= */ 200000), + createVideoFormat(/* bitrate= */ 400000), + createVideoFormat(/* bitrate= */ 800000)), + createStreamElement( + /* name= */ "audio", + C.TRACK_TYPE_AUDIO, + createAudioFormat(/* bitrate= */ 48000), + createAudioFormat(/* bitrate= */ 96000)), + createStreamElement(/* name= */ "text", C.TRACK_TYPE_TEXT, originalSubtitleFormat)); + + MediaPeriodId mediaPeriodId = + new MediaPeriodId( + new FakeTimeline(/* windowCount= */ 2).getUidOfPeriod(/* periodIndex= */ 0), + /* windowSequenceNumber= */ 0); + + SsMediaPeriod period = + new SsMediaPeriod( + testManifest, + mock(SsChunkSource.Factory.class), + mock(TransferListener.class), + mock(CompositeSequenceableLoaderFactory.class), + /* cmcdConfiguration= */ null, + mock(DrmSessionManager.class), + new DrmSessionEventListener.EventDispatcher() + .withParameters(/* windowIndex= */ 0, mediaPeriodId), + mock(LoadErrorHandlingPolicy.class), + new MediaSourceEventListener.EventDispatcher() + .withParameters(/* windowIndex= */ 0, mediaPeriodId), + mock(LoaderErrorThrower.class), + mock(Allocator.class), + subtitleParserFactory); + + Format subtitleFormat = period.getTrackGroups().get(2).getFormat(0); + assertThat(subtitleFormat).isEqualTo(expectedSubtitleFormat); + } + private static Format createVideoFormat(int bitrate) { return new Format.Builder() .setContainerMimeType(MimeTypes.VIDEO_MP4) diff --git a/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSourceTest.java b/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSourceTest.java index 34f9cad303..bfacb484ae 100644 --- a/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSourceTest.java +++ b/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSourceTest.java @@ -16,16 +16,24 @@ package androidx.media3.exoplayer.smoothstreaming; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; +import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.MediaItem; import androidx.media3.common.StreamKey; import androidx.media3.common.Timeline; import androidx.media3.datasource.ByteArrayDataSource; import androidx.media3.datasource.DataSource; +import androidx.media3.datasource.TransferListener; import androidx.media3.exoplayer.analytics.PlayerId; +import androidx.media3.exoplayer.smoothstreaming.manifest.SsManifest; import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.exoplayer.trackselection.ExoTrackSelection; +import androidx.media3.exoplayer.upstream.CmcdConfiguration; +import androidx.media3.exoplayer.upstream.LoaderErrorThrower; +import androidx.media3.test.utils.FakeDataSource; import androidx.media3.test.utils.TestUtil; import androidx.media3.test.utils.robolectric.RobolectricUtil; import androidx.test.core.app.ApplicationProvider; @@ -94,6 +102,27 @@ public class SsMediaSourceTest { assertThat(canUpdateMediaItem).isFalse(); } + @Test + public void + setExperimentalParseSubtitlesDuringExtraction_withNonDefaultChunkSourceFactory_setThrows() { + SsMediaSource.Factory ssMediaSourceFactory = + new SsMediaSource.Factory( + /* chunkSourceFactory= */ this::createSampleSsChunkSource, + /* manifestDataSourceFactory= */ () -> createSampleDataSource(SAMPLE_MANIFEST)); + assertThrows( + IllegalStateException.class, + () -> ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(false)); + } + + @Test + public void + setExperimentalParseSubtitlesDuringExtraction_withDefaultChunkSourceFactory_setSucceeds() { + SsMediaSource.Factory ssMediaSourceFactory = + new SsMediaSource.Factory(() -> createSampleDataSource(SAMPLE_MANIFEST)); + ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(false); + ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(true); + } + @Test public void canUpdateMediaItem_withChangedStreamKeys_returnsFalse() { MediaItem initialMediaItem = @@ -175,4 +204,21 @@ public class SsMediaSourceTest { } return new ByteArrayDataSource(manifestData); } + + private SsChunkSource createSampleSsChunkSource( + LoaderErrorThrower manifestLoaderErrorThrower, + SsManifest manifest, + int streamElementIndex, + ExoTrackSelection trackSelection, + @Nullable TransferListener transferListener, + @Nullable CmcdConfiguration cmcdConfiguration) { + return new DefaultSsChunkSource( + manifestLoaderErrorThrower, + manifest, + streamElementIndex, + trackSelection, + new FakeDataSource(), + cmcdConfiguration, + /* subtitleParserFactory= */ null); + } }