From 966b7108976706c1b0a1a8a2b82ae8daa747bda3 Mon Sep 17 00:00:00 2001 From: jbibik Date: Wed, 24 Jan 2024 08:36:02 -0800 Subject: [PATCH] Remove SubtitleParser.Factory references from Hls/Ss/DashMediaPeriod Those classes only needed to have access to a `SubtitleParser.Factory` to get a potentially updated `Format` for TrackGroups. The `SubtitleParser.Factory` was only used to check the support for the `mimeType` and getting some cue-related behaviour. This introduced complexity in a way that both Periods and Extractors needed to have the same `SubtitleParser.Factory` in their individual stacks. To ensure that the sample queue would get the same transcoded/original format. Instead, now we expose `getOutputTextFormat` methods on `ChunkExtractor.Factory`, `SsChunkSource.Factory` and `HlsExtractorFactory`. Those are the dependencies that Hls/Ss/DashMediaPeriod can make use of to delegate the format-updating logic to. #minor-release PiperOrigin-RevId: 601130714 --- .../source/chunk/BundledChunkExtractor.java | 29 +++++++++ .../source/chunk/ChunkExtractor.java | 22 +++++++ .../exoplayer/dash/DashChunkSource.java | 22 +++++++ .../exoplayer/dash/DashMediaPeriod.java | 33 +++------- .../exoplayer/dash/DashMediaSource.java | 9 +-- .../dash/DefaultDashChunkSource.java | 11 ++++ .../exoplayer/dash/DashMediaPeriodTest.java | 9 ++- .../hls/DefaultHlsExtractorFactory.java | 29 +++++++++ .../exoplayer/hls/HlsExtractorFactory.java | 22 +++++++ .../media3/exoplayer/hls/HlsMediaPeriod.java | 28 ++------- .../media3/exoplayer/hls/HlsMediaSource.java | 7 +-- .../exoplayer/hls/HlsMediaPeriodTest.java | 8 ++- .../smoothstreaming/DefaultSsChunkSource.java | 26 ++++++++ .../smoothstreaming/SsChunkSource.java | 24 ++++++++ .../smoothstreaming/SsMediaPeriod.java | 28 +++------ .../smoothstreaming/SsMediaSource.java | 8 +-- .../smoothstreaming/SsMediaPeriodTest.java | 60 +++++++++---------- 17 files changed, 249 insertions(+), 126 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java index d3c6d653cd..5305a0945c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java @@ -78,6 +78,35 @@ public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtrac return this; } + /** + * {@inheritDoc} + * + *

This implementation performs transcoding of the original format to {@link + * MimeTypes#APPLICATION_MEDIA3_CUES} if it is supported by {@link SubtitleParser.Factory}. + * + *

To modify the support behavior, you can {@linkplain + * #experimentalSetSubtitleParserFactory(SubtitleParser.Factory) set your own subtitle parser + * factory}. + */ + @Override + public Format getOutputTextFormat(Format sourceFormat) { + if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(sourceFormat)) { + @Format.CueReplacementBehavior + int cueReplacementBehavior = subtitleParserFactory.getCueReplacementBehavior(sourceFormat); + return sourceFormat + .buildUpon() + .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) + .setCueReplacementBehavior(cueReplacementBehavior) + .setCodecs( + sourceFormat.sampleMimeType + + (sourceFormat.codecs != null ? " " + sourceFormat.codecs : "")) + .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) + .build(); + } else { + return sourceFormat; + } + } + @Nullable @Override public ChunkExtractor createProgressiveMediaExtractor( diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ChunkExtractor.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ChunkExtractor.java index 4089f0cbf4..35ad0caff0 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ChunkExtractor.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ChunkExtractor.java @@ -18,9 +18,11 @@ package androidx.media3.exoplayer.source.chunk; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.analytics.PlayerId; import androidx.media3.extractor.ChunkIndex; +import androidx.media3.extractor.Extractor; import androidx.media3.extractor.ExtractorInput; import androidx.media3.extractor.TrackOutput; import java.io.IOException; @@ -57,6 +59,26 @@ public interface ChunkExtractor { List closedCaptionFormats, @Nullable TrackOutput playerEmsgTrackOutput, PlayerId playerId); + + /** + * Returns the output {@link Format} of emitted {@linkplain C#TRACK_TYPE_TEXT text samples} + * which were originally in {@code sourceFormat}. + * + *

In many cases, where an {@link Extractor} emits samples from the source without mutation, + * this method simply returns {@code sourceFormat}. In other cases, such as an {@link Extractor} + * that transcodes subtitles from the {@code sourceFormat} to {@link + * MimeTypes#APPLICATION_MEDIA3_CUES}, the format is updated to indicate the transcoding that is + * taking place. + * + *

Non-text source formats are always returned without mutation. + * + * @param sourceFormat The original text-based format. + * @return The {@link Format} that will be associated with a {@linkplain C#TRACK_TYPE_TEXT text + * track}. + */ + default Format getOutputTextFormat(Format sourceFormat) { + return sourceFormat; + } } /** Provides {@link TrackOutput} instances to be written to during extraction. */ diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashChunkSource.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashChunkSource.java index 37fd959526..7e4b40401b 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashChunkSource.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashChunkSource.java @@ -19,6 +19,7 @@ import android.os.SystemClock; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.common.util.UnstableApi; import androidx.media3.datasource.TransferListener; import androidx.media3.exoplayer.analytics.PlayerId; @@ -28,6 +29,7 @@ import androidx.media3.exoplayer.source.chunk.ChunkSource; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; +import androidx.media3.extractor.Extractor; import java.util.List; /** A {@link ChunkSource} for DASH streams. */ @@ -74,6 +76,26 @@ public interface DashChunkSource extends ChunkSource { @Nullable TransferListener transferListener, PlayerId playerId, @Nullable CmcdConfiguration cmcdConfiguration); + + /** + * Returns the output {@link Format} of emitted {@linkplain C#TRACK_TYPE_TEXT text samples} + * which were originally in {@code sourceFormat}. + * + *

In many cases, where an {@link Extractor} emits samples from the source without mutation, + * this method simply returns {@code sourceFormat}. In other cases, such as an {@link Extractor} + * that transcodes subtitles from the {@code sourceFormat} to {@link + * MimeTypes#APPLICATION_MEDIA3_CUES}, the format is updated to indicate the transcoding that is + * taking place. + * + *

Non-text source formats are always returned without mutation. + * + * @param sourceFormat The original text-based format. + * @return The {@link Format} that will be associated with a {@linkplain C#TRACK_TYPE_TEXT text + * track}. + */ + default Format getOutputTextFormat(Format sourceFormat) { + return sourceFormat; + } } /** diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java index e2089d8a32..811c75968f 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java @@ -58,7 +58,6 @@ import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; -import androidx.media3.extractor.text.SubtitleParser; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; @@ -132,8 +131,7 @@ import java.util.regex.Pattern; Allocator allocator, CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory, PlayerEmsgCallback playerEmsgCallback, - PlayerId playerId, - @Nullable SubtitleParser.Factory subtitleParserFactory) { + PlayerId playerId) { this.id = id; this.manifest = manifest; this.baseUrlExclusionList = baseUrlExclusionList; @@ -160,7 +158,7 @@ import java.util.regex.Pattern; eventStreams = period.eventStreams; Pair result = buildTrackGroups( - drmSessionManager, subtitleParserFactory, period.adaptationSets, eventStreams); + drmSessionManager, chunkSourceFactory, period.adaptationSets, eventStreams); trackGroups = result.first; trackGroupInfos = result.second; } @@ -505,7 +503,7 @@ import java.util.regex.Pattern; private static Pair buildTrackGroups( DrmSessionManager drmSessionManager, - @Nullable SubtitleParser.Factory subtitleParserFactory, + DashChunkSource.Factory chunkSourceFactory, List adaptationSets, List eventStreams) { int[][] groupedAdaptationSetIndices = getGroupedAdaptationSetIndices(adaptationSets); @@ -528,7 +526,7 @@ import java.util.regex.Pattern; int trackGroupCount = buildPrimaryAndEmbeddedTrackGroupInfos( drmSessionManager, - subtitleParserFactory, + chunkSourceFactory, adaptationSets, groupedAdaptationSetIndices, primaryGroupCount, @@ -668,7 +666,7 @@ import java.util.regex.Pattern; private static int buildPrimaryAndEmbeddedTrackGroupInfos( DrmSessionManager drmSessionManager, - @Nullable SubtitleParser.Factory subtitleParserFactory, + DashChunkSource.Factory chunkSourceFactory, List adaptationSets, int[][] groupedAdaptationSetIndices, int primaryGroupCount, @@ -704,7 +702,7 @@ import java.util.regex.Pattern; int closedCaptionTrackGroupIndex = primaryGroupClosedCaptionTrackFormats[i].length != 0 ? trackGroupCount++ : C.INDEX_UNSET; - maybeUpdateFormatsForParsedText(subtitleParserFactory, formats); + maybeUpdateFormatsForParsedText(chunkSourceFactory, formats); trackGroups[primaryTrackGroupIndex] = new TrackGroup(trackGroupId, formats); trackGroupInfos[primaryTrackGroupIndex] = TrackGroupInfo.primaryTrack( @@ -732,7 +730,7 @@ import java.util.regex.Pattern; primaryTrackGroupIndex, ImmutableList.copyOf(primaryGroupClosedCaptionTrackFormats[i])); maybeUpdateFormatsForParsedText( - subtitleParserFactory, primaryGroupClosedCaptionTrackFormats[i]); + chunkSourceFactory, primaryGroupClosedCaptionTrackFormats[i]); trackGroups[closedCaptionTrackGroupIndex] = new TrackGroup(closedCaptionTrackGroupId, primaryGroupClosedCaptionTrackFormats[i]); } @@ -928,22 +926,9 @@ import java.util.regex.Pattern; * during extraction. */ private static void maybeUpdateFormatsForParsedText( - SubtitleParser.Factory subtitleParserFactory, Format[] formats) { + DashChunkSource.Factory chunkSourceFactory, Format[] formats) { for (int i = 0; i < formats.length; i++) { - if (subtitleParserFactory == null || !subtitleParserFactory.supportsFormat(formats[i])) { - continue; - } - formats[i] = - formats[i] - .buildUpon() - .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) - .setCueReplacementBehavior( - subtitleParserFactory.getCueReplacementBehavior(formats[i])) - .setCodecs( - formats[i].sampleMimeType - + (formats[i].codecs != null ? " " + formats[i].codecs : "")) - .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) - .build(); + formats[i] = chunkSourceFactory.getOutputTextFormat(formats[i]); } } diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java index 4d02994303..59526b93a6 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java @@ -210,6 +210,7 @@ public final class DashMediaSource extends BaseMediaSource { */ // TODO: b/289916598 - Flip the default of this to true. @Override + @CanIgnoreReturnValue public Factory experimentalParseSubtitlesDuringExtraction( boolean parseSubtitlesDuringExtraction) { if (parseSubtitlesDuringExtraction) { @@ -347,7 +348,6 @@ public final class DashMediaSource extends BaseMediaSource { cmcdConfiguration, drmSessionManagerProvider.get(mediaItem), loadErrorHandlingPolicy, - subtitleParserFactory, fallbackTargetLiveOffsetMs, minLiveStartPositionUs); } @@ -386,7 +386,6 @@ public final class DashMediaSource extends BaseMediaSource { cmcdConfiguration, drmSessionManagerProvider.get(mediaItem), loadErrorHandlingPolicy, - subtitleParserFactory, fallbackTargetLiveOffsetMs, minLiveStartPositionUs); } @@ -445,7 +444,6 @@ public final class DashMediaSource extends BaseMediaSource { private final Runnable simulateManifestRefreshRunnable; private final PlayerEmsgCallback playerEmsgCallback; private final LoaderErrorThrower manifestLoadErrorThrower; - @Nullable private final SubtitleParser.Factory subtitleParserFactory; private DataSource dataSource; private Loader loader; @@ -481,7 +479,6 @@ public final class DashMediaSource extends BaseMediaSource { @Nullable CmcdConfiguration cmcdConfiguration, DrmSessionManager drmSessionManager, LoadErrorHandlingPolicy loadErrorHandlingPolicy, - @Nullable SubtitleParser.Factory subtitleParserFactory, long fallbackTargetLiveOffsetMs, long minLiveStartPositionUs) { this.mediaItem = mediaItem; @@ -495,7 +492,6 @@ public final class DashMediaSource extends BaseMediaSource { this.cmcdConfiguration = cmcdConfiguration; this.drmSessionManager = drmSessionManager; this.loadErrorHandlingPolicy = loadErrorHandlingPolicy; - this.subtitleParserFactory = subtitleParserFactory; this.fallbackTargetLiveOffsetMs = fallbackTargetLiveOffsetMs; this.minLiveStartPositionUs = minLiveStartPositionUs; this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory; @@ -601,8 +597,7 @@ public final class DashMediaSource extends BaseMediaSource { allocator, compositeSequenceableLoaderFactory, playerEmsgCallback, - getPlayerId(), - subtitleParserFactory); + getPlayerId()); periodsById.put(mediaPeriod.id, mediaPeriod); return mediaPeriod; } diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java index 5b756401dd..8d542f3566 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java @@ -166,6 +166,17 @@ public class DefaultDashChunkSource implements DashChunkSource { playerId, cmcdConfiguration); } + + /** + * {@inheritDoc} + * + *

This implementation delegates determining of the output format to the {@link + * ChunkExtractor.Factory} passed to the constructor of this class. + */ + @Override + public Format getOutputTextFormat(Format sourceFormat) { + return chunkExtractorFactory.getOutputTextFormat(sourceFormat); + } } private final LoaderErrorThrower manifestLoaderErrorThrower; diff --git a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java index 388b48912c..eba317eb50 100644 --- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java +++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java @@ -15,7 +15,9 @@ */ package androidx.media3.exoplayer.dash; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import android.net.Uri; import androidx.media3.common.Format; @@ -205,12 +207,14 @@ public final class DashMediaPeriodTest { private static DashMediaPeriod createDashMediaPeriod(DashManifest manifest, int periodIndex) { MediaPeriodId mediaPeriodId = new MediaPeriodId(/* periodUid= */ new Object()); + DashChunkSource.Factory chunkSourceFactory = mock(DashChunkSource.Factory.class); + when(chunkSourceFactory.getOutputTextFormat(any())).thenCallRealMethod(); return new DashMediaPeriod( /* id= */ periodIndex, manifest, new BaseUrlExclusionList(), periodIndex, - mock(DashChunkSource.Factory.class), + chunkSourceFactory, mock(TransferListener.class), /* cmcdConfiguration= */ null, DrmSessionManager.DRM_UNSUPPORTED, @@ -224,8 +228,7 @@ public final class DashMediaPeriodTest { mock(Allocator.class), mock(CompositeSequenceableLoaderFactory.class), mock(PlayerEmsgCallback.class), - PlayerId.UNSET, - /* subtitleParserFactory= */ null); + PlayerId.UNSET); } private static DashManifest parseManifest(String fileName) throws IOException { diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java index e5b77f3486..8e8b56b376 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java @@ -169,6 +169,35 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory { return this; } + /** + * {@inheritDoc} + * + *

This implementation performs transcoding of the original format to {@link + * MimeTypes#APPLICATION_MEDIA3_CUES} if it is supported by {@link SubtitleParser.Factory}. + * + *

To modify the support behavior, you can {@linkplain + * #experimentalSetSubtitleParserFactory(SubtitleParser.Factory) set your own subtitle parser + * factory}. + */ + @Override + public Format getOutputTextFormat(Format sourceFormat) { + if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(sourceFormat)) { + @Format.CueReplacementBehavior + int cueReplacementBehavior = subtitleParserFactory.getCueReplacementBehavior(sourceFormat); + return sourceFormat + .buildUpon() + .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) + .setCueReplacementBehavior(cueReplacementBehavior) + .setCodecs( + sourceFormat.sampleMimeType + + (sourceFormat.codecs != null ? " " + sourceFormat.codecs : "")) + .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) + .build(); + } else { + return sourceFormat; + } + } + private static void addFileTypeIfValidAndNotPresent( @FileTypes.Type int fileType, List fileTypes) { if (Ints.indexOf(DEFAULT_EXTRACTOR_ORDER, fileType) == -1 || fileTypes.contains(fileType)) { diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsExtractorFactory.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsExtractorFactory.java index f8ad476d9b..75ef496522 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsExtractorFactory.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsExtractorFactory.java @@ -17,7 +17,9 @@ package androidx.media3.exoplayer.hls; import android.net.Uri; import androidx.annotation.Nullable; +import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.common.util.TimestampAdjuster; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.analytics.PlayerId; @@ -60,4 +62,24 @@ public interface HlsExtractorFactory { ExtractorInput sniffingExtractorInput, PlayerId playerId) throws IOException; + + /** + * Returns the output {@link Format} of emitted {@linkplain C#TRACK_TYPE_TEXT text samples} which + * were originally in {@code sourceFormat}. + * + *

In many cases, where an {@link Extractor} emits samples from the source without mutation, + * this method simply returns {@code sourceFormat}. In other cases, such as an {@link Extractor} + * that transcodes subtitles from the {@code sourceFormat} to {@link + * MimeTypes#APPLICATION_MEDIA3_CUES}, the format is updated to indicate the transcoding that is + * taking place. + * + *

Non-text source formats are always returned without mutation. + * + * @param sourceFormat The original text-based format. + * @return The {@link Format} that will be associated with a {@linkplain C#TRACK_TYPE_TEXT text + * track}. + */ + default Format getOutputTextFormat(Format sourceFormat) { + return sourceFormat; + } } diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java index f61c34e578..e18220adbf 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java @@ -51,7 +51,6 @@ import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.extractor.Extractor; -import androidx.media3.extractor.text.SubtitleParser; import com.google.common.primitives.Ints; import java.io.IOException; import java.util.ArrayList; @@ -86,7 +85,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final PlayerId playerId; private final HlsSampleStreamWrapper.Callback sampleStreamWrapperCallback; private final long timestampAdjusterInitializationTimeoutMs; - @Nullable private final SubtitleParser.Factory subtitleParserFactory; @Nullable private MediaPeriod.Callback mediaPeriodCallback; private int pendingPrepareCount; @@ -141,8 +139,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @HlsMediaSource.MetadataType int metadataType, boolean useSessionKeys, PlayerId playerId, - long timestampAdjusterInitializationTimeoutMs, - @Nullable SubtitleParser.Factory subtitleParserFactory) { + long timestampAdjusterInitializationTimeoutMs) { this.extractorFactory = extractorFactory; this.playlistTracker = playlistTracker; this.dataSourceFactory = dataSourceFactory; @@ -167,7 +164,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; sampleStreamWrappers = new HlsSampleStreamWrapper[0]; enabledSampleStreamWrappers = new HlsSampleStreamWrapper[0]; manifestUrlIndicesPerWrapper = new int[0][]; - this.subtitleParserFactory = subtitleParserFactory; } public void release() { @@ -538,7 +534,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; sampleStreamWrapper.prepareWithMultivariantPlaylistInfo( new TrackGroup[] { new TrackGroup( - sampleStreamWrapperUid, maybeUpdateFormatForParsedText(originalSubtitleFormat)) + sampleStreamWrapperUid, + extractorFactory.getOutputTextFormat(originalSubtitleFormat)) }, /* primaryTrackGroupIndex= */ 0); } @@ -686,7 +683,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; for (int i = 0; i < ccFormats.size(); i++) { String ccId = sampleStreamWrapperUid + ":cc:" + i; muxedTrackGroups.add( - new TrackGroup(ccId, maybeUpdateFormatForParsedText(ccFormats.get(i)))); + new TrackGroup(ccId, extractorFactory.getOutputTextFormat(ccFormats.get(i)))); } } } else /* numberOfAudioCodecs > 0 */ { @@ -910,23 +907,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; .build(); } - /** - * Returns a modified {@link Format} if subtitle/caption parsing is configured to happen during - * extraction. - */ - private Format maybeUpdateFormatForParsedText(Format format) { - if (subtitleParserFactory == null || !subtitleParserFactory.supportsFormat(format)) { - return format; - } - return format - .buildUpon() - .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) - .setCueReplacementBehavior(subtitleParserFactory.getCueReplacementBehavior(format)) - .setCodecs(format.sampleMimeType + (format.codecs != null ? " " + format.codecs : "")) - .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) - .build(); - } - private class SampleStreamWrapperCallback implements HlsSampleStreamWrapper.Callback { @Override public void onPrepared() { diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java index f1697eb4da..6dd1366238 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java @@ -413,7 +413,6 @@ public final class HlsMediaSource extends BaseMediaSource mediaItem, hlsDataSourceFactory, extractorFactory, - subtitleParserFactory, compositeSequenceableLoaderFactory, cmcdConfiguration, drmSessionManagerProvider.get(mediaItem), @@ -445,7 +444,6 @@ public final class HlsMediaSource extends BaseMediaSource private final HlsPlaylistTracker playlistTracker; private final long elapsedRealTimeOffsetMs; private final long timestampAdjusterInitializationTimeoutMs; - @Nullable private final SubtitleParser.Factory subtitleParserFactory; private MediaItem.LiveConfiguration liveConfiguration; @Nullable private TransferListener mediaTransferListener; @@ -457,7 +455,6 @@ public final class HlsMediaSource extends BaseMediaSource MediaItem mediaItem, HlsDataSourceFactory dataSourceFactory, HlsExtractorFactory extractorFactory, - @Nullable SubtitleParser.Factory subtitleParserFactory, CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory, @Nullable CmcdConfiguration cmcdConfiguration, DrmSessionManager drmSessionManager, @@ -472,7 +469,6 @@ public final class HlsMediaSource extends BaseMediaSource this.liveConfiguration = mediaItem.liveConfiguration; this.dataSourceFactory = dataSourceFactory; this.extractorFactory = extractorFactory; - this.subtitleParserFactory = subtitleParserFactory; this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory; this.cmcdConfiguration = cmcdConfiguration; this.drmSessionManager = drmSessionManager; @@ -547,8 +543,7 @@ public final class HlsMediaSource extends BaseMediaSource metadataType, useSessionKeys, getPlayerId(), - timestampAdjusterInitializationTimeoutMs, - subtitleParserFactory); + timestampAdjusterInitializationTimeoutMs); } @Override diff --git a/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java b/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java index bdebc756be..05399df667 100644 --- a/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java +++ b/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java @@ -15,6 +15,7 @@ */ package androidx.media3.exoplayer.hls; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -73,6 +74,8 @@ public final class HlsMediaPeriodTest { createSubtitleFormat("eng"), createSubtitleFormat("gsw"))); FilterableManifestMediaPeriodFactory mediaPeriodFactory = (playlist, periodIndex) -> { + HlsExtractorFactory mockHlsExtractorFactory = mock(HlsExtractorFactory.class); + when(mockHlsExtractorFactory.getOutputTextFormat(any())).thenCallRealMethod(); HlsDataSourceFactory mockDataSourceFactory = mock(HlsDataSourceFactory.class); when(mockDataSourceFactory.createDataSource(anyInt())).thenReturn(mock(DataSource.class)); HlsPlaylistTracker mockPlaylistTracker = mock(HlsPlaylistTracker.class); @@ -80,7 +83,7 @@ public final class HlsMediaPeriodTest { .thenReturn((HlsMultivariantPlaylist) playlist); MediaPeriodId mediaPeriodId = new MediaPeriodId(/* periodUid= */ new Object()); return new HlsMediaPeriod( - mock(HlsExtractorFactory.class), + mockHlsExtractorFactory, mockPlaylistTracker, mockDataSourceFactory, mock(TransferListener.class), @@ -97,8 +100,7 @@ public final class HlsMediaPeriodTest { HlsMediaSource.METADATA_TYPE_ID3, /* useSessionKeys= */ false, PlayerId.UNSET, - /* timestampAdjusterInitializationTimeoutMs= */ 0, - /* subtitleParserFactory= */ null); + /* timestampAdjusterInitializationTimeoutMs= */ 0); }; MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration( diff --git a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java index c0ba6d26af..40cf102473 100644 --- a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java +++ b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java @@ -23,6 +23,7 @@ import android.os.SystemClock; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UriUtil; @@ -82,6 +83,31 @@ public class DefaultSsChunkSource implements SsChunkSource { return this; } + /** + * {@inheritDoc} + * + *

This implementation performs transcoding of the original format to {@link + * MimeTypes#APPLICATION_MEDIA3_CUES} if it is supported by {@link SubtitleParser.Factory}. + */ + @Override + public Format getOutputTextFormat(Format sourceFormat) { + if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(sourceFormat)) { + @Format.CueReplacementBehavior + int cueReplacementBehavior = subtitleParserFactory.getCueReplacementBehavior(sourceFormat); + return sourceFormat + .buildUpon() + .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) + .setCueReplacementBehavior(cueReplacementBehavior) + .setCodecs( + sourceFormat.sampleMimeType + + (sourceFormat.codecs != null ? " " + sourceFormat.codecs : "")) + .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) + .build(); + } else { + return sourceFormat; + } + } + @Override public SsChunkSource createChunkSource( LoaderErrorThrower manifestLoaderErrorThrower, diff --git a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsChunkSource.java b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsChunkSource.java index edca06db91..4592ad9c2c 100644 --- a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsChunkSource.java +++ b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsChunkSource.java @@ -16,6 +16,9 @@ package androidx.media3.exoplayer.smoothstreaming; import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.common.util.UnstableApi; import androidx.media3.datasource.TransferListener; import androidx.media3.exoplayer.smoothstreaming.manifest.SsManifest; @@ -23,6 +26,7 @@ import androidx.media3.exoplayer.source.chunk.ChunkSource; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; +import androidx.media3.extractor.Extractor; /** A {@link ChunkSource} for SmoothStreaming. */ @UnstableApi @@ -50,6 +54,26 @@ public interface SsChunkSource extends ChunkSource { ExoTrackSelection trackSelection, @Nullable TransferListener transferListener, @Nullable CmcdConfiguration cmcdConfiguration); + + /** + * Returns the output {@link Format} of emitted {@linkplain C#TRACK_TYPE_TEXT text samples} + * which were originally in {@code sourceFormat}. + * + *

In many cases, where an {@link Extractor} emits samples from the source without mutation, + * this method simply returns {@code sourceFormat}. In other cases, such as an {@link Extractor} + * that transcodes subtitles from the {@code sourceFormat} to {@link + * MimeTypes#APPLICATION_MEDIA3_CUES}, the format is updated to indicate the transcoding that is + * taking place. + * + *

Non-text source formats are always returned without mutation. + * + * @param sourceFormat The original text-based format. + * @return The {@link Format} that will be associated with a {@linkplain C#TRACK_TYPE_TEXT text + * track}. + */ + default Format getOutputTextFormat(Format sourceFormat) { + return sourceFormat; + } } /** diff --git a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java index 759e9e86e2..e5c757e52f 100644 --- a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java +++ b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java @@ -20,7 +20,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; -import androidx.media3.common.MimeTypes; import androidx.media3.common.StreamKey; import androidx.media3.common.TrackGroup; import androidx.media3.common.util.NullableType; @@ -42,7 +41,6 @@ import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; -import androidx.media3.extractor.text.SubtitleParser; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -79,8 +77,7 @@ import java.util.List; LoadErrorHandlingPolicy loadErrorHandlingPolicy, MediaSourceEventListener.EventDispatcher mediaSourceEventDispatcher, LoaderErrorThrower manifestLoaderErrorThrower, - Allocator allocator, - @Nullable SubtitleParser.Factory subtitleParserFactory) { + Allocator allocator) { this.manifest = manifest; this.chunkSourceFactory = chunkSourceFactory; this.transferListener = transferListener; @@ -92,7 +89,7 @@ import java.util.List; this.mediaSourceEventDispatcher = mediaSourceEventDispatcher; this.allocator = allocator; this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory; - trackGroups = buildTrackGroups(manifest, drmSessionManager, subtitleParserFactory); + trackGroups = buildTrackGroups(manifest, drmSessionManager, chunkSourceFactory); sampleStreams = newSampleStreamArray(0); compositeSequenceableLoader = compositeSequenceableLoaderFactory.createCompositeSequenceableLoader(sampleStreams); @@ -270,30 +267,19 @@ import java.util.List; private static TrackGroupArray buildTrackGroups( SsManifest manifest, DrmSessionManager drmSessionManager, - @Nullable SubtitleParser.Factory subtitleParserFactory) { + SsChunkSource.Factory chunkSourceFactory) { TrackGroup[] trackGroups = new TrackGroup[manifest.streamElements.length]; for (int i = 0; i < manifest.streamElements.length; i++) { Format[] manifestFormats = manifest.streamElements[i].formats; Format[] exposedFormats = new Format[manifestFormats.length]; for (int j = 0; j < manifestFormats.length; j++) { Format manifestFormat = manifestFormats[j]; - Format.Builder updatedFormat = + Format updatedFormatWithDrm = manifestFormat .buildUpon() - .setCryptoType(drmSessionManager.getCryptoType(manifestFormat)); - if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(manifestFormat)) { - updatedFormat - .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) - .setCueReplacementBehavior( - subtitleParserFactory.getCueReplacementBehavior(manifestFormat)) - .setCodecs( - manifestFormat.sampleMimeType - + (manifestFormat.codecs != null ? " " + manifestFormat.codecs : "")) - // Reset this value to the default. All non-default timestamp adjustments are done - // by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding. - .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE); - } - exposedFormats[j] = updatedFormat.build(); + .setCryptoType(drmSessionManager.getCryptoType(manifestFormat)) + .build(); + exposedFormats[j] = chunkSourceFactory.getOutputTextFormat(updatedFormatWithDrm); } trackGroups[i] = new TrackGroup(/* id= */ Integer.toString(i), exposedFormats); } diff --git a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java index 21fd45e906..50ecb5e838 100644 --- a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java +++ b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java @@ -305,7 +305,6 @@ public final class SsMediaSource extends BaseMediaSource cmcdConfiguration, drmSessionManagerProvider.get(mediaItem), loadErrorHandlingPolicy, - subtitleParserFactory, livePresentationDelayMs); } @@ -343,7 +342,6 @@ public final class SsMediaSource extends BaseMediaSource cmcdConfiguration, drmSessionManagerProvider.get(mediaItem), loadErrorHandlingPolicy, - subtitleParserFactory, livePresentationDelayMs); } @@ -387,7 +385,6 @@ public final class SsMediaSource extends BaseMediaSource private long manifestLoadStartTimestamp; private SsManifest manifest; private Handler manifestRefreshHandler; - @Nullable private final SubtitleParser.Factory subtitleParserFactory; @GuardedBy("this") private MediaItem mediaItem; @@ -402,7 +399,6 @@ public final class SsMediaSource extends BaseMediaSource @Nullable CmcdConfiguration cmcdConfiguration, DrmSessionManager drmSessionManager, LoadErrorHandlingPolicy loadErrorHandlingPolicy, - @Nullable SubtitleParser.Factory subtitleParserFactory, long livePresentationDelayMs) { Assertions.checkState(manifest == null || !manifest.isLive); this.mediaItem = mediaItem; @@ -419,7 +415,6 @@ public final class SsMediaSource extends BaseMediaSource this.cmcdConfiguration = cmcdConfiguration; this.drmSessionManager = drmSessionManager; this.loadErrorHandlingPolicy = loadErrorHandlingPolicy; - this.subtitleParserFactory = subtitleParserFactory; this.livePresentationDelayMs = livePresentationDelayMs; this.manifestEventDispatcher = createEventDispatcher(/* mediaPeriodId= */ null); sideloadedManifest = manifest != null; @@ -487,8 +482,7 @@ public final class SsMediaSource extends BaseMediaSource loadErrorHandlingPolicy, mediaSourceEventDispatcher, manifestLoaderErrorThrower, - allocator, - subtitleParserFactory); + allocator); mediaPeriods.add(period); return period; } diff --git a/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java b/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java index 865732ff01..8ed464642c 100644 --- a/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java +++ b/libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java @@ -18,7 +18,9 @@ package androidx.media3.exoplayer.smoothstreaming; import static androidx.media3.exoplayer.smoothstreaming.SsTestUtils.createSsManifest; import static androidx.media3.exoplayer.smoothstreaming.SsTestUtils.createStreamElement; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import androidx.media3.common.C; import androidx.media3.common.Format; @@ -63,24 +65,13 @@ public class SsMediaPeriodTest { createAudioFormat(/* bitrate= */ 96000)), createStreamElement( /* name= */ "text", C.TRACK_TYPE_TEXT, createTextFormat(/* language= */ "eng"))); + SsChunkSource.Factory chunkSourceFactory = mock(SsChunkSource.Factory.class); + when(chunkSourceFactory.getOutputTextFormat(any())).thenCallRealMethod(); + FilterableManifestMediaPeriodFactory mediaPeriodFactory = (manifest, periodIndex) -> { MediaPeriodId mediaPeriodId = new MediaPeriodId(/* periodUid= */ new Object()); - return new SsMediaPeriod( - manifest, - mock(SsChunkSource.Factory.class), - mock(TransferListener.class), - mock(CompositeSequenceableLoaderFactory.class), - /* cmcdConfiguration= */ null, - mock(DrmSessionManager.class), - new DrmSessionEventListener.EventDispatcher() - .withParameters(/* windowIndex= */ 0, mediaPeriodId), - mock(LoadErrorHandlingPolicy.class), - new MediaSourceEventListener.EventDispatcher() - .withParameters(/* windowIndex= */ 0, mediaPeriodId), - mock(LoaderErrorThrower.class), - mock(Allocator.class), - /* subtitleParserFactory= */ null); + return createSsMediaPeriod(manifest, mediaPeriodId, chunkSourceFactory); }; MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration( @@ -127,27 +118,34 @@ public class SsMediaPeriodTest { new FakeTimeline(/* windowCount= */ 2).getUidOfPeriod(/* periodIndex= */ 0), /* windowSequenceNumber= */ 0); - SsMediaPeriod period = - new SsMediaPeriod( - testManifest, - mock(SsChunkSource.Factory.class), - mock(TransferListener.class), - mock(CompositeSequenceableLoaderFactory.class), - /* cmcdConfiguration= */ null, - mock(DrmSessionManager.class), - new DrmSessionEventListener.EventDispatcher() - .withParameters(/* windowIndex= */ 0, mediaPeriodId), - mock(LoadErrorHandlingPolicy.class), - new MediaSourceEventListener.EventDispatcher() - .withParameters(/* windowIndex= */ 0, mediaPeriodId), - mock(LoaderErrorThrower.class), - mock(Allocator.class), - subtitleParserFactory); + SsChunkSource.Factory chunkSourceFactory = mock(SsChunkSource.Factory.class); + // Default implementation of SsChunkSource.Factory.getOutputTextFormat doesn't transcode + // DefaultSsChunkSource.Factory is final (not mockable) and has a null SubtitleParser.Factory + when(chunkSourceFactory.getOutputTextFormat(any())).thenReturn(expectedSubtitleFormat); + SsMediaPeriod period = createSsMediaPeriod(testManifest, mediaPeriodId, chunkSourceFactory); Format subtitleFormat = period.getTrackGroups().get(2).getFormat(0); assertThat(subtitleFormat).isEqualTo(expectedSubtitleFormat); } + private static SsMediaPeriod createSsMediaPeriod( + SsManifest manifest, MediaPeriodId mediaPeriodId, SsChunkSource.Factory chunkSourceFactory) { + return new SsMediaPeriod( + manifest, + chunkSourceFactory, + mock(TransferListener.class), + mock(CompositeSequenceableLoaderFactory.class), + /* cmcdConfiguration= */ null, + mock(DrmSessionManager.class), + new DrmSessionEventListener.EventDispatcher() + .withParameters(/* windowIndex= */ 0, mediaPeriodId), + mock(LoadErrorHandlingPolicy.class), + new MediaSourceEventListener.EventDispatcher() + .withParameters(/* windowIndex= */ 0, mediaPeriodId), + mock(LoaderErrorThrower.class), + mock(Allocator.class)); + } + private static Format createVideoFormat(int bitrate) { return new Format.Builder() .setContainerMimeType(MimeTypes.VIDEO_MP4)