loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add experimental opt-in to parse SS subtitles during extraction

Browse Source 
PiperOrigin-RevId: 586331888
This commit is contained in:
jbibik 2023-11-29 07:11:17 -08:00 committed by Copybara-Service
parent 9add30e582
commit 8a8b875c72
7 changed files with 213 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
RELEASENOTES.md
View File

@ -92,6 +92,9 @@
* Parse "f800" as channel count of 5 for Dolby in DASH manifest
([#688](https://github.com/androidx/media/issues/688)).
* Smooth Streaming Extension:
* Add experimental support for parsing subtitles during extraction. You
can enable this using
`SsMediaSource.Factory.experimentalParseSubtitlesDuringExtraction()`.
* RTSP Extension:
* Decoder Extensions (FFmpeg, VP9, AV1, MIDI, etc.):
* MIDI decoder: Ignore SysEx event messages

29
libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java
View File

@ -47,9 +47,12 @@ import androidx.media3.exoplayer.upstream.CmcdData;
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy.FallbackSelection;
import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
import androidx.media3.extractor.mp4.Track;
import androidx.media3.extractor.mp4.TrackEncryptionBox;
import androidx.media3.extractor.text.SubtitleParser;
import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
import java.io.IOException;
import java.util.List;
@ -60,11 +63,24 @@ public class DefaultSsChunkSource implements SsChunkSource {
public static final class Factory implements SsChunkSource.Factory {
private final DataSource.Factory dataSourceFactory;
@Nullable private SubtitleParser.Factory subtitleParserFactory;
public Factory(DataSource.Factory dataSourceFactory) {
this.dataSourceFactory = dataSourceFactory;
}
/**
* Sets the {@link SubtitleParser.Factory} to be used for parsing subtitles during extraction,
* or {@code null} to parse subtitles during decoding.
*
* <p>This may only be used with {@link BundledChunkExtractor.Factory}.
*/
/* package */ Factory setSubtitleParserFactory(
@Nullable SubtitleParser.Factory subtitleParserFactory) {
this.subtitleParserFactory = subtitleParserFactory;
return this;
}
@Override
public SsChunkSource createChunkSource(
LoaderErrorThrower manifestLoaderErrorThrower,
@ -83,7 +99,8 @@ public class DefaultSsChunkSource implements SsChunkSource {
streamElementIndex,
trackSelection,
dataSource,
cmcdConfiguration);
cmcdConfiguration,
subtitleParserFactory);
}
}
@ -112,6 +129,8 @@ public class DefaultSsChunkSource implements SsChunkSource {
* @param trackSelection The track selection.
* @param dataSource A {@link DataSource} suitable for loading the media data.
* @param cmcdConfiguration The {@link CmcdConfiguration} for this chunk source.
* @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
* extraction.
*/
public DefaultSsChunkSource(
LoaderErrorThrower manifestLoaderErrorThrower,
@ -119,7 +138,8 @@ public class DefaultSsChunkSource implements SsChunkSource {
int streamElementIndex,
ExoTrackSelection trackSelection,
DataSource dataSource,
@Nullable CmcdConfiguration cmcdConfiguration) {
@Nullable CmcdConfiguration cmcdConfiguration,
@Nullable SubtitleParser.Factory subtitleParserFactory) {
this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
this.manifest = manifest;
this.streamElementIndex = streamElementIndex;
@ -152,12 +172,15 @@ public class DefaultSsChunkSource implements SsChunkSource {
nalUnitLengthFieldLength,
null,
null);
FragmentedMp4Extractor extractor =
Extractor extractor =
new FragmentedMp4Extractor(
FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME
| FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX,
/* timestampAdjuster= */ null,
track);
if (subtitleParserFactory != null) {
extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
}
chunkExtractors[i] = new BundledChunkExtractor(extractor, streamElement.type, format);
}
}

30
libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java
View File

@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.StreamKey;
import androidx.media3.common.TrackGroup;
import androidx.media3.common.util.NullableType;
@ -41,6 +42,7 @@ import androidx.media3.exoplayer.upstream.Allocator;
import androidx.media3.exoplayer.upstream.CmcdConfiguration;
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
import androidx.media3.extractor.text.SubtitleParser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@ -77,7 +79,8 @@ import java.util.List;
LoadErrorHandlingPolicy loadErrorHandlingPolicy,
MediaSourceEventListener.EventDispatcher mediaSourceEventDispatcher,
LoaderErrorThrower manifestLoaderErrorThrower,
Allocator allocator) {
Allocator allocator,
@Nullable SubtitleParser.Factory subtitleParserFactory) {
this.manifest = manifest;
this.chunkSourceFactory = chunkSourceFactory;
this.transferListener = transferListener;
@ -89,7 +92,7 @@ import java.util.List;
this.mediaSourceEventDispatcher = mediaSourceEventDispatcher;
this.allocator = allocator;
this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
trackGroups = buildTrackGroups(manifest, drmSessionManager);
trackGroups = buildTrackGroups(manifest, drmSessionManager, subtitleParserFactory);
sampleStreams = newSampleStreamArray(0);
compositeSequenceableLoader =
compositeSequenceableLoaderFactory.createCompositeSequenceableLoader(sampleStreams);
@ -265,15 +268,32 @@ import java.util.List;
}
private static TrackGroupArray buildTrackGroups(
SsManifest manifest, DrmSessionManager drmSessionManager) {
SsManifest manifest,
DrmSessionManager drmSessionManager,
@Nullable SubtitleParser.Factory subtitleParserFactory) {
TrackGroup[] trackGroups = new TrackGroup[manifest.streamElements.length];
for (int i = 0; i < manifest.streamElements.length; i++) {
Format[] manifestFormats = manifest.streamElements[i].formats;
Format[] exposedFormats = new Format[manifestFormats.length];
for (int j = 0; j < manifestFormats.length; j++) {
Format manifestFormat = manifestFormats[j];
exposedFormats[j] =
manifestFormat.copyWithCryptoType(drmSessionManager.getCryptoType(manifestFormat));
Format.Builder updatedFormat =
manifestFormat
.buildUpon()
.setCryptoType(drmSessionManager.getCryptoType(manifestFormat));
if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(manifestFormat)) {
updatedFormat
.setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
.setCueReplacementBehavior(
subtitleParserFactory.getCueReplacementBehavior(manifestFormat))
.setCodecs(
manifestFormat.sampleMimeType
+ (manifestFormat.codecs != null ? " " + manifestFormat.codecs : ""))
// Reset this value to the default. All non-default timestamp adjustments are done
// by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding.
.setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE);
}
exposedFormats[j] = updatedFormat.build();
}
trackGroups[i] = new TrackGroup(/* id= */ Integer.toString(i), exposedFormats);
}

45
libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSource.java
View File

@ -65,6 +65,8 @@ import androidx.media3.exoplayer.upstream.Loader;
import androidx.media3.exoplayer.upstream.Loader.LoadErrorAction;
import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
import androidx.media3.exoplayer.upstream.ParsingLoadable;
import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
import androidx.media3.extractor.text.SubtitleParser;
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.IOException;
@ -91,6 +93,7 @@ public final class SsMediaSource extends BaseMediaSource
@Nullable private CmcdConfiguration.Factory cmcdConfigurationFactory;
private DrmSessionManagerProvider drmSessionManagerProvider;
private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
@Nullable private SubtitleParser.Factory subtitleParserFactory;
private long livePresentationDelayMs;
@Nullable private ParsingLoadable.Parser<? extends SsManifest> manifestParser;
@ -152,6 +155,40 @@ public final class SsMediaSource extends BaseMediaSource
return this;
}
/**
* Sets whether subtitles should be parsed as part of extraction (before being added to the
* sample queue) or as part of rendering (when being taken from the sample queue). Defaults to
* {@code false} (i.e. subtitles will be parsed as part of rendering).
*
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
* in a future release.
*
* <p>This method may only be used with {@link DefaultSsChunkSource.Factory}.
*
* @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or
* rendering.
* @return This factory, for convenience.
*/
// TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on
// DefaultMediaSourceFactory via the MediaSource.Factory interface).
public Factory experimentalParseSubtitlesDuringExtraction(
boolean parseSubtitlesDuringExtraction) {
if (parseSubtitlesDuringExtraction) {
if (subtitleParserFactory == null) {
this.subtitleParserFactory = new DefaultSubtitleParserFactory();
}
} else {
this.subtitleParserFactory = null;
}
if (chunkSourceFactory instanceof DefaultSsChunkSource.Factory) {
((DefaultSsChunkSource.Factory) chunkSourceFactory)
.setSubtitleParserFactory(subtitleParserFactory);
} else {
throw new IllegalStateException();
}
return this;
}
/**
* Sets the duration in milliseconds by which the default start position should precede the end
* of the live window for live playbacks. The default value is {@link
@ -273,6 +310,7 @@ public final class SsMediaSource extends BaseMediaSource
cmcdConfiguration,
drmSessionManagerProvider.get(mediaItem),
loadErrorHandlingPolicy,
subtitleParserFactory,
livePresentationDelayMs);
}
@ -310,6 +348,7 @@ public final class SsMediaSource extends BaseMediaSource
cmcdConfiguration,
drmSessionManagerProvider.get(mediaItem),
loadErrorHandlingPolicy,
subtitleParserFactory,
livePresentationDelayMs);
}
@ -353,6 +392,7 @@ public final class SsMediaSource extends BaseMediaSource
private long manifestLoadStartTimestamp;
private SsManifest manifest;
private Handler manifestRefreshHandler;
@Nullable private final SubtitleParser.Factory subtitleParserFactory;
@GuardedBy("this")
private MediaItem mediaItem;
@ -367,6 +407,7 @@ public final class SsMediaSource extends BaseMediaSource
@Nullable CmcdConfiguration cmcdConfiguration,
DrmSessionManager drmSessionManager,
LoadErrorHandlingPolicy loadErrorHandlingPolicy,
@Nullable SubtitleParser.Factory subtitleParserFactory,
long livePresentationDelayMs) {
Assertions.checkState(manifest == null || !manifest.isLive);
this.mediaItem = mediaItem;
@ -383,6 +424,7 @@ public final class SsMediaSource extends BaseMediaSource
this.cmcdConfiguration = cmcdConfiguration;
this.drmSessionManager = drmSessionManager;
this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
this.subtitleParserFactory = subtitleParserFactory;
this.livePresentationDelayMs = livePresentationDelayMs;
this.manifestEventDispatcher = createEventDispatcher(/* mediaPeriodId= */ null);
sideloadedManifest = manifest != null;
@ -450,7 +492,8 @@ public final class SsMediaSource extends BaseMediaSource
loadErrorHandlingPolicy,
mediaSourceEventDispatcher,
manifestLoaderErrorThrower,
allocator);
allocator,
subtitleParserFactory);
mediaPeriods.add(period);
return period;
}

3
libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSourceTest.java
View File

@ -297,6 +297,7 @@ public class DefaultSsChunkSourceTest {
/* streamElementIndex= */ 0,
adaptiveTrackSelection,
new FakeDataSource(),
cmcdConfiguration);
cmcdConfiguration,
/* subtitleParserFactory= */ null);
}
}

68
libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriodTest.java
View File

@ -17,6 +17,7 @@ package androidx.media3.exoplayer.smoothstreaming;
import static androidx.media3.exoplayer.smoothstreaming.SsTestUtils.createSsManifest;
import static androidx.media3.exoplayer.smoothstreaming.SsTestUtils.createStreamElement;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.mock;
import androidx.media3.common.C;
@ -32,6 +33,9 @@ import androidx.media3.exoplayer.source.MediaSourceEventListener;
import androidx.media3.exoplayer.upstream.Allocator;
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
import androidx.media3.extractor.text.SubtitleParser;
import androidx.media3.test.utils.FakeTimeline;
import androidx.media3.test.utils.MediaPeriodAsserts;
import androidx.media3.test.utils.MediaPeriodAsserts.FilterableManifestMediaPeriodFactory;
import androidx.test.ext.junit.runners.AndroidJUnit4;
@ -75,13 +79,75 @@ public class SsMediaPeriodTest {
new MediaSourceEventListener.EventDispatcher()
.withParameters(/* windowIndex= */ 0, mediaPeriodId),
mock(LoaderErrorThrower.class),
mock(Allocator.class));
mock(Allocator.class),
/* subtitleParserFactory= */ null);
};
MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration(
mediaPeriodFactory, testManifest);
}
@Test
public void getTrackGroups_withSubtitleParserFactory_matchesFormat() {
SubtitleParser.Factory subtitleParserFactory = new DefaultSubtitleParserFactory();
Format originalSubtitleFormat =
new Format.Builder()
.setContainerMimeType(MimeTypes.APPLICATION_MP4)
.setSampleMimeType(MimeTypes.TEXT_VTT)
.setLanguage("eng")
.build();
Format expectedSubtitleFormat =
new Format.Builder()
.setContainerMimeType(originalSubtitleFormat.containerMimeType)
.setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
.setCodecs(originalSubtitleFormat.sampleMimeType)
.setCueReplacementBehavior(
subtitleParserFactory.getCueReplacementBehavior(originalSubtitleFormat))
.setLanguage(originalSubtitleFormat.language)
.build();
SsManifest testManifest =
createSsManifest(
createStreamElement(
/* name= */ "video",
C.TRACK_TYPE_VIDEO,
createVideoFormat(/* bitrate= */ 200000),
createVideoFormat(/* bitrate= */ 400000),
createVideoFormat(/* bitrate= */ 800000)),
createStreamElement(
/* name= */ "audio",
C.TRACK_TYPE_AUDIO,
createAudioFormat(/* bitrate= */ 48000),
createAudioFormat(/* bitrate= */ 96000)),
createStreamElement(/* name= */ "text", C.TRACK_TYPE_TEXT, originalSubtitleFormat));
MediaPeriodId mediaPeriodId =
new MediaPeriodId(
new FakeTimeline(/* windowCount= */ 2).getUidOfPeriod(/* periodIndex= */ 0),
/* windowSequenceNumber= */ 0);
SsMediaPeriod period =
new SsMediaPeriod(
testManifest,
mock(SsChunkSource.Factory.class),
mock(TransferListener.class),
mock(CompositeSequenceableLoaderFactory.class),
/* cmcdConfiguration= */ null,
mock(DrmSessionManager.class),
new DrmSessionEventListener.EventDispatcher()
.withParameters(/* windowIndex= */ 0, mediaPeriodId),
mock(LoadErrorHandlingPolicy.class),
new MediaSourceEventListener.EventDispatcher()
.withParameters(/* windowIndex= */ 0, mediaPeriodId),
mock(LoaderErrorThrower.class),
mock(Allocator.class),
subtitleParserFactory);
Format subtitleFormat = period.getTrackGroups().get(2).getFormat(0);
assertThat(subtitleFormat).isEqualTo(expectedSubtitleFormat);
}
private static Format createVideoFormat(int bitrate) {
return new Format.Builder()
.setContainerMimeType(MimeTypes.VIDEO_MP4)

46
libraries/exoplayer_smoothstreaming/src/test/java/androidx/media3/exoplayer/smoothstreaming/SsMediaSourceTest.java
View File

@ -16,16 +16,24 @@
package androidx.media3.exoplayer.smoothstreaming;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.fail;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.MediaItem;
import androidx.media3.common.StreamKey;
import androidx.media3.common.Timeline;
import androidx.media3.datasource.ByteArrayDataSource;
import androidx.media3.datasource.DataSource;
import androidx.media3.datasource.TransferListener;
import androidx.media3.exoplayer.analytics.PlayerId;
import androidx.media3.exoplayer.smoothstreaming.manifest.SsManifest;
import androidx.media3.exoplayer.source.MediaSource;
import androidx.media3.exoplayer.trackselection.ExoTrackSelection;
import androidx.media3.exoplayer.upstream.CmcdConfiguration;
import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
import androidx.media3.test.utils.FakeDataSource;
import androidx.media3.test.utils.TestUtil;
import androidx.media3.test.utils.robolectric.RobolectricUtil;
import androidx.test.core.app.ApplicationProvider;
@ -94,6 +102,27 @@ public class SsMediaSourceTest {
assertThat(canUpdateMediaItem).isFalse();
}
@Test
public void
setExperimentalParseSubtitlesDuringExtraction_withNonDefaultChunkSourceFactory_setThrows() {
SsMediaSource.Factory ssMediaSourceFactory =
new SsMediaSource.Factory(
/* chunkSourceFactory= */ this::createSampleSsChunkSource,
/* manifestDataSourceFactory= */ () -> createSampleDataSource(SAMPLE_MANIFEST));
assertThrows(
IllegalStateException.class,
() -> ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(false));
}
@Test
public void
setExperimentalParseSubtitlesDuringExtraction_withDefaultChunkSourceFactory_setSucceeds() {
SsMediaSource.Factory ssMediaSourceFactory =
new SsMediaSource.Factory(() -> createSampleDataSource(SAMPLE_MANIFEST));
ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(false);
ssMediaSourceFactory.experimentalParseSubtitlesDuringExtraction(true);
}
@Test
public void canUpdateMediaItem_withChangedStreamKeys_returnsFalse() {
MediaItem initialMediaItem =
@ -175,4 +204,21 @@ public class SsMediaSourceTest {
}
return new ByteArrayDataSource(manifestData);
}
private SsChunkSource createSampleSsChunkSource(
LoaderErrorThrower manifestLoaderErrorThrower,
SsManifest manifest,
int streamElementIndex,
ExoTrackSelection trackSelection,
@Nullable TransferListener transferListener,
@Nullable CmcdConfiguration cmcdConfiguration) {
return new DefaultSsChunkSource(
manifestLoaderErrorThrower,
manifest,
streamElementIndex,
trackSelection,
new FakeDataSource(),
cmcdConfiguration,
/* subtitleParserFactory= */ null);
}
}