loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add experimental opt-in to parse DASH subtitles during extraction

Browse Source 
This currently only applies to subtitles muxed into mp4 segments, and
not standalone text files linked directly from the manifest.

Issue: androidx/media#288

#minor-release

PiperOrigin-RevId: 572263764
(cherry picked from commit 66fa5919590789b384506a4e604fe02a5a5e0877)
This commit is contained in:
ibaker 2023-10-10 08:51:07 -07:00 committed by Rohit Singh
parent 7254f5aca5
commit 03a3f77340
9 changed files with 170 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
RELEASENOTES.md
View File

@ -37,6 +37,13 @@ This release includes the following changes since the
Android Auto.
* DASH Extension:
* Allow multiple of the same DASH identifier in segment template url.
* Add experimental support for parsing subtitles during extraction. This
has better support for merging overlapping subtitles, including
resolving flickering when transitioning between subtitle segments. You
can enable this using
`DashMediaSource.Factory.experimentalParseSubtitlesDuringExtraction()`
([#288](https://github.com/androidx/media/issues/288)).
* Smooth Streaming Extension:
* RTSP Extension:
* Use RTSP Setup Response timeout value in time interval of sending
keep-alive RTSP Options requests

94
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java
View File

@ -26,6 +26,7 @@ import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Assertions;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.exoplayer.analytics.PlayerId;
import androidx.media3.extractor.ChunkIndex;
import androidx.media3.extractor.DummyTrackOutput;
import androidx.media3.extractor.Extractor;
@ -36,7 +37,10 @@ import androidx.media3.extractor.SeekMap;
import androidx.media3.extractor.TrackOutput;
import androidx.media3.extractor.mkv.MatroskaExtractor;
import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
import androidx.media3.extractor.text.SubtitleParser;
import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
import java.io.IOException;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
@ -46,36 +50,68 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@UnstableApi
public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtractor {
/** {@link ChunkExtractor.Factory} for instances of this class. */
public static final ChunkExtractor.Factory FACTORY =
(primaryTrackType,
format,
enableEventMessageTrack,
closedCaptionFormats,
playerEmsgTrackOutput,
playerId) -> {
@Nullable String containerMimeType = format.containerMimeType;
Extractor extractor;
if (MimeTypes.isText(containerMimeType)) {
// Text types do not need an extractor.
return null;
} else if (MimeTypes.isMatroska(containerMimeType)) {
extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES);
} else {
int flags = 0;
if (enableEventMessageTrack) {
flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
}
extractor =
new FragmentedMp4Extractor(
flags,
/* timestampAdjuster= */ null,
/* sideloadedTrack= */ null,
closedCaptionFormats,
playerEmsgTrackOutput);
/** {@link ChunkExtractor.Factory} for {@link BundledChunkExtractor}. */
public static final class Factory implements ChunkExtractor.Factory {
/** Non-null if subtitles should be parsed during extraction, null otherwise. */
@Nullable private SubtitleParser.Factory subtitleParserFactory;
/**
* Sets the {@link SubtitleParser.Factory} to use for parsing subtitles during extraction, or
* null to parse subtitles during decoding. The default is null (subtitles parsed after
* decoding).
*
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
* in a future release.
*
* @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
* extraction.
* @return This factory, for convenience.
*/
public Factory experimentalSetSubtitleParserFactory(
@Nullable SubtitleParser.Factory subtitleParserFactory) {
this.subtitleParserFactory = subtitleParserFactory;
return this;
}
@Nullable
@Override
public ChunkExtractor createProgressiveMediaExtractor(
@C.TrackType int primaryTrackType,
Format representationFormat,
boolean enableEventMessageTrack,
List<Format> closedCaptionFormats,
@Nullable TrackOutput playerEmsgTrackOutput,
PlayerId playerId) {
@Nullable String containerMimeType = representationFormat.containerMimeType;
Extractor extractor;
if (MimeTypes.isText(containerMimeType)) {
// Text types do not need an extractor.
return null;
} else if (MimeTypes.isMatroska(containerMimeType)) {
extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES);
} else {
int flags = 0;
if (enableEventMessageTrack) {
flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
}
return new BundledChunkExtractor(extractor, primaryTrackType, format);
};
extractor =
new FragmentedMp4Extractor(
flags,
/* timestampAdjuster= */ null,
/* sideloadedTrack= */ null,
closedCaptionFormats,
playerEmsgTrackOutput);
}
if (subtitleParserFactory != null) {
extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
}
return new BundledChunkExtractor(extractor, primaryTrackType, representationFormat);
}
}
/** {@link Factory} for {@link BundledChunkExtractor}. */
public static final Factory FACTORY = new Factory();
private static final PositionHolder POSITION_HOLDER = new PositionHolder();

30
libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java
View File

@ -58,6 +58,7 @@ import androidx.media3.exoplayer.upstream.Allocator;
import androidx.media3.exoplayer.upstream.CmcdConfiguration;
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
import androidx.media3.extractor.text.SubtitleParser;
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;
import java.io.IOException;
@ -130,7 +131,8 @@ import java.util.regex.Pattern;
Allocator allocator,
CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
PlayerEmsgCallback playerEmsgCallback,
PlayerId playerId) {
PlayerId playerId,
@Nullable SubtitleParser.Factory subtitleParserFactory) {
this.id = id;
this.manifest = manifest;
this.baseUrlExclusionList = baseUrlExclusionList;
@ -156,7 +158,8 @@ import java.util.regex.Pattern;
Period period = manifest.getPeriod(periodIndex);
eventStreams = period.eventStreams;
Pair<TrackGroupArray, TrackGroupInfo[]> result =
buildTrackGroups(drmSessionManager, period.adaptationSets, eventStreams);
buildTrackGroups(
drmSessionManager, subtitleParserFactory, period.adaptationSets, eventStreams);
trackGroups = result.first;
trackGroupInfos = result.second;
}
@ -501,6 +504,7 @@ import java.util.regex.Pattern;
private static Pair<TrackGroupArray, TrackGroupInfo[]> buildTrackGroups(
DrmSessionManager drmSessionManager,
@Nullable SubtitleParser.Factory subtitleParserFactory,
List<AdaptationSet> adaptationSets,
List<EventStream> eventStreams) {
int[][] groupedAdaptationSetIndices = getGroupedAdaptationSetIndices(adaptationSets);
@ -523,6 +527,7 @@ import java.util.regex.Pattern;
int trackGroupCount =
buildPrimaryAndEmbeddedTrackGroupInfos(
drmSessionManager,
subtitleParserFactory,
adaptationSets,
groupedAdaptationSetIndices,
primaryGroupCount,
@ -662,6 +667,7 @@ import java.util.regex.Pattern;
private static int buildPrimaryAndEmbeddedTrackGroupInfos(
DrmSessionManager drmSessionManager,
@Nullable SubtitleParser.Factory subtitleParserFactory,
List<AdaptationSet> adaptationSets,
int[][] groupedAdaptationSetIndices,
int primaryGroupCount,
@ -678,8 +684,24 @@ import java.util.regex.Pattern;
}
Format[] formats = new Format[representations.size()];
for (int j = 0; j < formats.length; j++) {
Format format = representations.get(j).format;
formats[j] = format.copyWithCryptoType(drmSessionManager.getCryptoType(format));
Format originalFormat = representations.get(j).format;
Format.Builder updatedFormat =
originalFormat
.buildUpon()
.setCryptoType(drmSessionManager.getCryptoType(originalFormat));
if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(originalFormat)) {
updatedFormat
.setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
.setCueReplacementBehavior(
subtitleParserFactory.getCueReplacementBehavior(originalFormat))
.setCodecs(
originalFormat.sampleMimeType
+ (originalFormat.codecs != null ? " " + originalFormat.codecs : ""))
// Reset this value to the default. All non-default timestamp adjustments are done
// by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding.
.setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE);
}
formats[j] = updatedFormat.build();
}
AdaptationSet firstAdaptationSet = adaptationSets.get(adaptationSetIndices[0]);

45
libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java
View File

@ -77,6 +77,8 @@ import androidx.media3.exoplayer.upstream.Loader.LoadErrorAction;
import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
import androidx.media3.exoplayer.upstream.ParsingLoadable;
import androidx.media3.exoplayer.util.SntpClient;
import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
import androidx.media3.extractor.text.SubtitleParser;
import com.google.common.base.Charsets;
import com.google.common.math.LongMath;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
@ -112,6 +114,7 @@ public final class DashMediaSource extends BaseMediaSource {
private DrmSessionManagerProvider drmSessionManagerProvider;
private CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
@Nullable private SubtitleParser.Factory subtitleParserFactory;
private long fallbackTargetLiveOffsetMs;
private long minLiveStartPositionUs;
@Nullable private ParsingLoadable.Parser<? extends DashManifest> manifestParser;
@ -196,6 +199,40 @@ public final class DashMediaSource extends BaseMediaSource {
return this;
}
/**
* Sets whether subtitles should be parsed as part of extraction (before the sample queue) or as
* part of rendering (after the sample queue). Defaults to false (i.e. subtitles will be parsed
* as part of rendering).
*
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
* in a future release.
*
* <p>This method may only be used with {@link DefaultDashChunkSource.Factory}.
*
* @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or
* rendering.
* @return This factory, for convenience.
*/
// TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on
// DefaultMediaSourceFactory via the MediaSource.Factory interface).
public Factory experimentalParseSubtitlesDuringExtraction(
boolean parseSubtitlesDuringExtraction) {
if (parseSubtitlesDuringExtraction) {
if (subtitleParserFactory == null) {
this.subtitleParserFactory = new DefaultSubtitleParserFactory();
}
} else {
this.subtitleParserFactory = null;
}
if (chunkSourceFactory instanceof DefaultDashChunkSource.Factory) {
((DefaultDashChunkSource.Factory) chunkSourceFactory)
.setSubtitleParserFactory(subtitleParserFactory);
} else {
throw new IllegalStateException();
}
return this;
}
/**
* Sets the target {@link Player#getCurrentLiveOffset() offset for live streams} that is used if
* no value is defined in the {@link MediaItem} or the manifest.
@ -315,6 +352,7 @@ public final class DashMediaSource extends BaseMediaSource {
cmcdConfiguration,
drmSessionManagerProvider.get(mediaItem),
loadErrorHandlingPolicy,
subtitleParserFactory,
fallbackTargetLiveOffsetMs,
minLiveStartPositionUs);
}
@ -353,6 +391,7 @@ public final class DashMediaSource extends BaseMediaSource {
cmcdConfiguration,
drmSessionManagerProvider.get(mediaItem),
loadErrorHandlingPolicy,
subtitleParserFactory,
fallbackTargetLiveOffsetMs,
minLiveStartPositionUs);
}
@ -411,6 +450,7 @@ public final class DashMediaSource extends BaseMediaSource {
private final Runnable simulateManifestRefreshRunnable;
private final PlayerEmsgCallback playerEmsgCallback;
private final LoaderErrorThrower manifestLoadErrorThrower;
@Nullable private final SubtitleParser.Factory subtitleParserFactory;
private DataSource dataSource;
private Loader loader;
@ -446,6 +486,7 @@ public final class DashMediaSource extends BaseMediaSource {
@Nullable CmcdConfiguration cmcdConfiguration,
DrmSessionManager drmSessionManager,
LoadErrorHandlingPolicy loadErrorHandlingPolicy,
@Nullable SubtitleParser.Factory subtitleParserFactory,
long fallbackTargetLiveOffsetMs,
long minLiveStartPositionUs) {
this.mediaItem = mediaItem;
@ -459,6 +500,7 @@ public final class DashMediaSource extends BaseMediaSource {
this.cmcdConfiguration = cmcdConfiguration;
this.drmSessionManager = drmSessionManager;
this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
this.subtitleParserFactory = subtitleParserFactory;
this.fallbackTargetLiveOffsetMs = fallbackTargetLiveOffsetMs;
this.minLiveStartPositionUs = minLiveStartPositionUs;
this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
@ -564,7 +606,8 @@ public final class DashMediaSource extends BaseMediaSource {
allocator,
compositeSequenceableLoaderFactory,
playerEmsgCallback,
getPlayerId());
getPlayerId(),
subtitleParserFactory);
periodsById.put(mediaPeriod.id, mediaPeriod);
return mediaPeriod;
}

17
libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java
View File

@ -60,6 +60,7 @@ import androidx.media3.exoplayer.upstream.CmcdData;
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
import androidx.media3.exoplayer.upstream.LoaderErrorThrower;
import androidx.media3.extractor.ChunkIndex;
import androidx.media3.extractor.text.SubtitleParser;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.util.ArrayList;
@ -71,6 +72,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@UnstableApi
public class DefaultDashChunkSource implements DashChunkSource {
/** {@link DashChunkSource.Factory} for {@link DefaultDashChunkSource} instances. */
public static final class Factory implements DashChunkSource.Factory {
private final DataSource.Factory dataSourceFactory;
@ -110,6 +112,21 @@ public class DefaultDashChunkSource implements DashChunkSource {
this.maxSegmentsPerLoad = maxSegmentsPerLoad;
}
/**
* Sets the {@link SubtitleParser.Factory} to be used for parsing subtitles during extraction,
* or null to parse subtitles during decoding.
*
* <p>This may only be used with {@link BundledChunkExtractor.Factory}.
*/
/* package */ Factory setSubtitleParserFactory(
@Nullable SubtitleParser.Factory subtitleParserFactory) {
if (chunkExtractorFactory instanceof BundledChunkExtractor.Factory) {
((BundledChunkExtractor.Factory) chunkExtractorFactory)
.experimentalSetSubtitleParserFactory(subtitleParserFactory);
}
return this;
}
@Override
public DashChunkSource createDashChunkSource(
LoaderErrorThrower manifestLoaderErrorThrower,

3
libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java
View File

@ -224,7 +224,8 @@ public final class DashMediaPeriodTest {
mock(Allocator.class),
mock(CompositeSequenceableLoaderFactory.class),
mock(PlayerEmsgCallback.class),
PlayerId.UNSET);
PlayerId.UNSET,
/* subtitleParserFactory= */ null);
}
private static DashManifest parseManifest(String fileName) throws IOException {

7
libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java
View File

@ -22,9 +22,11 @@ import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.media3.common.MediaItem;
import androidx.media3.common.Player;
import androidx.media3.datasource.DefaultDataSource;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.RenderersFactory;
import androidx.media3.exoplayer.dash.DashMediaSource;
import androidx.media3.exoplayer.metadata.MetadataDecoderFactory;
import androidx.media3.exoplayer.metadata.MetadataRenderer;
import androidx.media3.exoplayer.trackselection.DefaultTrackSelector;
@ -79,14 +81,15 @@ public final class DashPlaybackTest {
// https://github.com/google/ExoPlayer/issues/7985
@Test
@Ignore(
"Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].")
public void webvttInMp4() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactory capturingRenderersFactory =
new CapturingRenderersFactory(applicationContext);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
.setMediaSourceFactory(
new DashMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
.experimentalParseSubtitlesDuringExtraction(true))
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.build();
player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));

5
libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java
View File

@ -26,7 +26,6 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.DataReader;
import androidx.media3.common.Format;
import androidx.media3.common.Format.CueReplacementBehavior;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.Util;
@ -86,8 +85,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (currentSubtitleParser == null) {
delegate.format(format);
} else {
@CueReplacementBehavior
int nextCuesBehavior = currentSubtitleParser.getCueReplacementBehavior();
delegate.format(
format
.buildUpon()
@ -96,7 +93,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Reset this value to the default. All non-default timestamp adjustments are done
// below in sampleMetadata() and there are no 'subsamples' after transcoding.
.setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE)
.setCueReplacementBehavior(nextCuesBehavior)
.setCueReplacementBehavior(subtitleParserFactory.getCueReplacementBehavior(format))
.build());
}
}

3
libraries/test_data/src/test/assets/playbackdumps/dash/webvtt-in-mp4.dump
View File

@ -244,3 +244,6 @@ TextOutput:
position = 0.5
positionAnchor = 1
size = 1.0
Subtitle[4]:
presentationTimeUs = 456000
Cues = []