Fix HLS CEA-608 when parsing during extraction

HLS distinguishes between 'subtitles' (WebVTT or TTML distributed in
separate files with their own playlist) and 'captions' (CEA-608 or 708,
distributed muxed into the video file).

The format transformation added in 7b762642db
only applies to subtitles and not captions. This change makes the same
transformation for caption formats.

This resolves an error like:

```
SampleQueueMappingException: Unable to bind a sample queue to TrackGroup with MIME type application/cea-608.
```

Also add two playback tests for HLS CEA-608, one that parses during
decoding (old way) and one during extraction (new way). Adding these
tests is what alerted me to this issue.

PiperOrigin-RevId: 592571284
This commit is contained in:
ibaker 2023-12-20 08:43:52 -08:00 committed by Copybara-Service
parent 4ce47ccdd3
commit 770ca66fbc
6 changed files with 1691 additions and 24 deletions

View File

@ -535,29 +535,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
positionUs);
manifestUrlIndicesPerWrapper.add(new int[] {i});
sampleStreamWrappers.add(sampleStreamWrapper);
if (subtitleParserFactory != null
&& subtitleParserFactory.supportsFormat(originalSubtitleFormat)) {
Format updatedSubtitleFormat =
originalSubtitleFormat
.buildUpon()
.setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
.setCueReplacementBehavior(
subtitleParserFactory.getCueReplacementBehavior(originalSubtitleFormat))
.setCodecs(
originalSubtitleFormat.sampleMimeType
+ (originalSubtitleFormat.codecs != null
? " " + originalSubtitleFormat.codecs
: ""))
.setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE)
.build();
sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, updatedSubtitleFormat)},
/* primaryTrackGroupIndex= */ 0);
} else {
sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, originalSubtitleFormat)},
/* primaryTrackGroupIndex= */ 0);
}
sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
new TrackGroup[] {
new TrackGroup(
sampleStreamWrapperUid, maybeUpdateFormatForParsedText(originalSubtitleFormat))
},
/* primaryTrackGroupIndex= */ 0);
}
this.sampleStreamWrappers = sampleStreamWrappers.toArray(new HlsSampleStreamWrapper[0]);
@ -702,7 +685,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (ccFormats != null) {
for (int i = 0; i < ccFormats.size(); i++) {
String ccId = sampleStreamWrapperUid + ":cc:" + i;
muxedTrackGroups.add(new TrackGroup(ccId, ccFormats.get(i)));
muxedTrackGroups.add(
new TrackGroup(ccId, maybeUpdateFormatForParsedText(ccFormats.get(i))));
}
}
} else /* numberOfAudioCodecs > 0 */ {
@ -926,6 +910,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
.build();
}
/**
* Returns a modified {@link Format} if subtitle/caption parsing is configured to happen during
* extraction.
*/
private Format maybeUpdateFormatForParsedText(Format format) {
if (subtitleParserFactory == null || !subtitleParserFactory.supportsFormat(format)) {
return format;
}
return format
.buildUpon()
.setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
.setCueReplacementBehavior(subtitleParserFactory.getCueReplacementBehavior(format))
.setCodecs(format.sampleMimeType + (format.codecs != null ? " " + format.codecs : ""))
.setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE)
.build();
}
private class SampleStreamWrapperCallback implements HlsSampleStreamWrapper.Callback {
@Override
public void onPrepared() {

View File

@ -95,4 +95,62 @@ public final class HlsPlaybackTest {
DumpFileAsserts.assertOutput(
applicationContext, playbackOutput, "playbackdumps/hls/ttml-in-mp4.dump");
}
/**
* This test and {@link #cea608_parseDuringExtraction()} use the same output dump file, to
* demonstrate the flag has no effect on the resulting subtitles.
*/
@Test
public void cea608_parseDuringRendering() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactory capturingRenderersFactory =
new CapturingRenderersFactory(applicationContext);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
.setMediaSourceFactory(
new HlsMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
.experimentalParseSubtitlesDuringExtraction(false))
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.build();
player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));
PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory);
player.setMediaItem(MediaItem.fromUri("asset:///media/hls/cea608/manifest.m3u8"));
player.prepare();
player.play();
TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
player.release();
DumpFileAsserts.assertOutput(
applicationContext, playbackOutput, "playbackdumps/hls/cea608.dump");
}
/**
* This test and {@link #cea608_parseDuringRendering()} use the same output dump file, to
* demonstrate the flag has no effect on the resulting subtitles.
*/
@Test
public void cea608_parseDuringExtraction() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactory capturingRenderersFactory =
new CapturingRenderersFactory(applicationContext);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
.setMediaSourceFactory(
new HlsMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
.experimentalParseSubtitlesDuringExtraction(true))
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.build();
player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));
PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory);
player.setMediaItem(MediaItem.fromUri("asset:///media/hls/cea608/manifest.m3u8"));
player.prepare();
player.play();
TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
player.release();
DumpFileAsserts.assertOutput(
applicationContext, playbackOutput, "playbackdumps/hls/cea608.dump");
}
}

View File

@ -0,0 +1,4 @@
#EXTM3U
#EXT-X-MEDIA:TYPE=CLOSED-CAPTIONS,GROUP-ID="closed-captions",NAME="English",LANGUAGE="en-US",DEFAULT=YES,AUTOSELECT=YES,INSTREAM-ID="CC1"
#EXT-X-STREAM-INF:BANDWIDTH=926871,AVERAGE-BANDWIDTH=926871,RESOLUTION=640x360,CLOSED-CAPTIONS="closed-captions",CODECS="avc1.64001e,mp4a.40.2"
sd-hls.m3u8

View File

@ -0,0 +1,9 @@
#EXTM3U
#EXT-X-TARGETDURATION:6
#EXT-X-VERSION:4
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MEDIA-SEQUENCE:0
#EXTINF:2.458,
#EXT-X-BYTERANGE:284820@0
sd-hls0000000000.ts
#EXT-X-ENDLIST

File diff suppressed because it is too large Load Diff