Add HLS support for "stpp.*" codec support for SMPTE-TT fmp4 subtitle tracks

ExoPlayer needs a codec to decide among WEBVTT and TTML decoder mimeType.
Apple describes IMSC1 in MP4 in
[RFC-8216 Section 3.6](https://tools.ietf.org/html/draft-pantos-hls-rfc8216bis-04#section-3.6).

The DASH manifest specifies the SMPTE-TT captions in the codecs in the manifest
(from W3C [TTML Profiles for Internet Media Subtitles and Captions 1.1](https://www.w3.org/TR/ttml-imsc1.1/#general-0).
DASH just doesn't require the rendition linking, but HLS does.

Apple implies the CODECS attribute of the variant needs to be do this.  That is
with SHOULD and MAY language to imply the codec to use for it in the
[Authoring Guidelines](https://developer.apple.com/documentation/http_live_streaming/hls_authoring_specification_for_apple_devices)

This change defaults to WebVTT if no codec is specifed (same as current behavior) otherwise it picks it from the variants
referencing the media.
This commit is contained in:
Steve Mayhew 2020-04-02 09:50:22 -07:00
parent 918172cca7
commit 232820d3e1
3 changed files with 55 additions and 1 deletions

View File

@ -267,6 +267,8 @@ public final class MimeTypes {
return MimeTypes.AUDIO_VORBIS; return MimeTypes.AUDIO_VORBIS;
} else if (codec.startsWith("flac")) { } else if (codec.startsWith("flac")) {
return MimeTypes.AUDIO_FLAC; return MimeTypes.AUDIO_FLAC;
} else if (codec.startsWith("stpp")) {
return MimeTypes.APPLICATION_TTML;
} else { } else {
return getCustomMimeTypeForCodec(codec); return getCustomMimeTypeForCodec(codec);
} }

View File

@ -458,7 +458,19 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
} }
break; break;
case TYPE_SUBTITLES: case TYPE_SUBTITLES:
formatBuilder.setSampleMimeType(MimeTypes.TEXT_VTT).setMetadata(metadata); String subtitleMime = MimeTypes.TEXT_VTT; // Assume VTT unless variant declares it
variant = getVariantWithSubtitleGroup(variants, groupId);
if (variant != null) {
@Nullable
String codecs[] = Util.splitCodecs(variant.format.codecs);
for (String codec : codecs) {
if (codec.equalsIgnoreCase("stpp.ttml.im1t")) { // TOOD move this all to Utils.x
subtitleMime = MimeTypes.APPLICATION_TTML;
}
}
}
formatBuilder.setSampleMimeType(subtitleMime).setMetadata(metadata);
subtitles.add(new Rendition(uri, formatBuilder.build(), groupId, name)); subtitles.add(new Rendition(uri, formatBuilder.build(), groupId, name));
break; break;
case TYPE_CLOSED_CAPTIONS: case TYPE_CLOSED_CAPTIONS:
@ -516,6 +528,17 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
return null; return null;
} }
@Nullable
private static Variant getVariantWithSubtitleGroup(ArrayList<Variant> variants, String groupId) {
for (int i = 0; i < variants.size(); i++) {
Variant variant = variants.get(i);
if (groupId.equals(variant.subtitleGroupId)) {
return variant;
}
}
return null;
}
@Nullable @Nullable
private static Variant getVariantWithVideoGroup(ArrayList<Variant> variants, String groupId) { private static Variant getVariantWithVideoGroup(ArrayList<Variant> variants, String groupId) {
for (int i = 0; i < variants.size(); i++) { for (int i = 0; i < variants.size(); i++) {

View File

@ -194,6 +194,23 @@ public class HlsMasterPlaylistParserTest {
+ "#EXT-X-MEDIA:TYPE=SUBTITLES," + "#EXT-X-MEDIA:TYPE=SUBTITLES,"
+ "GROUP-ID=\"sub1\",NAME=\"English\",URI=\"s1/en/prog_index.m3u8\"\n"; + "GROUP-ID=\"sub1\",NAME=\"English\",URI=\"s1/en/prog_index.m3u8\"\n";
private static final String PLAYLIST_WITH_SUBTITLE_CODEC =
" #EXTM3U\n"
+ "\n"
+ "#EXT-X-VERSION:6\n"
+ "\n"
+ "#EXT-X-INDEPENDENT-SEGMENTS\n"
+ "\n"
+ "#EXT-X-STREAM-INF:BANDWIDTH=1280000,"
+ "CODECS=\"stpp.ttml.im1t,mp4a.40.2,avc1.66.30\",RESOLUTION=304x128,AUDIO=\"aud1\",SUBTITLES=\"sub1\"\n"
+ "http://example.com/low.m3u8\n"
+ "\n"
+ "#EXT-X-STREAM-INF:BANDWIDTH=1280000,CODECS=\"stpp.ttml.im1t,mp4a.40.2 , avc1.66.30 \",AUDIO=\"aud1\",SUBTITLES=\"sub1\"\n"
+ "http://example.com/spaces_in_codecs.m3u8\n"
+ "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud1\",NAME=\"English\",URI=\"a1/index.m3u8\"\n"
+ "#EXT-X-MEDIA:TYPE=SUBTITLES,"
+ "GROUP-ID=\"sub1\",NAME=\"English\",AUTOSELECT=YES,DEFAULT=YES,URI=\"s1/en/prog_index.m3u8\"\n";
@Test @Test
public void parseMasterPlaylist_withSimple_success() throws IOException { public void parseMasterPlaylist_withSimple_success() throws IOException {
HlsMasterPlaylist masterPlaylist = parseMasterPlaylist(PLAYLIST_URI, PLAYLIST_SIMPLE); HlsMasterPlaylist masterPlaylist = parseMasterPlaylist(PLAYLIST_URI, PLAYLIST_SIMPLE);
@ -321,6 +338,7 @@ public class HlsMasterPlaylistParserTest {
Format firstTextFormat = playlist.subtitles.get(0).format; Format firstTextFormat = playlist.subtitles.get(0).format;
assertThat(firstTextFormat.id).isEqualTo("sub1:Eng"); assertThat(firstTextFormat.id).isEqualTo("sub1:Eng");
assertThat(firstTextFormat.sampleMimeType).isEqualTo(MimeTypes.TEXT_VTT);
} }
@Test @Test
@ -345,6 +363,17 @@ public class HlsMasterPlaylistParserTest {
.isEqualTo(Uri.parse("http://example.com/This/{$nested}/reference/shouldnt/work")); .isEqualTo(Uri.parse("http://example.com/This/{$nested}/reference/shouldnt/work"));
} }
@Test
public void testSubtitleCodec() throws IOException {
HlsMasterPlaylist playlistWithSubtitles =
parseMasterPlaylist(PLAYLIST_URI, PLAYLIST_WITH_SUBTITLE_CODEC);
HlsMasterPlaylist.Variant variant = playlistWithSubtitles.variants.get(0);
Format firstTextFormat = playlistWithSubtitles.subtitles.get(0).format;
assertThat(firstTextFormat.id).isEqualTo("sub1:English");
assertThat(firstTextFormat.containerMimeType).isEqualTo(MimeTypes.APPLICATION_M3U8);
assertThat(firstTextFormat.sampleMimeType).isEqualTo(MimeTypes.APPLICATION_TTML);
assertThat(variant.format.codecs).isEqualTo("stpp.ttml.im1t,mp4a.40.2,avc1.66.30");
}
@Test @Test
public void parseMasterPlaylist_withMatchingStreamInfUrls_success() throws IOException { public void parseMasterPlaylist_withMatchingStreamInfUrls_success() throws IOException {
HlsMasterPlaylist playlist = HlsMasterPlaylist playlist =