From 232820d3e12c0486944bd3062e509ff5434dac6c Mon Sep 17 00:00:00 2001 From: Steve Mayhew Date: Thu, 2 Apr 2020 09:50:22 -0700 Subject: [PATCH 1/3] Add HLS support for "stpp.*" codec support for SMPTE-TT fmp4 subtitle tracks ExoPlayer needs a codec to decide among WEBVTT and TTML decoder mimeType. Apple describes IMSC1 in MP4 in [RFC-8216 Section 3.6](https://tools.ietf.org/html/draft-pantos-hls-rfc8216bis-04#section-3.6). The DASH manifest specifies the SMPTE-TT captions in the codecs in the manifest (from W3C [TTML Profiles for Internet Media Subtitles and Captions 1.1](https://www.w3.org/TR/ttml-imsc1.1/#general-0). DASH just doesn't require the rendition linking, but HLS does. Apple implies the CODECS attribute of the variant needs to be do this. That is with SHOULD and MAY language to imply the codec to use for it in the [Authoring Guidelines](https://developer.apple.com/documentation/http_live_streaming/hls_authoring_specification_for_apple_devices) This change defaults to WebVTT if no codec is specifed (same as current behavior) otherwise it picks it from the variants referencing the media. --- .../android/exoplayer2/util/MimeTypes.java | 2 ++ .../hls/playlist/HlsPlaylistParser.java | 25 +++++++++++++++- .../playlist/HlsMasterPlaylistParserTest.java | 29 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java b/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java index 43335cf51c..01eeb7e1f0 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java @@ -267,6 +267,8 @@ public final class MimeTypes { return MimeTypes.AUDIO_VORBIS; } else if (codec.startsWith("flac")) { return MimeTypes.AUDIO_FLAC; + } else if (codec.startsWith("stpp")) { + return MimeTypes.APPLICATION_TTML; } else { return getCustomMimeTypeForCodec(codec); } diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java index bcf06c5d2e..572d0661cb 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java @@ -458,7 +458,19 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser variants, String groupId) { + for (int i = 0; i < variants.size(); i++) { + Variant variant = variants.get(i); + if (groupId.equals(variant.subtitleGroupId)) { + return variant; + } + } + return null; + } + @Nullable private static Variant getVariantWithVideoGroup(ArrayList variants, String groupId) { for (int i = 0; i < variants.size(); i++) { diff --git a/library/hls/src/test/java/com/google/android/exoplayer2/source/hls/playlist/HlsMasterPlaylistParserTest.java b/library/hls/src/test/java/com/google/android/exoplayer2/source/hls/playlist/HlsMasterPlaylistParserTest.java index 40ba379a71..4f988c4e92 100644 --- a/library/hls/src/test/java/com/google/android/exoplayer2/source/hls/playlist/HlsMasterPlaylistParserTest.java +++ b/library/hls/src/test/java/com/google/android/exoplayer2/source/hls/playlist/HlsMasterPlaylistParserTest.java @@ -194,6 +194,23 @@ public class HlsMasterPlaylistParserTest { + "#EXT-X-MEDIA:TYPE=SUBTITLES," + "GROUP-ID=\"sub1\",NAME=\"English\",URI=\"s1/en/prog_index.m3u8\"\n"; + private static final String PLAYLIST_WITH_SUBTITLE_CODEC = + " #EXTM3U\n" + + "\n" + + "#EXT-X-VERSION:6\n" + + "\n" + + "#EXT-X-INDEPENDENT-SEGMENTS\n" + + "\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=1280000," + + "CODECS=\"stpp.ttml.im1t,mp4a.40.2,avc1.66.30\",RESOLUTION=304x128,AUDIO=\"aud1\",SUBTITLES=\"sub1\"\n" + + "http://example.com/low.m3u8\n" + + "\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=1280000,CODECS=\"stpp.ttml.im1t,mp4a.40.2 , avc1.66.30 \",AUDIO=\"aud1\",SUBTITLES=\"sub1\"\n" + + "http://example.com/spaces_in_codecs.m3u8\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud1\",NAME=\"English\",URI=\"a1/index.m3u8\"\n" + + "#EXT-X-MEDIA:TYPE=SUBTITLES," + + "GROUP-ID=\"sub1\",NAME=\"English\",AUTOSELECT=YES,DEFAULT=YES,URI=\"s1/en/prog_index.m3u8\"\n"; + @Test public void parseMasterPlaylist_withSimple_success() throws IOException { HlsMasterPlaylist masterPlaylist = parseMasterPlaylist(PLAYLIST_URI, PLAYLIST_SIMPLE); @@ -321,6 +338,7 @@ public class HlsMasterPlaylistParserTest { Format firstTextFormat = playlist.subtitles.get(0).format; assertThat(firstTextFormat.id).isEqualTo("sub1:Eng"); + assertThat(firstTextFormat.sampleMimeType).isEqualTo(MimeTypes.TEXT_VTT); } @Test @@ -345,6 +363,17 @@ public class HlsMasterPlaylistParserTest { .isEqualTo(Uri.parse("http://example.com/This/{$nested}/reference/shouldnt/work")); } + @Test + public void testSubtitleCodec() throws IOException { + HlsMasterPlaylist playlistWithSubtitles = + parseMasterPlaylist(PLAYLIST_URI, PLAYLIST_WITH_SUBTITLE_CODEC); + HlsMasterPlaylist.Variant variant = playlistWithSubtitles.variants.get(0); + Format firstTextFormat = playlistWithSubtitles.subtitles.get(0).format; + assertThat(firstTextFormat.id).isEqualTo("sub1:English"); + assertThat(firstTextFormat.containerMimeType).isEqualTo(MimeTypes.APPLICATION_M3U8); + assertThat(firstTextFormat.sampleMimeType).isEqualTo(MimeTypes.APPLICATION_TTML); + assertThat(variant.format.codecs).isEqualTo("stpp.ttml.im1t,mp4a.40.2,avc1.66.30"); + } @Test public void parseMasterPlaylist_withMatchingStreamInfUrls_success() throws IOException { HlsMasterPlaylist playlist = From 4ad4a826ab49e94d84493b57ba49d3ed26e48416 Mon Sep 17 00:00:00 2001 From: Steve Mayhew Date: Sat, 4 Apr 2020 16:59:33 -0700 Subject: [PATCH 2/3] Cleanup to use getCodecsOfType() Use the `Util.getCodecsOfType()` method rather than hard coded codec strings --- .../source/hls/playlist/HlsPlaylistParser.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java index 572d0661cb..60736189b5 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java @@ -458,19 +458,18 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser Date: Sun, 5 Apr 2020 13:23:03 -0700 Subject: [PATCH 3/3] Fix merge mistake, wrong track type. Fix mistake from the backport merge, should get TEXT track type for subtitles. Also update testcase to match pattern in `dev-v2` branch. --- .../exoplayer2/source/hls/playlist/HlsPlaylistParser.java | 2 +- .../source/hls/playlist/HlsMasterPlaylistParserTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java index 60736189b5..4392f746d4 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java @@ -462,7 +462,7 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser