Add video and closed caption renditions to HlsMasterPlaylist

Still skipping any renditions that have a null URL, which means that
closedCaptions in particular is always empty. This restriction will
be removed in a subsequent change, which will also remove muxedAudioFormat
and muxedAudioCaptions from HlsMasterPlaylist and instead derive them
from the Variant and Rendition information in HlsMediaPeriod.

Issue: #5596
Issue: #2600
PiperOrigin-RevId: 240623500

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaPeriod.java

@@ -157,6 +157,9 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
     return trackGroups;
   }
 
+  // TODO: When the master playlist does not de-duplicate variants by URL and allows Renditions with
+  // null URLs, this method must be updated to calculate stream keys that are compatible with those
+  // that may already be persisted for offline.
   @Override
   public List<StreamKey> getStreamKeys(List<TrackSelection> trackSelections) {
     // See HlsMasterPlaylist.copy for interpretation of StreamKeys.

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsMasterPlaylist.java

@@ -34,14 +34,17 @@ public final class HlsMasterPlaylist extends HlsPlaylist {
           /* baseUri= */ "",
           /* tags= */ Collections.emptyList(),
           /* variants= */ Collections.emptyList(),
+          /* videos= */ Collections.emptyList(),
           /* audios= */ Collections.emptyList(),
           /* subtitles= */ Collections.emptyList(),
+          /* closedCaptions= */ Collections.emptyList(),
           /* muxedAudioFormat= */ null,
           /* muxedCaptionFormats= */ Collections.emptyList(),
           /* hasIndependentSegments= */ false,
           /* variableDefinitions= */ Collections.emptyMap(),
           /* sessionKeyDrmInitData= */ Collections.emptyList());
 
+  // These constants must not be changed because they are persisted in offline stream keys.
   public static final int GROUP_INDEX_VARIANT = 0;
   public static final int GROUP_INDEX_AUDIO = 1;
   public static final int GROUP_INDEX_SUBTITLE = 2;
@@ -156,12 +159,16 @@ public final class HlsMasterPlaylist extends HlsPlaylist {
 
   }
 
-  /** The list of variants declared by the playlist. */
+  /** The variants declared by the playlist. */
   public final List<Variant> variants;
-  /** The list of demuxed audios declared by the playlist. */
+  /** The video renditions declared by the playlist. */
+  public final List<Rendition> videos;
+  /** The audio renditions declared by the playlist. */
   public final List<Rendition> audios;
-  /** The list of subtitles declared by the playlist. */
+  /** The subtitle renditions declared by the playlist. */
   public final List<Rendition> subtitles;
+  /** The closed caption renditions declared by the playlist. */
+  public final List<Rendition> closedCaptions;
 
   /**
    * The format of the audio muxed in the variants. May be null if the playlist does not declare any
@@ -183,8 +190,10 @@ public final class HlsMasterPlaylist extends HlsPlaylist {
    * @param baseUri See {@link #baseUri}.
    * @param tags See {@link #tags}.
    * @param variants See {@link #variants}.
+   * @param videos See {@link #videos}.
    * @param audios See {@link #audios}.
    * @param subtitles See {@link #subtitles}.
+   * @param closedCaptions See {@link #closedCaptions}.
    * @param muxedAudioFormat See {@link #muxedAudioFormat}.
    * @param muxedCaptionFormats See {@link #muxedCaptionFormats}.
    * @param hasIndependentSegments See {@link #hasIndependentSegments}.
@@ -195,8 +204,10 @@ public final class HlsMasterPlaylist extends HlsPlaylist {
       String baseUri,
       List<String> tags,
       List<Variant> variants,
+      List<Rendition> videos,
       List<Rendition> audios,
       List<Rendition> subtitles,
+      List<Rendition> closedCaptions,
       Format muxedAudioFormat,
       List<Format> muxedCaptionFormats,
       boolean hasIndependentSegments,
@@ -204,8 +215,10 @@ public final class HlsMasterPlaylist extends HlsPlaylist {
       List<DrmInitData> sessionKeyDrmInitData) {
     super(baseUri, tags, hasIndependentSegments);
     this.variants = Collections.unmodifiableList(variants);
+    this.videos = Collections.unmodifiableList(videos);
     this.audios = Collections.unmodifiableList(audios);
     this.subtitles = Collections.unmodifiableList(subtitles);
+    this.closedCaptions = Collections.unmodifiableList(closedCaptions);
     this.muxedAudioFormat = muxedAudioFormat;
     this.muxedCaptionFormats = muxedCaptionFormats != null
         ? Collections.unmodifiableList(muxedCaptionFormats) : null;
@@ -218,9 +231,13 @@ public final class HlsMasterPlaylist extends HlsPlaylist {
     return new HlsMasterPlaylist(
         baseUri,
         tags,
-        copyRenditionsList(variants, GROUP_INDEX_VARIANT, streamKeys),
+        copyStreams(variants, GROUP_INDEX_VARIANT, streamKeys),
-        copyRenditionsList(audios, GROUP_INDEX_AUDIO, streamKeys),
+        // TODO: Allow stream keys to specify video renditions to be retained.
-        copyRenditionsList(subtitles, GROUP_INDEX_SUBTITLE, streamKeys),
+        /* videos= */ Collections.emptyList(),
+        copyStreams(audios, GROUP_INDEX_AUDIO, streamKeys),
+        copyStreams(subtitles, GROUP_INDEX_SUBTITLE, streamKeys),
+        // TODO: Update to retain all closed captions.
+        /* closedCaptions= */ Collections.emptyList(),
         muxedAudioFormat,
         muxedCaptionFormats,
         hasIndependentSegments,
@@ -238,11 +255,13 @@ public final class HlsMasterPlaylist extends HlsPlaylist {
     List<Variant> variant =
         Collections.singletonList(Variant.createMediaPlaylistVariantUrl(variantUrl));
     return new HlsMasterPlaylist(
-        null,
+        /* baseUri= */ null,
-        Collections.emptyList(),
+        /* tags= */ Collections.emptyList(),
         variant,
-        Collections.emptyList(),
+        /* videos= */ Collections.emptyList(),
-        Collections.emptyList(),
+        /* audios= */ Collections.emptyList(),
+        /* subtitles= */ Collections.emptyList(),
+        /* closedCaptions= */ Collections.emptyList(),
         /* muxedAudioFormat= */ null,
         /* muxedCaptionFormats= */ null,
         /* hasIndependentSegments= */ false,
@@ -250,20 +269,30 @@ public final class HlsMasterPlaylist extends HlsPlaylist {
         /* sessionKeyDrmInitData= */ Collections.emptyList());
   }
 
-  private static <T extends HlsUrl> List<T> copyRenditionsList(
+  private static <T extends HlsUrl> List<T> copyStreams(
-      List<T> renditions, int groupIndex, List<StreamKey> streamKeys) {
+      List<T> streams, int groupIndex, List<StreamKey> streamKeys) {
-    List<T> copiedRenditions = new ArrayList<>(streamKeys.size());
+    List<T> copiedStreams = new ArrayList<>(streamKeys.size());
-    for (int i = 0; i < renditions.size(); i++) {
+    // TODO:
-      T rendition = renditions.get(i);
+    // 1. When variants with the same URL are not de-duplicated, duplicates must not increment
+    //    trackIndex so as to avoid breaking stream keys that have been persisted for offline. All
+    //    duplicates should be copied if the first variant is copied, or discarded otherwise.
+    // 2. When renditions with null URLs are permitted, they must not increment trackIndex so as to
+    //    avoid breaking stream keys that have been persisted for offline. All renitions with null
+    //    URLs should be copied. They may become unreachable if all variants that reference them are
+    //    removed, but this is OK.
+    // 3. Renditions with URLs matching copied variants should always themselves be copied, even if
+    //    the corresponding stream key is omitted. Else we're throwing away information for no gain.
+    for (int i = 0; i < streams.size(); i++) {
+      T stream = streams.get(i);
       for (int j = 0; j < streamKeys.size(); j++) {
         StreamKey streamKey = streamKeys.get(j);
         if (streamKey.groupIndex == groupIndex && streamKey.trackIndex == i) {
-          copiedRenditions.add(rendition);
+          copiedStreams.add(stream);
           break;
         }
       }
     }
-    return copiedRenditions;
+    return copiedStreams;
   }
 
 }

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java

@@ -252,11 +252,13 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
   private static HlsMasterPlaylist parseMasterPlaylist(LineIterator iterator, String baseUri)
       throws IOException {
     HashSet<String> variantUrls = new HashSet<>();
-    HashMap<String, String> audioGroupIdToCodecs = new HashMap<>();
     HashMap<String, String> variableDefinitions = new HashMap<>();
     ArrayList<Variant> variants = new ArrayList<>();
+    ArrayList<Variant> deduplicatedVariants = new ArrayList<>();
+    ArrayList<Rendition> videos = new ArrayList<>();
     ArrayList<Rendition> audios = new ArrayList<>();
     ArrayList<Rendition> subtitles = new ArrayList<>();
+    ArrayList<Rendition> closedCaptions = new ArrayList<>();
     ArrayList<String> mediaTags = new ArrayList<>();
     ArrayList<DrmInitData> sessionKeyDrmInitData = new ArrayList<>();
     ArrayList<String> tags = new ArrayList<>();
@@ -332,35 +334,30 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
             parseOptionalStringAttr(line, REGEX_SUBTITLES, variableDefinitions);
         String closedCaptionsGroupId =
             parseOptionalStringAttr(line, REGEX_CLOSED_CAPTIONS, variableDefinitions);
-        if (audioGroupId != null && codecs != null) {
-          audioGroupIdToCodecs.put(audioGroupId, Util.getCodecsOfType(codecs, C.TRACK_TYPE_AUDIO));
-        }
         line =
             replaceVariableReferences(
                 iterator.next(), variableDefinitions); // #EXT-X-STREAM-INF's URI.
+        Format format =
+            Format.createVideoContainerFormat(
+                /* id= */ Integer.toString(variants.size()),
+                /* label= */ null,
+                /* containerMimeType= */ MimeTypes.APPLICATION_M3U8,
+                /* sampleMimeType= */ null,
+                codecs,
+                bitrate,
+                width,
+                height,
+                frameRate,
+                /* initializationData= */ null,
+                /* selectionFlags= */ 0,
+                /* roleFlags= */ 0);
+        Variant variant =
+            new Variant(
+                line, format, videoGroupId, audioGroupId, subtitlesGroupId, closedCaptionsGroupId);
+        variants.add(variant);
+        // TODO: Don't deduplicate variants by URL.
         if (variantUrls.add(line)) {
-          Format format =
+          deduplicatedVariants.add(variant);
-              Format.createVideoContainerFormat(
-                  /* id= */ Integer.toString(variants.size()),
-                  /* label= */ null,
-                  /* containerMimeType= */ MimeTypes.APPLICATION_M3U8,
-                  /* sampleMimeType= */ null,
-                  codecs,
-                  bitrate,
-                  width,
-                  height,
-                  frameRate,
-                  /* initializationData= */ null,
-                  /* selectionFlags= */ 0,
-                  /* roleFlags= */ 0);
-          variants.add(
-              new Variant(
-                  line,
-                  format,
-                  videoGroupId,
-                  audioGroupId,
-                  subtitlesGroupId,
-                  closedCaptionsGroupId));
         }
       }
     }
@@ -376,10 +373,48 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
       String formatId = groupId + ":" + name;
       Format format;
       switch (parseStringAttr(line, REGEX_TYPE, variableDefinitions)) {
-        case TYPE_AUDIO:
+        case TYPE_VIDEO:
-          String codecs = audioGroupIdToCodecs.get(groupId);
+          Variant variant = getVariantWithVideoGroup(variants, groupId);
-          int channelCount = parseChannelsAttribute(line, variableDefinitions);
+          String codecs = null;
+          int width = Format.NO_VALUE;
+          int height = Format.NO_VALUE;
+          float frameRate = Format.NO_VALUE;
+          if (variant != null) {
+            Format variantFormat = variant.format;
+            codecs = Util.getCodecsOfType(variantFormat.codecs, C.TRACK_TYPE_VIDEO);
+            width = variantFormat.width;
+            height = variantFormat.height;
+            frameRate = variantFormat.frameRate;
+          }
           String sampleMimeType = codecs != null ? MimeTypes.getMediaMimeType(codecs) : null;
+          format =
+              Format.createVideoContainerFormat(
+                  /* id= */ formatId,
+                  /* label= */ name,
+                  /* containerMimeType= */ MimeTypes.APPLICATION_M3U8,
+                  sampleMimeType,
+                  codecs,
+                  /* bitrate= */ Format.NO_VALUE,
+                  width,
+                  height,
+                  frameRate,
+                  /* initializationData= */ null,
+                  selectionFlags,
+                  roleFlags);
+          if (uri == null) {
+            // TODO: Remove this case and add a Rendition with a null uri to videos.
+          } else {
+            videos.add(new Rendition(uri, format, groupId, name));
+          }
+          break;
+        case TYPE_AUDIO:
+          variant = getVariantWithAudioGroup(variants, groupId);
+          codecs =
+              variant != null
+                  ? Util.getCodecsOfType(variant.format.codecs, C.TRACK_TYPE_AUDIO)
+                  : null;
+          sampleMimeType = codecs != null ? MimeTypes.getMediaMimeType(codecs) : null;
+          int channelCount = parseChannelsAttribute(line, variableDefinitions);
           format =
               Format.createAudioContainerFormat(
                   /* id= */ formatId,
@@ -395,6 +430,7 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
                   roleFlags,
                   language);
           if (uri == null) {
+            // TODO: Remove muxedAudioFormat and add a Rendition with a null uri to audios.
             muxedAudioFormat = format;
           } else {
             audios.add(new Rendition(uri, format, groupId, name));
@@ -440,6 +476,7 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
                   roleFlags,
                   language,
                   accessibilityChannel));
+          // TODO: Remove muxedCaptionFormats and add a Rendition with a null uri to closedCaptions.
           break;
         default:
           // Do nothing.
@@ -454,9 +491,11 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
     return new HlsMasterPlaylist(
         baseUri,
         tags,
-        variants,
+        deduplicatedVariants,
+        videos,
         audios,
         subtitles,
+        closedCaptions,
         muxedAudioFormat,
         muxedCaptionFormats,
         hasIndependentSegmentsTag,
@@ -464,6 +503,26 @@ public final class HlsPlaylistParser implements ParsingLoadable.Parser<HlsPlayli
         sessionKeyDrmInitData);
   }
 
+  private static Variant getVariantWithAudioGroup(ArrayList<Variant> variants, String groupId) {
+    for (int i = 0; i < variants.size(); i++) {
+      Variant variant = variants.get(i);
+      if (groupId.equals(variant.audioGroupId)) {
+        return variant;
+      }
+    }
+    return null;
+  }
+
+  private static Variant getVariantWithVideoGroup(ArrayList<Variant> variants, String groupId) {
+    for (int i = 0; i < variants.size(); i++) {
+      Variant variant = variants.get(i);
+      if (groupId.equals(variant.videoGroupId)) {
+        return variant;
+      }
+    }
+    return null;
+  }
+
   private static HlsMediaPlaylist parseMediaPlaylist(
       HlsMasterPlaylist masterPlaylist, LineIterator iterator, String baseUri) throws IOException {
     @HlsMediaPlaylist.PlaylistType int playlistType = HlsMediaPlaylist.PLAYLIST_TYPE_UNKNOWN;

library/hls/src/test/java/com/google/android/exoplayer2/source/hls/HlsMediaPeriodTest.java

@@ -107,8 +107,10 @@ public final class HlsMediaPeriodTest {
         "http://baseUri",
         /* tags= */ Collections.emptyList(),
         variants,
+        /* videos= */ Collections.emptyList(),
         audios,
         subtitles,
+        /* closedCaptions= */ Collections.emptyList(),
         muxedAudioFormat,
         muxedCaptionFormats,
         /* hasIndependentSegments= */ true,

library/hls/src/test/java/com/google/android/exoplayer2/source/hls/playlist/HlsMediaPlaylistParserTest.java

@@ -451,8 +451,10 @@ public class HlsMediaPlaylistParserTest {
             /* baseUri= */ "https://example.com/",
             /* tags= */ Collections.emptyList(),
             /* variants= */ Collections.emptyList(),
+            /* videos= */ Collections.emptyList(),
             /* audios= */ Collections.emptyList(),
             /* subtitles= */ Collections.emptyList(),
+            /* closedCaptions= */ Collections.emptyList(),
             /* muxedAudioFormat= */ null,
             /* muxedCaptionFormats= */ null,
             /* hasIndependentSegments= */ true,
@@ -511,8 +513,10 @@ public class HlsMediaPlaylistParserTest {
             /* baseUri= */ "",
             /* tags= */ Collections.emptyList(),
             /* variants= */ Collections.emptyList(),
+            /* videos= */ Collections.emptyList(),
             /* audios= */ Collections.emptyList(),
             /* subtitles= */ Collections.emptyList(),
+            /* closedCaptions= */ Collections.emptyList(),
             /* muxedAudioFormat= */ null,
             /* muxedCaptionFormats= */ Collections.emptyList(),
             /* hasIndependentSegments= */ false,