Fix DRM protected SmoothStreaming with subtitles

Issue: #5378
PiperOrigin-RevId: 229261658

RELEASENOTES.md

@@ -4,6 +4,8 @@
 
 * IMA extension: Clear ads loader listeners on release
   ([#4114](https://github.com/google/ExoPlayer/issues/4114)).
+* SmoothStreaming: Fix support for subtitles in DRM protected streams
+  ([#5378](https://github.com/google/ExoPlayer/issues/5378)).
 * FFmpeg extension: Treat invalid data errors as non-fatal to match the behavior
   of MediaCodec ([#5293](https://github.com/google/ExoPlayer/issues/5293)).
 * Fix issue where sending callbacks for playlist changes may cause problems

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/DefaultSsChunkSource.java

@@ -61,14 +61,13 @@ public class DefaultSsChunkSource implements SsChunkSource {
         SsManifest manifest,
         int elementIndex,
         TrackSelection trackSelection,
-        TrackEncryptionBox[] trackEncryptionBoxes,
         @Nullable TransferListener transferListener) {
       DataSource dataSource = dataSourceFactory.createDataSource();
       if (transferListener != null) {
         dataSource.addTransferListener(transferListener);
       }
-      return new DefaultSsChunkSource(manifestLoaderErrorThrower, manifest, elementIndex,
+      return new DefaultSsChunkSource(
-          trackSelection, dataSource, trackEncryptionBoxes);
+          manifestLoaderErrorThrower, manifest, elementIndex, trackSelection, dataSource);
     }
 
   }
@@ -90,15 +89,13 @@ public class DefaultSsChunkSource implements SsChunkSource {
    * @param streamElementIndex The index of the stream element in the manifest.
    * @param trackSelection The track selection.
    * @param dataSource A {@link DataSource} suitable for loading the media data.
-   * @param trackEncryptionBoxes Track encryption boxes for the stream.
    */
   public DefaultSsChunkSource(
       LoaderErrorThrower manifestLoaderErrorThrower,
       SsManifest manifest,
       int streamElementIndex,
       TrackSelection trackSelection,
-      DataSource dataSource,
+      DataSource dataSource) {
-      TrackEncryptionBox[] trackEncryptionBoxes) {
     this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
     this.manifest = manifest;
     this.streamElementIndex = streamElementIndex;
@@ -110,6 +107,8 @@ public class DefaultSsChunkSource implements SsChunkSource {
     for (int i = 0; i < extractorWrappers.length; i++) {
       int manifestTrackIndex = trackSelection.getIndexInTrackGroup(i);
       Format format = streamElement.formats[manifestTrackIndex];
+      TrackEncryptionBox[] trackEncryptionBoxes =
+          format.drmInitData != null ? manifest.protectionElement.trackEncryptionBoxes : null;
       int nalUnitLengthFieldLength = streamElement.type == C.TRACK_TYPE_VIDEO ? 4 : 0;
       Track track = new Track(manifestTrackIndex, streamElement.type, streamElement.timescale,
           C.TIME_UNSET, manifest.durationUs, format, Track.TRANSFORMATION_NONE,

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsChunkSource.java

@@ -16,7 +16,6 @@
 package com.google.android.exoplayer2.source.smoothstreaming;
 
 import android.support.annotation.Nullable;
-import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
 import com.google.android.exoplayer2.source.chunk.ChunkSource;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest;
 import com.google.android.exoplayer2.trackselection.TrackSelection;
@@ -38,7 +37,6 @@ public interface SsChunkSource extends ChunkSource {
      * @param manifest The initial manifest.
      * @param streamElementIndex The index of the corresponding stream element in the manifest.
      * @param trackSelection The track selection.
-     * @param trackEncryptionBoxes Track encryption boxes for the stream.
      * @param transferListener The transfer listener which should be informed of any data transfers.
      *     May be null if no listener is available.
      * @return The created {@link SsChunkSource}.
@@ -48,7 +46,6 @@ public interface SsChunkSource extends ChunkSource {
         SsManifest manifest,
         int streamElementIndex,
         TrackSelection trackSelection,
-        TrackEncryptionBox[] trackEncryptionBoxes,
         @Nullable TransferListener transferListener);
   }
 

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriod.java

@@ -29,7 +29,6 @@ import com.google.android.exoplayer2.source.TrackGroup;
 import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.source.chunk.ChunkSampleStream;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest;
-import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest.ProtectionElement;
 import com.google.android.exoplayer2.trackselection.TrackSelection;
 import com.google.android.exoplayer2.upstream.Allocator;
 import com.google.android.exoplayer2.upstream.LoadErrorHandlingPolicy;
@@ -44,8 +43,6 @@ import java.util.ArrayList;
 /* package */ final class SsMediaPeriod implements MediaPeriod,
     SequenceableLoader.Callback<ChunkSampleStream<SsChunkSource>> {
 
-  private static final int INITIALIZATION_VECTOR_SIZE = 8;
-
   private final SsChunkSource.Factory chunkSourceFactory;
   private final @Nullable TransferListener transferListener;
   private final LoaderErrorThrower manifestLoaderErrorThrower;
@@ -53,7 +50,6 @@ import java.util.ArrayList;
   private final EventDispatcher eventDispatcher;
   private final Allocator allocator;
   private final TrackGroupArray trackGroups;
-  private final TrackEncryptionBox[] trackEncryptionBoxes;
   private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
 
   private @Nullable Callback callback;
@@ -71,6 +67,7 @@ import java.util.ArrayList;
       EventDispatcher eventDispatcher,
       LoaderErrorThrower manifestLoaderErrorThrower,
       Allocator allocator) {
+    this.manifest = manifest;
     this.chunkSourceFactory = chunkSourceFactory;
     this.transferListener = transferListener;
     this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
@@ -78,18 +75,7 @@ import java.util.ArrayList;
     this.eventDispatcher = eventDispatcher;
     this.allocator = allocator;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
-
     trackGroups = buildTrackGroups(manifest);
-    ProtectionElement protectionElement = manifest.protectionElement;
-    if (protectionElement != null) {
-      byte[] keyId = getProtectionElementKeyId(protectionElement.data);
-      // We assume pattern encryption does not apply.
-      trackEncryptionBoxes = new TrackEncryptionBox[] {
-          new TrackEncryptionBox(true, null, INITIALIZATION_VECTOR_SIZE, keyId, 0, 0, null)};
-    } else {
-      trackEncryptionBoxes = null;
-    }
-    this.manifest = manifest;
     sampleStreams = newSampleStreamArray(0);
     compositeSequenceableLoader =
         compositeSequenceableLoaderFactory.createCompositeSequenceableLoader(sampleStreams);
@@ -229,7 +215,6 @@ import java.util.ArrayList;
             manifest,
             streamElementIndex,
             selection,
-            trackEncryptionBoxes,
             transferListener);
     return new ChunkSampleStream<>(
         manifest.streamElements[streamElementIndex].type,
@@ -277,5 +262,4 @@ import java.util.ArrayList;
     data[firstPosition] = data[secondPosition];
     data[secondPosition] = temp;
   }
-
 }

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/manifest/SsManifest.java

@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.source.smoothstreaming.manifest;
 import android.net.Uri;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
 import com.google.android.exoplayer2.offline.FilterableManifest;
 import com.google.android.exoplayer2.offline.StreamKey;
 import com.google.android.exoplayer2.util.Assertions;
@@ -41,10 +42,12 @@ public class SsManifest implements FilterableManifest<SsManifest> {
 
     public final UUID uuid;
     public final byte[] data;
+    public final TrackEncryptionBox[] trackEncryptionBoxes;
 
-    public ProtectionElement(UUID uuid, byte[] data) {
+    public ProtectionElement(UUID uuid, byte[] data, TrackEncryptionBox[] trackEncryptionBoxes) {
       this.uuid = uuid;
       this.data = data;
+      this.trackEncryptionBoxes = trackEncryptionBoxes;
     }
   }
 

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/manifest/SsManifestParser.java

@@ -25,6 +25,7 @@ import com.google.android.exoplayer2.ParserException;
 import com.google.android.exoplayer2.drm.DrmInitData;
 import com.google.android.exoplayer2.drm.DrmInitData.SchemeData;
 import com.google.android.exoplayer2.extractor.mp4.PsshAtomUtil;
+import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest.ProtectionElement;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest.StreamElement;
 import com.google.android.exoplayer2.upstream.ParsingLoadable;
@@ -397,9 +398,10 @@ public class SsManifestParser implements ParsingLoadable.Parser<SsManifest> {
 
     public static final String TAG = "Protection";
     public static final String TAG_PROTECTION_HEADER = "ProtectionHeader";
-
     public static final String KEY_SYSTEM_ID = "SystemID";
 
+    private static final int INITIALIZATION_VECTOR_SIZE = 8;
+
     private boolean inProtectionHeader;
     private UUID uuid;
     private byte[] initData;
@@ -439,7 +441,44 @@ public class SsManifestParser implements ParsingLoadable.Parser<SsManifest> {
 
     @Override
     public Object build() {
-      return new ProtectionElement(uuid, PsshAtomUtil.buildPsshAtom(uuid, initData));
+      return new ProtectionElement(
+          uuid, PsshAtomUtil.buildPsshAtom(uuid, initData), buildTrackEncryptionBoxes(initData));
+    }
+
+    private static TrackEncryptionBox[] buildTrackEncryptionBoxes(byte[] initData) {
+      return new TrackEncryptionBox[] {
+        new TrackEncryptionBox(
+            /* isEncrypted= */ true,
+            /* schemeType= */ null,
+            INITIALIZATION_VECTOR_SIZE,
+            getProtectionElementKeyId(initData),
+            /* defaultEncryptedBlocks= */ 0,
+            /* defaultClearBlocks= */ 0,
+            /* defaultInitializationVector= */ null)
+      };
+    }
+
+    private static byte[] getProtectionElementKeyId(byte[] initData) {
+      StringBuilder initDataStringBuilder = new StringBuilder();
+      for (int i = 0; i < initData.length; i += 2) {
+        initDataStringBuilder.append((char) initData[i]);
+      }
+      String initDataString = initDataStringBuilder.toString();
+      String keyIdString =
+          initDataString.substring(
+              initDataString.indexOf("<KID>") + 5, initDataString.indexOf("</KID>"));
+      byte[] keyId = Base64.decode(keyIdString, Base64.DEFAULT);
+      swap(keyId, 0, 3);
+      swap(keyId, 1, 2);
+      swap(keyId, 4, 5);
+      swap(keyId, 6, 7);
+      return keyId;
+    }
+
+    private static void swap(byte[] data, int firstPosition, int secondPosition) {
+      byte temp = data[firstPosition];
+      data[firstPosition] = data[secondPosition];
+      data[secondPosition] = temp;
     }
 
     private static String stripCurlyBraces(String uuidString) {

library/smoothstreaming/src/test/assets/sample_ismc_1

@@ -3,7 +3,7 @@
     Duration="2300000000" TimeScale="10000000">
     <Protection>
         <ProtectionHeader SystemID="9A04F079-9840-4286-AB92-E65BE0885F95">
-            <!-- Base 64-Encoded data omitted for clarity -->
+            fgMAAAEAAQB0AzwAVwBSAE0ASABFAEEARABFAFIAIAB4AG0AbABuAHMAPQAiAGgAdAB0AHAAOgAvAC8AcwBjAGgAZQBtAGEAcwAuAG0AaQBjAHIAbwBzAG8AZgB0AC4AYwBvAG0ALwBEAFIATQAvADIAMAAwADcALwAwADMALwBQAGwAYQB5AFIAZQBhAGQAeQBIAGUAYQBkAGUAcgAiACAAdgBlAHIAcwBpAG8AbgA9ACIANAAuADAALgAwAC4AMAAiAD4APABEAEEAVABBAD4APABQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsARQBZAEwARQBOAD4AMQA2ADwALwBLAEUAWQBMAEUATgA+ADwAQQBMAEcASQBEAD4AQQBFAFMAQwBUAFIAPAAvAEEATABHAEkARAA+ADwALwBQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsASQBEAD4AQgBhAFUATQBPAEcAYwBzAGgAVQBDAEQAZAB3ADMANABZAGMAawBmAFoAQQA9AD0APAAvAEsASQBEAD4APABDAEgARQBDAEsAUwBVAE0APgBnADcATgBhAFIARABJAEkATwA5ADAAPQA8AC8AQwBIAEUAQwBLAFMAVQBNAD4APABMAEEAXwBVAFIATAA+AGgAdAB0AHAAcwA6AC8ALwBUAC0ATwBOAEwASQBOAEUALgBEAFUATQBNAFkALQBTAEUAUgBWAEUAUgAvAEEAcgB0AGUAbQBpAHMATABpAGMAZQBuAHMAZQBTAGUAcgB2AGUAcgAvAFAAbABhAHkAUgBlAGEAZAB5AE0AYQBuAGEAZwBlAHIALgBhAHMAbQB4ADwALwBMAEEAXwBVAFIATAA+ADwAQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwAQwBJAEQAPgAxADcANQA4ADIANgA8AC8AQwBJAEQAPgA8AEkASQBTAF8ARABSAE0AXwBWAEUAUgBTAEkATwBOAD4ANwAuADEALgAxADUANgA1AC4ANAA8AC8ASQBJAFMAXwBEAFIATQBfAFYARQBSAFMASQBPAE4APgA8AC8AQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwALwBEAEEAVABBAD4APAAvAFcAUgBNAEgARQBBAEQARQBSAD4A
         </ProtectionHeader>
     </Protection>
 

library/smoothstreaming/src/test/assets/sample_ismc_2

@@ -3,7 +3,7 @@
     Duration="2300000000" TimeScale="10000000">
     <Protection>
         <ProtectionHeader SystemID="{9A04F079-9840-4286-AB92-E65BE0885F95}">
-            <!-- Base 64-Encoded data omitted for clarity -->
+            fgMAAAEAAQB0AzwAVwBSAE0ASABFAEEARABFAFIAIAB4AG0AbABuAHMAPQAiAGgAdAB0AHAAOgAvAC8AcwBjAGgAZQBtAGEAcwAuAG0AaQBjAHIAbwBzAG8AZgB0AC4AYwBvAG0ALwBEAFIATQAvADIAMAAwADcALwAwADMALwBQAGwAYQB5AFIAZQBhAGQAeQBIAGUAYQBkAGUAcgAiACAAdgBlAHIAcwBpAG8AbgA9ACIANAAuADAALgAwAC4AMAAiAD4APABEAEEAVABBAD4APABQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsARQBZAEwARQBOAD4AMQA2ADwALwBLAEUAWQBMAEUATgA+ADwAQQBMAEcASQBEAD4AQQBFAFMAQwBUAFIAPAAvAEEATABHAEkARAA+ADwALwBQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsASQBEAD4AQgBhAFUATQBPAEcAYwBzAGgAVQBDAEQAZAB3ADMANABZAGMAawBmAFoAQQA9AD0APAAvAEsASQBEAD4APABDAEgARQBDAEsAUwBVAE0APgBnADcATgBhAFIARABJAEkATwA5ADAAPQA8AC8AQwBIAEUAQwBLAFMAVQBNAD4APABMAEEAXwBVAFIATAA+AGgAdAB0AHAAcwA6AC8ALwBUAC0ATwBOAEwASQBOAEUALgBEAFUATQBNAFkALQBTAEUAUgBWAEUAUgAvAEEAcgB0AGUAbQBpAHMATABpAGMAZQBuAHMAZQBTAGUAcgB2AGUAcgAvAFAAbABhAHkAUgBlAGEAZAB5AE0AYQBuAGEAZwBlAHIALgBhAHMAbQB4ADwALwBMAEEAXwBVAFIATAA+ADwAQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwAQwBJAEQAPgAxADcANQA4ADIANgA8AC8AQwBJAEQAPgA8AEkASQBTAF8ARABSAE0AXwBWAEUAUgBTAEkATwBOAD4ANwAuADEALgAxADUANgA1AC4ANAA8AC8ASQBJAFMAXwBEAFIATQBfAFYARQBSAFMASQBPAE4APgA8AC8AQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwALwBEAEEAVABBAD4APAAvAFcAUgBNAEgARQBBAEQARQBSAD4A
         </ProtectionHeader>
     </Protection>