diff --git a/library/core/src/main/java/com/google/android/exoplayer2/Playlist.java b/library/core/src/main/java/com/google/android/exoplayer2/Playlist.java
index 58158239d6..c5476a151b 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/Playlist.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/Playlist.java
@@ -132,6 +132,11 @@ import java.util.Set;
         mediaSourceByUid.put(holder.uid, holder);
         if (isPrepared) {
           prepareChildSource(holder);
+          if (mediaSourceByMediaPeriod.isEmpty()) {
+            enabledMediaSourceHolders.add(holder);
+          } else {
+            disableChildSource(holder);
+          }
         }
       }
     }
@@ -267,6 +272,7 @@ import java.util.Set;
     for (int i = 0; i < mediaSourceHolders.size(); i++) {
       MediaSourceHolder mediaSourceHolder = mediaSourceHolders.get(i);
       prepareChildSource(mediaSourceHolder);
+      enabledMediaSourceHolders.add(mediaSourceHolder);
     }
     isPrepared = true;
   }
@@ -357,15 +363,19 @@ import java.util.Set;
     while (iterator.hasNext()) {
       MediaSourceHolder holder = iterator.next();
       if (holder.activeMediaPeriodIds.isEmpty()) {
-        @Nullable MediaSourceAndListener disabledChild = childSources.get(holder);
-        if (disabledChild != null) {
-          disabledChild.mediaSource.disable(disabledChild.caller);
-        }
+        disableChildSource(holder);
         iterator.remove();
       }
     }
   }
 
+  private void disableChildSource(MediaSourceHolder holder) {
+    @Nullable MediaSourceAndListener disabledChild = childSources.get(holder);
+    if (disabledChild != null) {
+      disabledChild.mediaSource.disable(disabledChild.caller);
+    }
+  }
+
   private void removeMediaSourcesInternal(int fromIndex, int toIndex) {
     for (int index = toIndex - 1; index >= fromIndex; index--) {
       MediaSourceHolder holder = mediaSourceHolders.remove(index);