diff --git a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
index dbb1b8959b..b92c4b7f1e 100644
--- a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
+++ b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
@@ -362,11 +362,8 @@ public class HlsSampleSource implements SampleSource, Loader.Callback {
     if (!currentLoadableExceptionFatal) {
       clearCurrentLoadable();
     }
-    if (enabledTrackCount > 0) {
+    if (enabledTrackCount > 0 || !prepared) {
       maybeStartLoading();
-    } else {
-      clearState();
-      loadControl.trimAllocator();
     }
   }