diff --git a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
index 38239448be..36c1517fab 100644
--- a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
+++ b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
@@ -345,7 +345,12 @@ public class HlsSampleSource implements SampleSource, Loader.Callback {
     if (!currentLoadableExceptionFatal) {
       clearCurrentLoadable();
     }
-    maybeStartLoading();
+    if (enabledTrackCount > 0) {
+      maybeStartLoading();
+    } else {
+      clearState();
+      allocator.trim(0);
+    }
   }
 
   @Override