diff --git a/library/src/main/java/com/google/android/exoplayer/chunk/ChunkSampleSource.java b/library/src/main/java/com/google/android/exoplayer/chunk/ChunkSampleSource.java
index 6c7c0d2a7a..88abe1f867 100644
--- a/library/src/main/java/com/google/android/exoplayer/chunk/ChunkSampleSource.java
+++ b/library/src/main/java/com/google/android/exoplayer/chunk/ChunkSampleSource.java
@@ -68,7 +68,6 @@ public class ChunkSampleSource implements SampleSource, TrackStream, Loader.Call
   private long pendingResetPositionUs;
   private long lastPreferredQueueSizeEvaluationTimeMs;
   private boolean pendingReset;
-  private boolean loadControlRegistered;
 
   private TrackGroupArray trackGroups;
   private long durationUs;
@@ -169,6 +168,7 @@ public class ChunkSampleSource implements SampleSource, TrackStream, Loader.Call
     Assertions.checkState(prepared);
     Assertions.checkState(oldStreams.size() <= 1);
     Assertions.checkState(newSelections.size() <= 1);
+    boolean trackWasEnabled = trackEnabled;
     // Unselect old tracks.
     if (!oldStreams.isEmpty()) {
       Assertions.checkState(trackEnabled);
@@ -185,9 +185,8 @@ public class ChunkSampleSource implements SampleSource, TrackStream, Loader.Call
     }
     // Cancel or start requests as necessary.
     if (!trackEnabled) {
-      if (loadControlRegistered) {
+      if (trackWasEnabled) {
         loadControl.unregister(this);
-        loadControlRegistered = false;
       }
       if (loader.isLoading()) {
         loader.cancelLoading();
@@ -195,10 +194,9 @@ public class ChunkSampleSource implements SampleSource, TrackStream, Loader.Call
         clearState();
         loadControl.trimAllocator();
       }
-    } else if (trackEnabled) {
-      if (!loadControlRegistered) {
+    } else {
+      if (!trackWasEnabled) {
         loadControl.register(this, bufferSizeContribution);
-        loadControlRegistered = true;
       }
       downstreamFormat = null;
       downstreamSampleFormat = null;
@@ -214,7 +212,9 @@ public class ChunkSampleSource implements SampleSource, TrackStream, Loader.Call
   public void continueBuffering(long positionUs) {
     downstreamPositionUs = positionUs;
     chunkSource.continueBuffering(positionUs);
-    maybeStartLoading();
+    if (!loader.isLoading()) {
+      maybeStartLoading();
+    }
   }
 
   @Override
@@ -253,8 +253,10 @@ public class ChunkSampleSource implements SampleSource, TrackStream, Loader.Call
 
   @Override
   public void release() {
-    prepared = false;
-    trackEnabled = false;
+    if (trackEnabled) {
+      loadControl.unregister(this);
+      trackEnabled = false;
+    }
     loader.release();
   }
 
@@ -420,10 +422,6 @@ public class ChunkSampleSource implements SampleSource, TrackStream, Loader.Call
   }
 
   private void maybeStartLoading() {
-    if (loader.isLoading()) {
-      return;
-    }
-
     long now = SystemClock.elapsedRealtime();
     if (now - lastPreferredQueueSizeEvaluationTimeMs > 5000) {
       int queueSize = chunkSource.getPreferredQueueSize(downstreamPositionUs, readOnlyMediaChunks);
diff --git a/library/src/main/java/com/google/android/exoplayer/extractor/ExtractorSampleSource.java b/library/src/main/java/com/google/android/exoplayer/extractor/ExtractorSampleSource.java
index 563d855eeb..92c6883a36 100644
--- a/library/src/main/java/com/google/android/exoplayer/extractor/ExtractorSampleSource.java
+++ b/library/src/main/java/com/google/android/exoplayer/extractor/ExtractorSampleSource.java
@@ -336,27 +336,30 @@ public final class ExtractorSampleSource implements SampleSource, ExtractorOutpu
     if (prepared) {
       return true;
     }
-    maybeStartLoading();
-    if (seekMap == null || !tracksBuilt || !haveFormatsForAllTracks()) {
-      maybeThrowError();
-      return false;
+    if (seekMap != null && tracksBuilt && haveFormatsForAllTracks()) {
+      int trackCount = sampleQueues.size();
+      TrackGroup[] trackArray = new TrackGroup[trackCount];
+      trackEnabledStates = new boolean[trackCount];
+      pendingResets = new boolean[trackCount];
+      pendingMediaFormat = new boolean[trackCount];
+      durationUs = seekMap.getDurationUs();
+      for (int i = 0; i < trackCount; i++) {
+        trackArray[i] = new TrackGroup(sampleQueues.valueAt(i).getFormat());
+      }
+      tracks = new TrackGroupArray(trackArray);
+      if (minLoadableRetryCount == MIN_RETRY_COUNT_DEFAULT_FOR_MEDIA && !seekMap.isSeekable()
+          && durationUs == C.UNKNOWN_TIME_US) {
+        loader.setMinRetryCount(DEFAULT_MIN_LOADABLE_RETRY_COUNT_LIVE);
+      }
+      prepared = true;
+      return true;
     }
-    int trackCount = sampleQueues.size();
-    TrackGroup[] trackArray = new TrackGroup[trackCount];
-    trackEnabledStates = new boolean[trackCount];
-    pendingResets = new boolean[trackCount];
-    pendingMediaFormat = new boolean[trackCount];
-    durationUs = seekMap.getDurationUs();
-    for (int i = 0; i < trackCount; i++) {
-      trackArray[i] = new TrackGroup(sampleQueues.valueAt(i).getFormat());
+    // We're not prepared.
+    maybeThrowError();
+    if (!loader.isLoading()) {
+      startLoading();
     }
-    tracks = new TrackGroupArray(trackArray);
-    if (minLoadableRetryCount == MIN_RETRY_COUNT_DEFAULT_FOR_MEDIA && !seekMap.isSeekable()
-        && durationUs == C.UNKNOWN_TIME_US) {
-      loader.setMinRetryCount(DEFAULT_MIN_LOADABLE_RETRY_COUNT_LIVE);
-    }
-    prepared = true;
-    return true;
+    return false;
   }
 
   @Override
@@ -414,10 +417,6 @@ public final class ExtractorSampleSource implements SampleSource, ExtractorOutpu
   public void continueBuffering(long playbackPositionUs) {
     downstreamPositionUs = playbackPositionUs;
     discardSamplesForDisabledTracks(downstreamPositionUs);
-    if (loadingFinished) {
-      return;
-    }
-    maybeStartLoading();
   }
 
   @Override
@@ -588,15 +587,11 @@ public final class ExtractorSampleSource implements SampleSource, ExtractorOutpu
       loader.cancelLoading();
     } else {
       clearState();
-      maybeStartLoading();
+      startLoading();
     }
   }
 
-  private void maybeStartLoading() {
-    if (loadingFinished || loader.isLoading() || fatalException != null) {
-      return;
-    }
-
+  private void startLoading() {
     sampleTimeOffsetUs = 0;
     havePendingNextSampleUs = false;
     loadable = new ExtractingLoadable(uri, dataSource, extractorHolder, allocator,
diff --git a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
index 1131496d51..8c2234f7c0 100644
--- a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
+++ b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java
@@ -69,7 +69,6 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
 
   private boolean prepared;
   private boolean seenFirstTrackSelection;
-  private boolean loadControlRegistered;
   private int enabledTrackCount;
 
   private Format downstreamFormat;
@@ -140,7 +139,6 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
         HlsExtractorWrapper extractor = extractors.getFirst();
         if (extractor.isPrepared()) {
           buildTracks(extractor);
-          maybeStartLoading(); // Update the load control.
           prepared = true;
           return true;
         } else if (extractors.size() > 1) {
@@ -150,20 +148,16 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
         }
       }
     }
-    // We're not prepared and we haven't loaded what we need.
-    if (!loadControlRegistered) {
-      loadControl.register(this, bufferSizeContribution);
-      loadControlRegistered = true;
-    }
+    // We're not prepared.
+    maybeThrowError();
     if (!loader.isLoading()) {
       // We're going to have to start loading a chunk to get what we need for preparation. We should
       // attempt to load the chunk at positionUs, so that we'll already be loading the correct chunk
       // in the common case where the renderer is subsequently enabled at this position.
       pendingResetPositionUs = positionUs;
       downstreamPositionUs = positionUs;
+      maybeStartLoading();
     }
-    maybeStartLoading();
-    maybeThrowError();
     return false;
   }
 
@@ -181,6 +175,7 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
   public TrackStream[] selectTracks(List<TrackStream> oldStreams,
       List<TrackSelection> newSelections, long positionUs) {
     Assertions.checkState(prepared);
+    boolean tracksWereEnabled = enabledTrackCount > 0;
     // Unselect old tracks.
     for (int i = 0; i < oldStreams.size(); i++) {
       int group = ((TrackStreamImpl) oldStreams.get(i)).group;
@@ -206,24 +201,22 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
       chunkSource.reset();
       downstreamPositionUs = Long.MIN_VALUE;
       downstreamFormat = null;
-      if (loader != null) {
-        if (loadControlRegistered) {
-          loadControl.unregister(this);
-          loadControlRegistered = false;
-        }
-        if (loader.isLoading()) {
-          loader.cancelLoading();
-        } else {
-          clearState();
-          loadControl.trimAllocator();
-        }
+      if (tracksWereEnabled) {
+        loadControl.unregister(this);
       }
-    } else if (primaryTracksDeselected || (seenFirstTrackSelection && newStreams.length > 0)) {
-      if (!loadControlRegistered) {
+      if (loader.isLoading()) {
+        loader.cancelLoading();
+      } else {
+        clearState();
+        loadControl.trimAllocator();
+      }
+    } else {
+      if (!tracksWereEnabled) {
         loadControl.register(this, bufferSizeContribution);
-        loadControlRegistered = true;
       }
-      seekToInternal(positionUs);
+      if (primaryTracksDeselected || (seenFirstTrackSelection && newStreams.length > 0)) {
+        seekToInternal(positionUs);
+      }
     }
     seenFirstTrackSelection = true;
     return newStreams;
@@ -235,7 +228,9 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
     if (!extractors.isEmpty()) {
       discardSamplesForDisabledTracks(getCurrentExtractor(), downstreamPositionUs);
     }
-    maybeStartLoading();
+    if (!loader.isLoading()) {
+      maybeStartLoading();
+    }
   }
 
   @Override
@@ -269,10 +264,9 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
 
   @Override
   public void release() {
-    enabledTrackCount = 0;
-    if (loadControlRegistered) {
+    if (enabledTrackCount > 0) {
       loadControl.unregister(this);
-      loadControlRegistered = false;
+      enabledTrackCount = 0;
     }
     loader.release();
   }
@@ -629,13 +623,9 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
   }
 
   private void maybeStartLoading() {
-    if (loader.isLoading()) {
-      return;
-    }
-
-    long nextLoadPositionUs = getNextLoadPositionUs();
-    boolean isNext = loadControl.update(this, downstreamPositionUs, nextLoadPositionUs, false);
-    if (!isNext || (prepared && enabledTrackCount == 0)) {
+    boolean shouldStartLoading = !prepared || (enabledTrackCount > 0
+        && loadControl.update(this, downstreamPositionUs, getNextLoadPositionUs(), false));
+    if (!shouldStartLoading) {
       return;
     }
 
@@ -648,7 +638,9 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
 
     if (endOfStream) {
       loadingFinished = true;
-      loadControl.update(this, downstreamPositionUs, -1, false);
+      if (prepared) {
+        loadControl.update(this, downstreamPositionUs, -1, false);
+      }
       return;
     }
 
@@ -676,8 +668,10 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
           currentLoadable.trigger, currentLoadable.format, -1, -1);
     }
     loader.startLoading(currentLoadable, this);
-    // Update the load control again to indicate that we're now loading.
-    loadControl.update(this, downstreamPositionUs, getNextLoadPositionUs(), true);
+    if (prepared) {
+      // Update the load control again to indicate that we're now loading.
+      loadControl.update(this, downstreamPositionUs, getNextLoadPositionUs(), true);
+    }
   }
 
   /**
@@ -688,8 +682,8 @@ public final class HlsSampleSource implements SampleSource, Loader.Callback {
     if (isPendingReset()) {
       return pendingResetPositionUs;
     } else {
-      return loadingFinished || (prepared && enabledTrackCount == 0) ? -1
-          : currentTsLoadable != null ? currentTsLoadable.endTimeUs : previousTsLoadable.endTimeUs;
+      return loadingFinished ? -1 : (currentTsLoadable != null ? currentTsLoadable.endTimeUs
+          : previousTsLoadable.endTimeUs);
     }
   }