diff --git a/library/src/main/java/com/google/android/exoplayer/ExoPlayerImplInternal.java b/library/src/main/java/com/google/android/exoplayer/ExoPlayerImplInternal.java
index e7783e9c1a..f18bfe1463 100644
--- a/library/src/main/java/com/google/android/exoplayer/ExoPlayerImplInternal.java
+++ b/library/src/main/java/com/google/android/exoplayer/ExoPlayerImplInternal.java
@@ -364,7 +364,7 @@ import java.util.concurrent.atomic.AtomicInteger;
     long operationStartTimeMs = SystemClock.elapsedRealtime();
     if (sampleSource == null) {
       timeline.updateSources();
-      sampleSource = timeline.getSampleSource(internalPositionUs);
+      sampleSource = timeline.getSampleSource();
       if (sampleSource != null) {
         resumeInternal();
       } else {
@@ -380,7 +380,7 @@ import java.util.concurrent.atomic.AtomicInteger;
       // Process reset if there is one, else update the position.
       if (!checkForSourceResetInternal()) {
         updatePositionUs();
-        sampleSource = timeline.getSampleSource(internalPositionUs);
+        sampleSource = timeline.getSampleSource();
       }
       updateBufferedPositionUs();
       timeline.updateSources();
@@ -405,17 +405,17 @@ import java.util.concurrent.atomic.AtomicInteger;
       allRenderersReadyOrEnded = allRenderersReadyOrEnded && rendererReadyOrEnded;
     }
 
+    boolean timelineIsReady = timeline.isReady();
     if (allRenderersEnded && (durationUs == C.UNSET_TIME_US || durationUs <= positionUs)) {
       setState(ExoPlayer.STATE_ENDED);
       stopRenderers();
     } else if (state == ExoPlayer.STATE_BUFFERING && allRenderersReadyOrEnded
-        && haveSufficientBuffer() && timeline.isReady(internalPositionUs)) {
+        && haveSufficientBuffer() && timelineIsReady) {
       setState(ExoPlayer.STATE_READY);
       if (playWhenReady) {
         startRenderers();
       }
-    } else if (state == ExoPlayer.STATE_READY && (!allRenderersReadyOrEnded
-        || !timeline.isReady(internalPositionUs))) {
+    } else if (state == ExoPlayer.STATE_READY && (!allRenderersReadyOrEnded || !timelineIsReady)) {
       rebuffering = playWhenReady;
       setState(ExoPlayer.STATE_BUFFERING);
       stopRenderers();
@@ -485,9 +485,8 @@ import java.util.concurrent.atomic.AtomicInteger;
     if (allRenderersEnded && (durationUs == C.UNSET_TIME_US || durationUs <= positionUs)) {
       setState(ExoPlayer.STATE_ENDED);
     } else {
-      setState(allRenderersReadyOrEnded && haveSufficientBuffer()
-          && timeline.isReady(internalPositionUs) ? ExoPlayer.STATE_READY
-              : ExoPlayer.STATE_BUFFERING);
+      setState(allRenderersReadyOrEnded && haveSufficientBuffer() && timeline.isReady()
+          ? ExoPlayer.STATE_READY : ExoPlayer.STATE_BUFFERING);
     }
 
     // Start the renderers if ready, and schedule the first piece of work.
@@ -596,7 +595,7 @@ import java.util.concurrent.atomic.AtomicInteger;
     private Source bufferingSource;
 
     private long playingSourceEndPositionUs;
-    private long nextSourceOffsetUs;
+    private long bufferingSourceOffsetUs;
 
     public Timeline() {
       rendererWasEnabledFlags = new boolean[renderers.length];
@@ -620,6 +619,7 @@ import java.util.concurrent.atomic.AtomicInteger;
             Source newSource = new Source(sampleSource, index, renderers.length);
             if (bufferingSource != null) {
               bufferingSource.nextSource = newSource;
+              bufferingSourceOffsetUs += bufferingSource.sampleSource.getDurationUs();
             }
             bufferingSource = newSource;
           }
@@ -637,23 +637,14 @@ import java.util.concurrent.atomic.AtomicInteger;
             bufferingSource.selectTracks(result.first, result.second, startPositionUs);
             if (playingSource == null) {
               // This is the first prepared source, so start playing it.
-              sourceOffsetUs = 0;
-              setPlayingSource(bufferingSource);
+              setPlayingSource(bufferingSource, 0);
             }
           }
         }
 
         if (bufferingSource.hasEnabledTracks) {
-          long bufferingPositionUs;
-          if (bufferingSource == playingSource) {
-            bufferingPositionUs = internalPositionUs - sourceOffsetUs;
-          } else if (bufferingSource == readingSource) {
-            // TODO[playlists]: Make sure continueBuffering supports a negative downstream position.
-            bufferingPositionUs = internalPositionUs - nextSourceOffsetUs;
-          } else {
-            bufferingPositionUs = 0;
-          }
-          bufferingSource.sampleSource.continueBuffering(bufferingPositionUs);
+          long sourcePositionUs = internalPositionUs - bufferingSourceOffsetUs;
+          bufferingSource.sampleSource.continueBuffering(sourcePositionUs);
         }
       }
 
@@ -669,22 +660,7 @@ import java.util.concurrent.atomic.AtomicInteger;
         }
       }
       if (playingSourceEndPositionUs == C.UNSET_TIME_US) {
-        // Calculate the next source's start position in the timeline.
-        long playingSourceDurationUs = playingSource.sampleSource.getDurationUs();
-        if (playingSourceDurationUs == C.UNSET_TIME_US) {
-          // The duration of the current source is unknown, so use the maximum rendered timestamp
-          // plus a small extra offset to make sure that renderers don't read two buffers with the
-          // same timestamp.
-          playingSourceEndPositionUs = 0;
-          for (TrackRenderer renderer : enabledRenderers) {
-            playingSourceEndPositionUs =
-                Math.max(playingSourceEndPositionUs, renderer.getMaximumTimeUs());
-          }
-          nextSourceOffsetUs = playingSourceEndPositionUs + 10000;
-        } else {
-          playingSourceEndPositionUs = sourceOffsetUs + playingSourceDurationUs;
-          nextSourceOffsetUs = playingSourceEndPositionUs;
-        }
+        playingSourceEndPositionUs = sourceOffsetUs + playingSource.sampleSource.getDurationUs();
       }
       if (sourceCount != SampleSourceProvider.UNKNOWN_SOURCE_COUNT
           && readingSource.index == sourceCount - 1) {
@@ -694,9 +670,7 @@ import java.util.concurrent.atomic.AtomicInteger;
         }
         readingSource = null;
         playingSourceEndPositionUs = C.UNSET_TIME_US;
-        return;
-      }
-      if (playingSource.nextSource != null && playingSource.nextSource.prepared) {
+      } else if (playingSource.nextSource != null && playingSource.nextSource.prepared) {
         readingSource = playingSource.nextSource;
         // Suppress reading a reset so that the transition can be seamless.
         readingSource.sampleSource.readReset();
@@ -713,27 +687,26 @@ import java.util.concurrent.atomic.AtomicInteger;
             for (int j = 0; j < formats.length; j++) {
               formats[j] = groups.get(selection.group).getFormat(selection.getTrack(j));
             }
-            renderer.replaceTrackStream(formats, readingSource.trackStreams[i], nextSourceOffsetUs);
+            renderer.replaceTrackStream(formats, readingSource.trackStreams[i],
+                playingSourceEndPositionUs);
           }
         }
       }
     }
 
-    public boolean isReady(long positionUs) {
+    public boolean isReady() {
       return playingSourceEndPositionUs == C.UNSET_TIME_US
-          || positionUs < playingSourceEndPositionUs || playingSource.nextSource != null;
+          || internalPositionUs < playingSourceEndPositionUs || playingSource.nextSource != null;
     }
 
-    public SampleSource getSampleSource(long positionUs) throws ExoPlaybackException {
+    public SampleSource getSampleSource() throws ExoPlaybackException {
       if (playingSource == null) {
         return null;
       }
       if (readingSource != playingSource && playingSourceEndPositionUs != C.UNSET_TIME_US
-          && positionUs >= playingSourceEndPositionUs) {
-        // Renderers are playing the next source, so update the timeline.
+          && internalPositionUs >= playingSourceEndPositionUs) {
         playingSource.release();
-        sourceOffsetUs = nextSourceOffsetUs;
-        setPlayingSource(readingSource);
+        setPlayingSource(readingSource, playingSourceEndPositionUs);
       }
       return playingSource.sampleSource;
     }
@@ -753,8 +726,9 @@ import java.util.concurrent.atomic.AtomicInteger;
       if (newPlayingSource != null) {
         nextSourceIndex = sourceIndex + 1;
         newPlayingSource.nextSource = null;
-        setPlayingSource(newPlayingSource);
+        setPlayingSource(newPlayingSource, sourceOffsetUs);
         bufferingSource = playingSource;
+        bufferingSourceOffsetUs = sourceOffsetUs;
         if (playingSource.hasEnabledTracks) {
           sampleSource.seekToUs(sourcePositionUs);
         }
@@ -762,6 +736,7 @@ import java.util.concurrent.atomic.AtomicInteger;
         playingSource = null;
         readingSource = null;
         bufferingSource = null;
+        bufferingSourceOffsetUs = 0;
         durationUs = C.UNSET_TIME_US;
         sampleSource = null;
         // Set the next source index so that the required source is created in updateSources.
@@ -803,6 +778,7 @@ import java.util.concurrent.atomic.AtomicInteger;
       bufferingSource = playingSource;
       nextSourceIndex = playingSource.index + 1;
       playingSourceEndPositionUs = C.UNSET_TIME_US;
+      bufferingSourceOffsetUs = sourceOffsetUs;
 
       // Update the track selection for the playing source.
       Pair<TrackSelectionArray, Object> result =
@@ -842,10 +818,10 @@ import java.util.concurrent.atomic.AtomicInteger;
       readingSource = null;
       bufferingSource = null;
       durationUs = C.UNSET_TIME_US;
+      playingSourceEndPositionUs = C.UNSET_TIME_US;
       nextSourceIndex = 0;
       sourceOffsetUs = 0;
-      playingSourceEndPositionUs = C.UNSET_TIME_US;
-      nextSourceOffsetUs = 0;
+      bufferingSourceOffsetUs = 0;
     }
 
     @Override
@@ -864,8 +840,8 @@ import java.util.concurrent.atomic.AtomicInteger;
       return sb.toString();
     }
 
-    private void setPlayingSource(Source source) throws ExoPlaybackException {
-      playingSourceEndPositionUs = C.UNSET_TIME_US;
+    private void setPlayingSource(Source source, long offsetUs) throws ExoPlaybackException {
+      sourceOffsetUs = offsetUs;
       durationUs = source.sampleSource.getDurationUs();
 
       // Disable/enable renderers for the new source.
@@ -875,6 +851,7 @@ import java.util.concurrent.atomic.AtomicInteger;
       }
       readingSource = source;
       playingSource = source;
+      playingSourceEndPositionUs = C.UNSET_TIME_US;
       enableRenderers(source.trackSelections, enabledRendererCount);
 
       // Update the timeline position for the new source index.
diff --git a/library/src/main/java/com/google/android/exoplayer/SampleSource.java b/library/src/main/java/com/google/android/exoplayer/SampleSource.java
index a38e901ff1..704c576e13 100644
--- a/library/src/main/java/com/google/android/exoplayer/SampleSource.java
+++ b/library/src/main/java/com/google/android/exoplayer/SampleSource.java
@@ -37,7 +37,10 @@ public interface SampleSource {
   boolean prepare(long positionUs) throws IOException;
 
   /**
-   * Returns the duration of the source.
+   * Returns the duration of the source in microseconds, or {@link C#UNSET_TIME_US} if not known.
+   * <p>
+   * If {@link #getBufferedPositionUs()} returns {@link C#END_OF_SOURCE_US}, the duration is
+   * guaranteed to be known.
    * <p>
    * This method should only be called after the source has been prepared.
    *
diff --git a/library/src/main/java/com/google/android/exoplayer/TrackRenderer.java b/library/src/main/java/com/google/android/exoplayer/TrackRenderer.java
index 6b4289c24f..0956e9817f 100644
--- a/library/src/main/java/com/google/android/exoplayer/TrackRenderer.java
+++ b/library/src/main/java/com/google/android/exoplayer/TrackRenderer.java
@@ -109,7 +109,6 @@ public abstract class TrackRenderer implements ExoPlayerComponent {
   private int state;
   private TrackStream stream;
   private long streamOffsetUs;
-  private long maximumTimeUs;
   private boolean readEndOfStream;
   private boolean streamIsFinal;
 
@@ -235,7 +234,6 @@ public abstract class TrackRenderer implements ExoPlayerComponent {
    */
   /* package */ final void reset(long positionUs) throws ExoPlaybackException {
     streamIsFinal = false;
-    maximumTimeUs = C.UNSET_TIME_US;
     onReset(positionUs, false);
   }
 
@@ -260,14 +258,6 @@ public abstract class TrackRenderer implements ExoPlayerComponent {
     return readEndOfStream;
   }
 
-  /**
-   * Returns the maximum buffer timestamp read from the stream since the last reset, or
-   * {@link C#UNSET_TIME_US} if no buffers have been read.
-   */
-  /* package */ final long getMaximumTimeUs() {
-    return maximumTimeUs;
-  }
-
   /**
    * Signals to the renderer that the current {@link TrackStream} will be the final one supplied
    * before it is next disabled or reset.
@@ -372,9 +362,6 @@ public abstract class TrackRenderer implements ExoPlayerComponent {
         return streamIsFinal ? TrackStream.BUFFER_READ : TrackStream.NOTHING_READ;
       }
       buffer.timeUs += streamOffsetUs;
-      if (buffer.timeUs > maximumTimeUs) {
-        maximumTimeUs = buffer.timeUs;
-      }
     }
     return result;
   }
diff --git a/library/src/main/java/com/google/android/exoplayer/extractor/ExtractorSampleSource.java b/library/src/main/java/com/google/android/exoplayer/extractor/ExtractorSampleSource.java
index e89609dba9..a6cd21b728 100644
--- a/library/src/main/java/com/google/android/exoplayer/extractor/ExtractorSampleSource.java
+++ b/library/src/main/java/com/google/android/exoplayer/extractor/ExtractorSampleSource.java
@@ -111,6 +111,12 @@ public final class ExtractorSampleSource implements SampleSource, ExtractorOutpu
 
   private static final int MIN_RETRY_COUNT_DEFAULT_FOR_MEDIA = -1;
 
+  /**
+   * When the source's duration is unknown, it is calculated by adding this value to the largest
+   * sample timestamp seen when buffering completes.
+   */
+  private static final long DEFAULT_LAST_SAMPLE_DURATION_US = 10000;
+
   // Lazily initialized default extractor classes in priority order.
   private static List<Class<? extends Extractor>> defaultExtractorClasses;
 
@@ -397,11 +403,7 @@ public final class ExtractorSampleSource implements SampleSource, ExtractorOutpu
     } else if (isPendingReset()) {
       return pendingResetPositionUs;
     } else {
-      long largestQueuedTimestampUs = Long.MIN_VALUE;
-      for (DefaultTrackOutput sampleQueue : sampleQueues) {
-        largestQueuedTimestampUs = Math.max(largestQueuedTimestampUs,
-            sampleQueue.getLargestQueuedTimestampUs());
-      }
+      long largestQueuedTimestampUs = getLargestQueuedTimestampUs();
       return largestQueuedTimestampUs == Long.MIN_VALUE ? lastSeekPositionUs
           : largestQueuedTimestampUs;
     }
@@ -459,6 +461,11 @@ public final class ExtractorSampleSource implements SampleSource, ExtractorOutpu
       long loadDurationMs) {
     copyLengthFromLoader(loadable);
     loadingFinished = true;
+    if (durationUs == C.UNSET_TIME_US) {
+      long largestQueuedTimestampUs = getLargestQueuedTimestampUs();
+      durationUs = largestQueuedTimestampUs == Long.MIN_VALUE ? 0
+          : largestQueuedTimestampUs + DEFAULT_LAST_SAMPLE_DURATION_US;
+    }
   }
 
   @Override
@@ -579,6 +586,15 @@ public final class ExtractorSampleSource implements SampleSource, ExtractorOutpu
     return extractedSamplesCount;
   }
 
+  private long getLargestQueuedTimestampUs() {
+    long largestQueuedTimestampUs = Long.MIN_VALUE;
+    for (DefaultTrackOutput sampleQueue : sampleQueues) {
+      largestQueuedTimestampUs = Math.max(largestQueuedTimestampUs,
+          sampleQueue.getLargestQueuedTimestampUs());
+    }
+    return largestQueuedTimestampUs;
+  }
+
   private boolean haveFormatsForAllTracks() {
     for (DefaultTrackOutput sampleQueue : sampleQueues) {
       if (sampleQueue.getUpstreamFormat() == null) {