diff --git a/library/src/main/java/com/google/android/exoplayer/MediaCodecAudioTrackRenderer.java b/library/src/main/java/com/google/android/exoplayer/MediaCodecAudioTrackRenderer.java
index f1ab2b5006..530ef7ad86 100644
--- a/library/src/main/java/com/google/android/exoplayer/MediaCodecAudioTrackRenderer.java
+++ b/library/src/main/java/com/google/android/exoplayer/MediaCodecAudioTrackRenderer.java
@@ -426,7 +426,8 @@ public class MediaCodecAudioTrackRenderer extends MediaCodecTrackRenderer {
 
   @Override
   protected boolean isReady() {
-    return super.isReady() || getPendingFrameCount() > 0;
+    return getPendingFrameCount() > 0
+        || (super.isReady() && getSourceState() == SOURCE_STATE_READY_READ_MAY_FAIL);
   }
 
   /**
diff --git a/library/src/main/java/com/google/android/exoplayer/MediaCodecTrackRenderer.java b/library/src/main/java/com/google/android/exoplayer/MediaCodecTrackRenderer.java
index 6f7262f79a..3004032076 100644
--- a/library/src/main/java/com/google/android/exoplayer/MediaCodecTrackRenderer.java
+++ b/library/src/main/java/com/google/android/exoplayer/MediaCodecTrackRenderer.java
@@ -78,6 +78,22 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
 
   }
 
+  /**
+   * Value of {@link #sourceState} when the source is not ready.
+   */
+  protected static final int SOURCE_STATE_NOT_READY = 0;
+  /**
+   * Value of {@link #sourceState} when the source is ready and we're able to read from it.
+   */
+  protected static final int SOURCE_STATE_READY = 1;
+  /**
+   * Value of {@link #sourceState} when the source is ready but we might not be able to read from
+   * it. We transition to this state when an attempt to read a sample fails despite the source
+   * reporting that samples are available. This can occur when the next sample to be provided by
+   * the source is for another renderer.
+   */
+  protected static final int SOURCE_STATE_READY_READ_MAY_FAIL = 2;
+
   /**
    * If the {@link MediaCodec} is hotswapped (i.e. replaced during playback), this is the period of
    * time during which {@link #isReady()} will report true regardless of whether the new codec has
@@ -128,7 +144,7 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
   private int codecReconfigurationState;
 
   private int trackIndex;
-  private boolean sourceIsReady;
+  private int sourceState;
   private boolean inputStreamEnded;
   private boolean outputStreamEnded;
   private boolean waitingForKeys;
@@ -202,7 +218,7 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
   @Override
   protected void onEnabled(long timeUs, boolean joining) {
     source.enable(trackIndex, timeUs);
-    sourceIsReady = false;
+    sourceState = SOURCE_STATE_NOT_READY;
     inputStreamEnded = false;
     outputStreamEnded = false;
     waitingForKeys = false;
@@ -353,7 +369,7 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
   protected void seekTo(long timeUs) throws ExoPlaybackException {
     currentPositionUs = timeUs;
     source.seekToUs(timeUs);
-    sourceIsReady = false;
+    sourceState = SOURCE_STATE_NOT_READY;
     inputStreamEnded = false;
     outputStreamEnded = false;
     waitingForKeys = false;
@@ -372,7 +388,9 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
   @Override
   protected void doSomeWork(long timeUs) throws ExoPlaybackException {
     try {
-      sourceIsReady = source.continueBuffering(timeUs);
+      sourceState = source.continueBuffering(timeUs)
+          ? (sourceState == SOURCE_STATE_NOT_READY ? SOURCE_STATE_READY : sourceState)
+          : SOURCE_STATE_NOT_READY;
       checkForDiscontinuity();
       if (format == null) {
         readFormat();
@@ -384,7 +402,9 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
         }
         if (codec != null) {
           while (drainOutputBuffer(timeUs)) {}
-          while (feedInputBuffer()) {}
+          if (feedInputBuffer(true)) {
+            while (feedInputBuffer(false)) {}
+          }
         }
       }
       codecCounters.ensureUpdated();
@@ -446,11 +466,13 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
   }
 
   /**
+   * @param firstFeed True if this is the first call to this method from the current invocation of
+   *     {@link #doSomeWork(long)}. False otherwise.
    * @return True if it may be possible to feed more input data. False otherwise.
    * @throws IOException If an error occurs reading data from the upstream source.
    * @throws ExoPlaybackException If an error occurs feeding the input buffer.
    */
-  private boolean feedInputBuffer() throws IOException, ExoPlaybackException {
+  private boolean feedInputBuffer(boolean firstFeed) throws IOException, ExoPlaybackException {
     if (inputStreamEnded) {
       return false;
     }
@@ -478,6 +500,9 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
         codecReconfigurationState = RECONFIGURATION_STATE_QUEUE_PENDING;
       }
       result = source.readData(trackIndex, currentPositionUs, formatHolder, sampleHolder, false);
+      if (firstFeed && sourceState == SOURCE_STATE_READY && result == SampleSource.NOTHING_READ) {
+        sourceState = SOURCE_STATE_READY_READ_MAY_FAIL;
+      }
     }
 
     if (result == SampleSource.NOTHING_READ) {
@@ -646,7 +671,17 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
   @Override
   protected boolean isReady() {
     return format != null && !waitingForKeys
-        && (sourceIsReady || outputIndex >= 0 || isWithinHotswapPeriod());
+        && sourceState != SOURCE_STATE_NOT_READY || outputIndex >= 0 || isWithinHotswapPeriod();
+  }
+
+  /**
+   * Gets the source state.
+   *
+   * @return One of {@link #SOURCE_STATE_NOT_READY}, {@link #SOURCE_STATE_READY} and
+   *     {@link #SOURCE_STATE_READY_READ_MAY_FAIL}.
+   */
+  protected final int getSourceState() {
+    return sourceState;
   }
 
   private boolean isWithinHotswapPeriod() {
diff --git a/library/src/main/java/com/google/android/exoplayer/MediaCodecVideoTrackRenderer.java b/library/src/main/java/com/google/android/exoplayer/MediaCodecVideoTrackRenderer.java
index b941767955..0fd1f4fd47 100644
--- a/library/src/main/java/com/google/android/exoplayer/MediaCodecVideoTrackRenderer.java
+++ b/library/src/main/java/com/google/android/exoplayer/MediaCodecVideoTrackRenderer.java
@@ -235,7 +235,8 @@ public class MediaCodecVideoTrackRenderer extends MediaCodecTrackRenderer {
 
   @Override
   protected boolean isReady() {
-    if (super.isReady()) {
+    if (super.isReady() && (renderedFirstFrame || !codecInitialized()
+        || getSourceState() == SOURCE_STATE_READY_READ_MAY_FAIL)) {
       // Ready. If we were joining then we've now joined, so clear the joining deadline.
       joiningDeadlineUs = -1;
       return true;