diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
index 2396bd42c7..3ccfa6c395 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
@@ -1723,11 +1723,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
 
     // We are not rendering on a surface, the renderer will wait until a surface is set.
     if (displaySurface == null) {
-      // Skip frames in sync with playback, so we'll be at the right frame if the mode changes.
-      if ((videoFrameReleaseInfo.getEarlyUs() < 0
-              && shouldSkipLateBuffersWhileUsingPlaceholderSurface())
-          || (videoFrameReleaseInfo.getEarlyUs() < 30_000
-              && frameReleaseAction != VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER)) {
+      // Skip frames in sync with playback, so we'll be at the right frame if a surface is set.
+      if (getState() == STATE_STARTED && videoFrameReleaseInfo.getEarlyUs() < 30_000) {
         skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
         updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs());
         return true;
@@ -1876,11 +1873,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
     return true;
   }
 
-  /** Returns whether to skip late buffers while using a placeholder surface. */
-  protected boolean shouldSkipLateBuffersWhileUsingPlaceholderSurface() {
-    return true;
-  }
-
   /**
    * Returns whether to force rendering an output buffer.
    *
diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java
index 7a2b56c240..aaf4d80be9 100644
--- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java
+++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParseAv1SampleDependenciesPlaybackTest.java
@@ -210,12 +210,6 @@ public class ParseAv1SampleDependenciesPlaybackTest {
         return false;
       }
 
-      @Override
-      protected boolean shouldSkipLateBuffersWhileUsingPlaceholderSurface() {
-        // Do not skip buffers while using placeholder surface due to slow processing.
-        return false;
-      }
-
       @Override
       protected boolean shouldForceRenderOutputBuffer(long earlyUs, long elapsedSinceLastRenderUs) {
         // An auto-advancing FakeClock can make a lot of progress before
diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
index f533573392..e4e0c49ada 100644
--- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
+++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java
@@ -265,12 +265,6 @@ public class CapturingRenderersFactory implements RenderersFactory, Dumper.Dumpa
       // Do not skip buffers with identical vsync times as we can't control this from tests.
       return false;
     }
-
-    @Override
-    protected boolean shouldSkipLateBuffersWhileUsingPlaceholderSurface() {
-      // Do not skip buffers while using placeholder surface due to slow processing.
-      return false;
-    }
   }
 
   /**