diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/SilenceMediaSource.java b/library/core/src/main/java/com/google/android/exoplayer2/source/SilenceMediaSource.java
index b802717ee2..447a3e3ad5 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/source/SilenceMediaSource.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/source/SilenceMediaSource.java
@@ -305,11 +305,15 @@ public final class SilenceMediaSource extends BaseMediaSource {
         return C.RESULT_BUFFER_READ;
       }
 
+      buffer.timeUs = getAudioPositionUs(positionBytes);
+      buffer.addFlag(C.BUFFER_FLAG_KEY_FRAME);
+      if (buffer.isFlagsOnly()) {
+        return C.RESULT_BUFFER_READ;
+      }
+
       int bytesToWrite = (int) min(SILENCE_SAMPLE.length, bytesRemaining);
       buffer.ensureSpaceForWrite(bytesToWrite);
       buffer.data.put(SILENCE_SAMPLE, /* offset= */ 0, bytesToWrite);
-      buffer.timeUs = getAudioPositionUs(positionBytes);
-      buffer.addFlag(C.BUFFER_FLAG_KEY_FRAME);
       positionBytes += bytesToWrite;
       return C.RESULT_BUFFER_READ;
     }