diff --git a/library/core/src/main/java/com/google/android/exoplayer2/analytics/PlaybackStatsListener.java b/library/core/src/main/java/com/google/android/exoplayer2/analytics/PlaybackStatsListener.java
index fd349ea60c..e061066788 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/analytics/PlaybackStatsListener.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/analytics/PlaybackStatsListener.java
@@ -769,10 +769,12 @@ public final class PlaybackStatsListener
           }
         }
       }
-      mediaTimeHistory.add(
-          mediaTimeMs == C.TIME_UNSET
-              ? guessMediaTimeBasedOnElapsedRealtime(realtimeMs)
-              : new long[] {realtimeMs, mediaTimeMs});
+
+      if (mediaTimeMs != C.TIME_UNSET) {
+        mediaTimeHistory.add(new long[] {realtimeMs, mediaTimeMs});
+      } else if (!mediaTimeHistory.isEmpty()) {
+        mediaTimeHistory.add(guessMediaTimeBasedOnElapsedRealtime(realtimeMs));
+      }
     }
 
     private long[] guessMediaTimeBasedOnElapsedRealtime(long realtimeMs) {