Add buffer flag for last sample to improve buffered position calculation.

The buffered position is currently based on the mimimum queued timestamp of
all AV tracks. If the tracks have unequal lengths, one track continues loading
without bounds as the "buffered position" will always stay at the shorter
track's duration.

This change adds an optional buffer flag to mark the last sample of the
stream. This is set in the Mp4Extractor only so far. ExtractorMediaSource
uses this flag to ignore AV streams in the buffered duration calculation if
they already finished loading.

Issue:#3670
PiperOrigin-RevId: 229359899

RELEASENOTES.md

@@ -48,6 +48,8 @@
   ([#5351](https://github.com/google/ExoPlayer/issues/5351)).
 * Downloading/Caching: Improve cache performance
   ([#4253](https://github.com/google/ExoPlayer/issues/4253)).
+* Fix issue where uneven track durations in MP4 streams can cause OOM problems
+  ([#3670](https://github.com/google/ExoPlayer/issues/3670)).
 
 ### 2.9.3 ###
 

library/core/src/main/java/com/google/android/exoplayer2/C.java

@@ -460,8 +460,8 @@ public final class C {
 
   /**
    * Flags which can apply to a buffer containing a media sample. Possible flag values are {@link
-   * #BUFFER_FLAG_KEY_FRAME}, {@link #BUFFER_FLAG_END_OF_STREAM}, {@link #BUFFER_FLAG_ENCRYPTED} and
+   * #BUFFER_FLAG_KEY_FRAME}, {@link #BUFFER_FLAG_END_OF_STREAM}, {@link #BUFFER_FLAG_LAST_SAMPLE},
-   * {@link #BUFFER_FLAG_DECODE_ONLY}.
+   * {@link #BUFFER_FLAG_ENCRYPTED} and {@link #BUFFER_FLAG_DECODE_ONLY}.
    */
   @Documented
   @Retention(RetentionPolicy.SOURCE)
@@ -470,6 +470,7 @@ public final class C {
       value = {
         BUFFER_FLAG_KEY_FRAME,
         BUFFER_FLAG_END_OF_STREAM,
+        BUFFER_FLAG_LAST_SAMPLE,
         BUFFER_FLAG_ENCRYPTED,
         BUFFER_FLAG_DECODE_ONLY
       })
@@ -482,6 +483,8 @@ public final class C {
    * Flag for empty buffers that signal that the end of the stream was reached.
    */
   public static final int BUFFER_FLAG_END_OF_STREAM = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
+  /** Indicates that a buffer is known to contain the last media sample of the stream. */
+  public static final int BUFFER_FLAG_LAST_SAMPLE = 1 << 29; // 0x20000000
   /** Indicates that a buffer is (at least partially) encrypted. */
   public static final int BUFFER_FLAG_ENCRYPTED = 1 << 30; // 0x40000000
   /** Indicates that a buffer should be decoded but not rendered. */

library/core/src/main/java/com/google/android/exoplayer2/extractor/mp4/TrackSampleTable.java

@@ -64,6 +64,9 @@ import com.google.android.exoplayer2.util.Util;
     this.flags = flags;
     this.durationUs = durationUs;
     sampleCount = offsets.length;
+    if (flags.length > 0) {
+      flags[flags.length - 1] |= C.BUFFER_FLAG_LAST_SAMPLE;
+    }
   }
 
   /**

library/core/src/main/java/com/google/android/exoplayer2/source/ExtractorMediaPeriod.java

@@ -356,18 +356,19 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
     } else if (isPendingReset()) {
       return pendingResetPositionUs;
     }
-    long largestQueuedTimestampUs;
+    long largestQueuedTimestampUs = C.TIME_UNSET;
     if (haveAudioVideoTracks) {
       // Ignore non-AV tracks, which may be sparse or poorly interleaved.
       largestQueuedTimestampUs = Long.MAX_VALUE;
       int trackCount = sampleQueues.length;
       for (int i = 0; i < trackCount; i++) {
-        if (trackIsAudioVideoFlags[i]) {
+        if (trackIsAudioVideoFlags[i] && !sampleQueues[i].isLastSampleQueued()) {
           largestQueuedTimestampUs = Math.min(largestQueuedTimestampUs,
               sampleQueues[i].getLargestQueuedTimestampUs());
         }
       }
-    } else {
+    }
+    if (largestQueuedTimestampUs == C.TIME_UNSET) {
       largestQueuedTimestampUs = getLargestQueuedTimestampUs();
     }
     return largestQueuedTimestampUs == Long.MIN_VALUE ? lastSeekPositionUs

library/core/src/main/java/com/google/android/exoplayer2/source/SampleMetadataQueue.java

@@ -57,6 +57,7 @@ import com.google.android.exoplayer2.util.Util;
 
   private long largestDiscardedTimestampUs;
   private long largestQueuedTimestampUs;
+  private boolean isLastSampleQueued;
   private boolean upstreamKeyframeRequired;
   private boolean upstreamFormatRequired;
   private Format upstreamFormat;
@@ -93,6 +94,7 @@ import com.google.android.exoplayer2.util.Util;
     upstreamKeyframeRequired = true;
     largestDiscardedTimestampUs = Long.MIN_VALUE;
     largestQueuedTimestampUs = Long.MIN_VALUE;
+    isLastSampleQueued = false;
     if (resetUpstreamFormat) {
       upstreamFormat = null;
       upstreamFormatRequired = true;
@@ -118,6 +120,7 @@ import com.google.android.exoplayer2.util.Util;
     Assertions.checkArgument(0 <= discardCount && discardCount <= (length - readPosition));
     length -= discardCount;
     largestQueuedTimestampUs = Math.max(largestDiscardedTimestampUs, getLargestTimestamp(length));
+    isLastSampleQueued = discardCount == 0 && isLastSampleQueued;
     if (length == 0) {
       return 0;
     } else {
@@ -186,6 +189,19 @@ import com.google.android.exoplayer2.util.Util;
     return largestQueuedTimestampUs;
   }
 
+  /**
+   * Returns whether the last sample of the stream has knowingly been queued. A return value of
+   * {@code false} means that the last sample had not been queued or that it's unknown whether the
+   * last sample has been queued.
+   *
+   * <p>Samples that were discarded by calling {@link #discardUpstreamSamples(int)} are not
+   * considered as having been queued. Samples that were dequeued from the front of the queue are
+   * considered as having been queued.
+   */
+  public synchronized boolean isLastSampleQueued() {
+    return isLastSampleQueued;
+  }
+
   /** Returns the timestamp of the first sample, or {@link Long#MIN_VALUE} if the queue is empty. */
   public synchronized long getFirstTimestampUs() {
     return length == 0 ? Long.MIN_VALUE : timesUs[relativeFirstIndex];
@@ -224,7 +240,7 @@ import com.google.android.exoplayer2.util.Util;
       boolean formatRequired, boolean loadingFinished, Format downstreamFormat,
       SampleExtrasHolder extrasHolder) {
     if (!hasNextSample()) {
-      if (loadingFinished) {
+      if (loadingFinished || isLastSampleQueued) {
         buffer.setFlags(C.BUFFER_FLAG_END_OF_STREAM);
         return C.RESULT_BUFFER_READ;
       } else if (upstreamFormat != null
@@ -388,7 +404,9 @@ import com.google.android.exoplayer2.util.Util;
       upstreamKeyframeRequired = false;
     }
     Assertions.checkState(!upstreamFormatRequired);
-    commitSampleTimestamp(timeUs);
+
+    isLastSampleQueued = (sampleFlags & C.BUFFER_FLAG_LAST_SAMPLE) != 0;
+    largestQueuedTimestampUs = Math.max(largestQueuedTimestampUs, timeUs);
 
     int relativeEndIndex = getRelativeIndex(length);
     timesUs[relativeEndIndex] = timeUs;
@@ -439,10 +457,6 @@ import com.google.android.exoplayer2.util.Util;
     }
   }
 
-  public synchronized void commitSampleTimestamp(long timeUs) {
-    largestQueuedTimestampUs = Math.max(largestQueuedTimestampUs, timeUs);
-  }
-
   /**
    * Attempts to discard samples from the end of the queue to allow samples starting from the
    * specified timestamp to be spliced in. Samples will not be discarded prior to the read position.

library/core/src/main/java/com/google/android/exoplayer2/source/SampleQueue.java

@@ -224,6 +224,15 @@ public class SampleQueue implements TrackOutput {
     return metadataQueue.getLargestQueuedTimestampUs();
   }
 
+  /**
+   * Returns whether the last sample of the stream has knowingly been queued. A return value of
+   * {@code false} means that the last sample had not been queued or that it's unknown whether the
+   * last sample has been queued.
+   */
+  public boolean isLastSampleQueued() {
+    return metadataQueue.isLastSampleQueued();
+  }
+
   /** Returns the timestamp of the first sample, or {@link Long#MIN_VALUE} if the queue is empty. */
   public long getFirstTimestampUs() {
     return metadataQueue.getFirstTimestampUs();

library/core/src/test/assets/mp4/sample.mp4.0.dump

@@ -147,7 +147,7 @@ track 0:
     data = length 530, hash C98BC6A8
   sample 29:
     time = 934266
-    flags = 0
+    flags = 536870912
     data = length 568, hash 4FE5C8EA
 track 1:
   format:
@@ -352,6 +352,6 @@ track 1:
     data = length 229, hash FFF98DF0
   sample 44:
     time = 1065678
-    flags = 1
+    flags = 536870913
     data = length 6, hash 31B22286
 tracksEnded = true

library/core/src/test/assets/mp4/sample.mp4.1.dump

@@ -147,7 +147,7 @@ track 0:
     data = length 530, hash C98BC6A8
   sample 29:
     time = 934266
-    flags = 0
+    flags = 536870912
     data = length 568, hash 4FE5C8EA
 track 1:
   format:
@@ -304,6 +304,6 @@ track 1:
     data = length 229, hash FFF98DF0
   sample 32:
     time = 1065678
-    flags = 1
+    flags = 536870913
     data = length 6, hash 31B22286
 tracksEnded = true

library/core/src/test/assets/mp4/sample.mp4.2.dump

@@ -147,7 +147,7 @@ track 0:
     data = length 530, hash C98BC6A8
   sample 29:
     time = 934266
-    flags = 0
+    flags = 536870912
     data = length 568, hash 4FE5C8EA
 track 1:
   format:
@@ -244,6 +244,6 @@ track 1:
     data = length 229, hash FFF98DF0
   sample 17:
     time = 1065678
-    flags = 1
+    flags = 536870913
     data = length 6, hash 31B22286
 tracksEnded = true

library/core/src/test/assets/mp4/sample.mp4.3.dump

@@ -147,7 +147,7 @@ track 0:
     data = length 530, hash C98BC6A8
   sample 29:
     time = 934266
-    flags = 0
+    flags = 536870912
     data = length 568, hash 4FE5C8EA
 track 1:
   format:
@@ -184,6 +184,6 @@ track 1:
     data = length 229, hash FFF98DF0
   sample 2:
     time = 1065678
-    flags = 1
+    flags = 536870913
     data = length 6, hash 31B22286
 tracksEnded = true