From 6650270a4e1bd294ba95baea0342e3fa0e669c4e Mon Sep 17 00:00:00 2001
From: dancho <dancho@google.com>
Date: Tue, 9 Jul 2024 03:40:23 -0700
Subject: [PATCH] MediaCodecVideoRenderer skips decoder inputs unused as
 reference

During a seek, or when playing a media with clipped start,
MCVR encounters preroll decode-only buffers that are not rendered.
Use C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS to determine
whether a decode-only buffer is unused as reference.
These buffers can be dropped before the decoder.
When this optimization is triggered, increment
decoderCounters.skippedInputBufferCount.

Tested in ExoPlayer demo app on "One hour frame counter (MP4)"
after enabling extractorsFactory.setMp4ExtractorFlags(
    FLAG_READ_WITHIN_GOP_SAMPLE_DEPENDENCIES);
Observe: "sib" increases on each seek.
PiperOrigin-RevId: 650566216
---
 RELEASENOTES.md                               |  2 +
 .../main/java/androidx/media3/common/C.java   |  9 ++-
 .../java/androidx/media3/decoder/Buffer.java  |  5 ++
 .../mediacodec/MediaCodecRenderer.java        | 18 +++++
 .../video/MediaCodecVideoRenderer.java        | 23 ++++++-
 .../video/MediaCodecVideoRendererTest.java    | 67 +++++++++++++++++++
 .../extractor/mp4/FragmentedMp4Extractor.java |  4 +-
 .../media3/extractor/mp4/Mp4Extractor.java    |  4 +-
 8 files changed, 122 insertions(+), 10 deletions(-)

diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index 3a22248fff..584067a1a4 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -19,6 +19,8 @@
     *   Add `PreloadMediaSource.PreloadControl.onPreloadError` to allow
         `PreloadMediaSource.PreloadControl` implementations to take actions when
         error occurs.
+    *   `MediaCodecVideoRenderer` avoids decoding samples that are neither
+        rendered nor used as reference by other samples.
 *   Transformer:
     *   Add `SurfaceAssetLoader`, which supports queueing video data to
         Transformer via a `Surface`.
diff --git a/libraries/common/src/main/java/androidx/media3/common/C.java b/libraries/common/src/main/java/androidx/media3/common/C.java
index cf3eb9abaa..dfe8381711 100644
--- a/libraries/common/src/main/java/androidx/media3/common/C.java
+++ b/libraries/common/src/main/java/androidx/media3/common/C.java
@@ -620,7 +620,7 @@ public final class C {
    * <ul>
    *   <li>{@link #BUFFER_FLAG_KEY_FRAME}
    *   <li>{@link #BUFFER_FLAG_END_OF_STREAM}
-   *   <li>{@link #BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}
+   *   <li>{@link #BUFFER_FLAG_NOT_DEPENDED_ON}
    *   <li>{@link #BUFFER_FLAG_FIRST_SAMPLE}
    *   <li>{@link #BUFFER_FLAG_LAST_SAMPLE}
    *   <li>{@link #BUFFER_FLAG_ENCRYPTED}
@@ -635,7 +635,7 @@ public final class C {
       value = {
         BUFFER_FLAG_KEY_FRAME,
         BUFFER_FLAG_END_OF_STREAM,
-        BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS,
+        BUFFER_FLAG_NOT_DEPENDED_ON,
         BUFFER_FLAG_FIRST_SAMPLE,
         BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA,
         BUFFER_FLAG_LAST_SAMPLE,
@@ -650,9 +650,8 @@ public final class C {
   @UnstableApi
   public static final int BUFFER_FLAG_END_OF_STREAM = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
 
-  /** Indicates that future buffers do not depend on the data in this buffer. */
-  @UnstableApi
-  public static final int BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS = 1 << 26; // 0x04000000
+  /** Indicates that no other buffers depend on the data in this buffer. */
+  @UnstableApi public static final int BUFFER_FLAG_NOT_DEPENDED_ON = 1 << 26; // 0x04000000
 
   /** Indicates that a buffer is known to contain the first media sample of the stream. */
   @UnstableApi public static final int BUFFER_FLAG_FIRST_SAMPLE = 1 << 27; // 0x08000000
diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java b/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java
index ef61c48d95..b992c5d38f 100644
--- a/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java
+++ b/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java
@@ -60,6 +60,11 @@ public abstract class Buffer {
     return getFlag(C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA);
   }
 
+  /** Returns whether the {@link C#BUFFER_FLAG_NOT_DEPENDED_ON} flag is set. */
+  public final boolean notDependedOn() {
+    return getFlag(C.BUFFER_FLAG_NOT_DEPENDED_ON);
+  }
+
   /**
    * Replaces this buffer's flags with {@code flags}.
    *
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
index 0d87f1c793..c53a02f0c3 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
@@ -1452,6 +1452,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
       return true;
     }
 
+    if (shouldSkipDecoderInputBuffer(buffer)) {
+      buffer.clear();
+      decoderCounters.skippedInputBufferCount += 1;
+      return true;
+    }
+
     boolean bufferEncrypted = buffer.isEncrypted();
     if (bufferEncrypted) {
       buffer.cryptoInfo.increaseClearDataFirstSubSampleBy(adaptiveReconfigurationBytes);
@@ -1754,6 +1760,18 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     return 0;
   }
 
+  /**
+   * Returns whether the input buffer should be skipped before the decoder.
+   *
+   * <p>This can be used to skip decoding of buffers that are not depended on during seeking. See
+   * {@link C#BUFFER_FLAG_NOT_DEPENDED_ON}.
+   *
+   * @param buffer The input buffer.
+   */
+  protected boolean shouldSkipDecoderInputBuffer(DecoderInputBuffer buffer) {
+    return false;
+  }
+
   /**
    * Returns the presentation time of the last buffer in the stream.
    *
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
index 5f995af7ac..e4a0cd27fd 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
@@ -1247,7 +1247,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
 
   @Override
   protected int getCodecBufferFlags(DecoderInputBuffer buffer) {
-    if (Util.SDK_INT >= 34 && tunneling && buffer.timeUs < getLastResetPositionUs()) {
+    if (Util.SDK_INT >= 34 && tunneling && isBufferBeforeStartTime(buffer)) {
       // The buffer likely needs to be dropped because its timestamp is less than the start time.
       // We can't decide to do this after decoding because we won't get the buffer back from the
       // codec in tunneling mode. This may not work perfectly, e.g. when the codec is doing frame
@@ -1257,6 +1257,27 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
     return 0;
   }
 
+  @Override
+  protected boolean shouldSkipDecoderInputBuffer(DecoderInputBuffer buffer) {
+    // TODO: b/351164714 - Do not apply this optimization for buffers with timestamp near
+    //  the media duration.
+    if (hasReadStreamToEnd() || buffer.isLastSample()) {
+      // Last buffer is always decoded.
+      return false;
+    }
+    if (buffer.isEncrypted()) {
+      // Commonly used decryption algorithms require updating the initialization vector for each
+      // block processed. Skipping input buffers before the decoder is not allowed.
+      return false;
+    }
+    // Skip buffers without sample dependencies that won't be rendered.
+    return isBufferBeforeStartTime(buffer) && buffer.notDependedOn();
+  }
+
+  private boolean isBufferBeforeStartTime(DecoderInputBuffer buffer) {
+    return buffer.timeUs < getLastResetPositionUs();
+  }
+
   @Override
   protected void onOutputFormatChanged(Format format, @Nullable MediaFormat mediaFormat) {
     @Nullable MediaCodecAdapter codec = getCodec();
diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
index 02fec74843..12d8fee2ab 100644
--- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
+++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
@@ -356,6 +356,73 @@ public class MediaCodecVideoRendererTest {
     assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2);
   }
 
+  @Test
+  public void render_withoutSampleDependencies_rendersLastFrameAfterEndOfStream() throws Exception {
+    ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
+        ArgumentCaptor.forClass(DecoderCounters.class);
+    FakeSampleStream fakeSampleStream =
+        new FakeSampleStream(
+            new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+            /* mediaSourceEventDispatcher= */ null,
+            DrmSessionManager.DRM_UNSUPPORTED,
+            new DrmSessionEventListener.EventDispatcher(),
+            /* initialFormat= */ VIDEO_H264,
+            ImmutableList.of(
+                oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
+                // Second buffer will be skipped before decoder during a seek.
+                oneByteSample(/* timeUs= */ 10_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+                // Last buffer without sample dependencies will be rendered.
+                oneByteSample(/* timeUs= */ 20_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+                END_OF_STREAM_ITEM));
+    fakeSampleStream.writeData(/* startPositionUs= */ 0);
+    // Seek to time after samples.
+    fakeSampleStream.seekToUs(30_000, /* allowTimeBeyondBuffer= */ true);
+    mediaCodecVideoRenderer =
+        new MediaCodecVideoRenderer(
+            ApplicationProvider.getApplicationContext(),
+            new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3),
+            mediaCodecSelector,
+            /* allowedJoiningTimeMs= */ 0,
+            /* enableDecoderFallback= */ false,
+            /* eventHandler= */ new Handler(testMainLooper),
+            eventListener,
+            /* maxDroppedFramesToNotify= */ 1);
+    mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
+    mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
+    mediaCodecVideoRenderer.enable(
+        RendererConfiguration.DEFAULT,
+        new Format[] {VIDEO_H264},
+        fakeSampleStream,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ true,
+        /* startPositionUs= */ 30_000,
+        /* offsetUs= */ 0,
+        new MediaSource.MediaPeriodId(new Object()));
+
+    mediaCodecVideoRenderer.start();
+    mediaCodecVideoRenderer.setCurrentStreamFinal();
+    mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
+    // Call to render has reads all samples including the END_OF_STREAM_ITEM because the
+    // previous sample is skipped before decoding.
+    assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue();
+    int posUs = 30_000;
+    while (!mediaCodecVideoRenderer.isEnded()) {
+      mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+      posUs += 40_000;
+    }
+    shadowOf(testMainLooper).idle();
+
+    verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
+    verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
+    // First key frame is decoded and skipped as an output buffer.
+    assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(1);
+    // Second frame is skipped before the decoder, as an input buffer.
+    assertThat(argumentDecoderCounters.getValue().skippedInputBufferCount).isEqualTo(1);
+    // Last frame is rendered.
+    assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1);
+  }
+
   @Test
   public void
       render_withClippingMediaPeriodAndBufferContainingLastAndClippingSamples_rendersLastFrame()
diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java
index 7b19c4bafc..c47f731a02 100644
--- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java
+++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java
@@ -135,7 +135,7 @@ public class FragmentedMp4Extractor implements Extractor {
 
   /**
    * Flag to extract additional sample dependency information, and mark output buffers with {@link
-   * C#BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}.
+   * C#BUFFER_FLAG_NOT_DEPENDED_ON}.
    *
    * <p>This class always marks the samples at the start of each group of picture (GOP) with {@link
    * C#BUFFER_FLAG_KEY_FRAME}. Usually, key frames can be decoded independently, without depending
@@ -1661,7 +1661,7 @@ public class FragmentedMp4Extractor implements Extractor {
 
     @C.BufferFlags int sampleFlags = trackBundle.getCurrentSampleFlags();
     if ((flags & FLAG_READ_WITHIN_GOP_SAMPLE_DEPENDENCIES) != 0 && !isSampleDependedOn) {
-      sampleFlags |= C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS;
+      sampleFlags |= C.BUFFER_FLAG_NOT_DEPENDED_ON;
     }
 
     // Encryption data.
diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java
index 5850bba24e..23e42bafc1 100644
--- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java
+++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java
@@ -125,7 +125,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
 
   /**
    * Flag to extract additional sample dependency information, and mark output buffers with {@link
-   * C#BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}.
+   * C#BUFFER_FLAG_NOT_DEPENDED_ON}.
    *
    * <p>This class always marks the samples at the start of each group of picture (GOP) with {@link
    * C#BUFFER_FLAG_KEY_FRAME}. Usually, key frames can be decoded independently, without depending
@@ -804,7 +804,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
     long timeUs = track.sampleTable.timestampsUs[sampleIndex];
     @C.BufferFlags int sampleFlags = track.sampleTable.flags[sampleIndex];
     if (!isSampleDependedOn) {
-      sampleFlags |= C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS;
+      sampleFlags |= C.BUFFER_FLAG_NOT_DEPENDED_ON;
     }
     if (trueHdSampleRechunker != null) {
       trueHdSampleRechunker.sampleMetadata(