From 8ec18086f2290b16e3a0e9b030f23ec83a62886f Mon Sep 17 00:00:00 2001
From: kimvde <kimvde@google.com>
Date: Thu, 27 Jan 2022 15:59:22 +0000
Subject: [PATCH] Fix transformer stuck forever

When the decoder output buffer was partially read, a call to
Codec.getOutputBuffer() was returning the same buffer, but with the
position reset to 0. The reason was that, in
Codec.maybeDequeueAndSetOutputBuffer(), mediaCodec.getOutputBuffer()
was called with the same buffer index (L350 in old rev), even though
there was already a buffer available (outputBufferIndex >=0). This
change avoids calling mediaCodec.getOutputBuffer() if the previous
buffer has not been released.

#minor-release

PiperOrigin-RevId: 424612197
---
 RELEASENOTES.md                               |  1 +
 .../android/exoplayer2/transformer/Codec.java | 46 ++++++++-----------
 2 files changed, 20 insertions(+), 27 deletions(-)

diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index fbb025afab..0e70059a0f 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -111,6 +111,7 @@
         during a transformation.
     *   Add `TransformationRequest` for specifying the transformation options.
     *   Allow multiple listeners to be registered.
+    *   Fix Transformer being stuck when the codec output is partially read.
 *   MediaSession extension:
     *   Remove deprecated call to `onStop(/* reset= */ true)` and provide an
         opt-out flag for apps that don't want to clear the playlist on stop.
diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java
index 59a93e7019..834b54a404 100644
--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java
@@ -261,7 +261,7 @@ public final class Codec {
   @Nullable
   public Format getOutputFormat() throws TransformationException {
     // The format is updated when dequeueing a 'special' buffer index, so attempt to dequeue now.
-    maybeDequeueOutputBuffer();
+    maybeDequeueOutputBuffer(/* setOutputBuffer= */ false);
     return outputFormat;
   }
 
@@ -272,7 +272,7 @@ public final class Codec {
    */
   @Nullable
   public ByteBuffer getOutputBuffer() throws TransformationException {
-    return maybeDequeueAndSetOutputBuffer() ? outputBuffer : null;
+    return maybeDequeueOutputBuffer(/* setOutputBuffer= */ true) ? outputBuffer : null;
   }
 
   /**
@@ -282,7 +282,7 @@ public final class Codec {
    */
   @Nullable
   public BufferInfo getOutputBufferInfo() throws TransformationException {
-    return maybeDequeueOutputBuffer() ? outputBufferInfo : null;
+    return maybeDequeueOutputBuffer(/* setOutputBuffer= */ false) ? outputBufferInfo : null;
   }
 
   /**
@@ -334,33 +334,15 @@ public final class Codec {
   }
 
   /**
-   * Tries obtaining an output buffer and sets {@link #outputBuffer} to the obtained output buffer.
+   * Attempts to dequeue an output buffer if there is no output buffer pending. Does nothing
+   * otherwise.
    *
-   * @return {@code true} if a buffer is successfully obtained, {@code false} otherwise.
+   * @param setOutputBuffer Whether to read the bytes of the dequeued output buffer and copy them
+   *     into {@link #outputBuffer}.
+   * @return Whether there is an output buffer available.
    * @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
    */
-  private boolean maybeDequeueAndSetOutputBuffer() throws TransformationException {
-    if (!maybeDequeueOutputBuffer()) {
-      return false;
-    }
-
-    try {
-      outputBuffer = checkNotNull(mediaCodec.getOutputBuffer(outputBufferIndex));
-    } catch (RuntimeException e) {
-      throw createTransformationException(e);
-    }
-    outputBuffer.position(outputBufferInfo.offset);
-    outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
-    return true;
-  }
-
-  /**
-   * Returns true if there is already an output buffer pending. Otherwise attempts to dequeue an
-   * output buffer and returns whether there is a new output buffer.
-   *
-   * @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
-   */
-  private boolean maybeDequeueOutputBuffer() throws TransformationException {
+  private boolean maybeDequeueOutputBuffer(boolean setOutputBuffer) throws TransformationException {
     if (outputBufferIndex >= 0) {
       return true;
     }
@@ -391,6 +373,16 @@ public final class Codec {
       releaseOutputBuffer();
       return false;
     }
+
+    if (setOutputBuffer) {
+      try {
+        outputBuffer = checkNotNull(mediaCodec.getOutputBuffer(outputBufferIndex));
+      } catch (RuntimeException e) {
+        throw createTransformationException(e);
+      }
+      outputBuffer.position(outputBufferInfo.offset);
+      outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
+    }
     return true;
   }