Improve throughput on devices requiring workaround

On some devices, decoding gets stuck when the number of frames pending at the
`SurfaceTexture` is too high. We added a workaround that only allows one frame
to be pending at a time. That fixed the issue, however, based on on-device
testing it seems that it's safe to queue more than one frame.

Add a method that returns a safe estimate of the number of frames that can be
pending at a time, and use this to limit the number of frames that can be
released from the decoder but not processed by the frame processor chain.

PiperOrigin-RevId: 437057075

libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java

@@ -55,7 +55,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
  * <p>Input becomes available on its {@link #getInputSurface() input surface} asynchronously and is
  * processed on a background thread as it becomes available. All input frames should be {@link
  * #registerInputFrame() registered} before they are rendered to the input surface. {@link
- * #hasPendingFrames()} can be used to check whether there are frames that have not been fully
+ * #getPendingFrameCount()} can be used to check whether there are frames that have not been fully
  * processed yet. Output is written to its {@link #configure(Surface, int, int, SurfaceView) output
  * surface}.
  */
@@ -298,16 +298,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   }
 
   /**
-   * Returns whether there are input frames that have been {@link #registerInputFrame() registered}
+   * Returns the number of input frames that have been {@link #registerInputFrame() registered} but
-   * but not completely processed yet.
+   * not completely processed yet.
    */
-  public boolean hasPendingFrames() {
+  public int getPendingFrameCount() {
-    return pendingFrameCount.get() > 0;
+    return pendingFrameCount.get();
   }
 
   /** Returns whether all frames have been processed. */
   public boolean isEnded() {
-    return inputStreamEnded && !hasPendingFrames();
+    return inputStreamEnded && getPendingFrameCount() == 0;
   }
 
   /** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */

libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java

@@ -40,8 +40,8 @@ import androidx.media3.decoder.DecoderInputBuffer;
   void queueInputBuffer() throws TransformationException;
 
   /**
-   * Processes the input data and returns whether more data can be processed by calling this method
+   * Processes the input data and returns whether it may be possible to process more data by calling
-   * again.
+   * this method again.
    */
   boolean processData() throws TransformationException;
 

libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java

@@ -17,14 +17,11 @@
 package androidx.media3.transformer;
 
 import static androidx.media3.common.util.Assertions.checkNotNull;
-import static androidx.media3.common.util.Util.SDK_INT;
 
 import android.content.Context;
 import android.media.MediaCodec;
-import android.media.MediaFormat;
 import android.util.Size;
 import androidx.annotation.Nullable;
-import androidx.annotation.RequiresApi;
 import androidx.media3.common.Format;
 import androidx.media3.common.util.Util;
 import androidx.media3.decoder.DecoderInputBuffer;
@@ -37,9 +34,12 @@ import org.checkerframework.dataflow.qual.Pure;
  */
 /* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline {
 
+  private static final int FRAME_COUNT_UNLIMITED = -1;
+
   private final int outputRotationDegrees;
   private final DecoderInputBuffer decoderInputBuffer;
   private final Codec decoder;
+  private final int maxPendingFrameCount;
 
   private final FrameProcessorChain frameProcessorChain;
 
@@ -121,6 +121,7 @@ import org.checkerframework.dataflow.qual.Pure;
 
     decoder =
         decoderFactory.createForVideoDecoding(inputFormat, frameProcessorChain.getInputSurface());
+    maxPendingFrameCount = getMaxPendingFrameCount();
   }
 
   @Override
@@ -148,51 +149,15 @@ import org.checkerframework.dataflow.qual.Pure;
       return false;
     }
 
-    boolean canProcessMoreDataImmediately = false;
+    boolean processedData = false;
-    if (SDK_INT >= 29
+    while (maybeProcessDecoderOutput()) {
-        && !(("samsung".equals(Util.MANUFACTURER) || "OnePlus".equals(Util.MANUFACTURER))
+      processedData = true;
-            && SDK_INT < 31)) {
-      // TODO(b/213455700): Fix Samsung and OnePlus devices filling the decoder in processDataV29().
-      processDataV29();
-    } else {
-      canProcessMoreDataImmediately = processDataDefault();
     }
     if (decoder.isEnded()) {
       frameProcessorChain.signalEndOfInputStream();
     }
-    return canProcessMoreDataImmediately;
+    // If the decoder produced output, signal that it may be possible to process data again.
-  }
+    return processedData;
-
-  /**
-   * Processes input data from API 29.
-   *
-   * <p>In this method the decoder could decode multiple frames in one invocation; as compared to
-   * {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently,
-   * if {@link FrameProcessorChain} processes frames slower than the decoder, decoded frames are
-   * queued up in the decoder's output surface.
-   *
-   * <p>Prior to API 29, decoders may drop frames to keep their output surface from growing out of
-   * bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame
-   * dropping even when the surface is full. As dropping random frames is not acceptable in {@code
-   * Transformer}, using this method requires API level 29 or higher.
-   */
-  @RequiresApi(29)
-  private void processDataV29() throws TransformationException {
-    while (maybeProcessDecoderOutput()) {}
-  }
-
-  /**
-   * Processes at most one input frame and returns whether a frame was processed.
-   *
-   * <p>Only renders decoder output to the {@link FrameProcessorChain}'s input surface if the {@link
-   * FrameProcessorChain} has finished processing the previous frame.
-   */
-  private boolean processDataDefault() throws TransformationException {
-    // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
-    if (frameProcessorChain.hasPendingFrames()) {
-      return false;
-    }
-    return maybeProcessDecoderOutput();
   }
 
   @Override
@@ -275,8 +240,40 @@ import org.checkerframework.dataflow.qual.Pure;
       return false;
     }
 
+    if (maxPendingFrameCount != FRAME_COUNT_UNLIMITED
+        && frameProcessorChain.getPendingFrameCount() == maxPendingFrameCount) {
+      return false;
+    }
+
     frameProcessorChain.registerInputFrame();
     decoder.releaseOutputBuffer(/* render= */ true);
     return true;
   }
+
+  /**
+   * Returns the maximum number of frames that may be pending in the output {@link
+   * FrameProcessorChain} at a time, or {@link #FRAME_COUNT_UNLIMITED} if it's not necessary to
+   * enforce a limit.
+   */
+  private static int getMaxPendingFrameCount() {
+    if (Util.SDK_INT < 29) {
+      // Prior to API 29, decoders may drop frames to keep their output surface from growing out of
+      // bounds, while from API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame
+      // dropping even when the surface is full. We never want frame dropping so allow a maximum of
+      // one frame to be pending at a time.
+      // TODO(b/226330223): Investigate increasing this limit.
+      return 1;
+    }
+    if (Util.SDK_INT < 31
+        && ("OnePlus".equals(Util.MANUFACTURER) || "samsung".equals(Util.MANUFACTURER))) {
+      // Some OMX decoders don't correctly track their number of output buffers available, and get
+      // stuck if too many frames are rendered without being processed, so we limit the number of
+      // pending frames to avoid getting stuck. This value is experimentally determined. See also
+      // b/213455700.
+      return 10;
+    }
+    // Otherwise don't limit the number of frames that can be pending at a time, to maximize
+    // throughput.
+    return FRAME_COUNT_UNLIMITED;
+  }
 }