Make ByteBufferConcurrentEffect asynchronous

* Changes to GlUtil to manage Pixel Buffer Objects and asynchronous
GPU -> CPU glReadPixels
* Make ByteBufferConcurrentEffect non-blocking

PiperOrigin-RevId: 667908805

libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java

@@ -862,6 +862,133 @@ public final class GlUtil {
     checkGlError();
   }
 
+  /**
+   * Creates a pixel buffer object with a data store of the given size and usage {@link
+   * GLES30#GL_DYNAMIC_READ}.
+   *
+   * <p>The buffer is suitable for repeated modification by OpenGL and reads by the application.
+   *
+   * @param size The size of the buffer object's data store.
+   * @return The pixel buffer object.
+   */
+  public static int createPixelBufferObject(int size) throws GlException {
+    int[] ids = new int[1];
+    GLES30.glGenBuffers(/* n= */ 1, ids, /* offset= */ 0);
+    GlUtil.checkGlError();
+
+    GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, ids[0]);
+    GlUtil.checkGlError();
+
+    GLES30.glBufferData(
+        GLES30.GL_PIXEL_PACK_BUFFER, /* size= */ size, /* data= */ null, GLES30.GL_DYNAMIC_READ);
+    GlUtil.checkGlError();
+
+    GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, /* buffer= */ 0);
+    GlUtil.checkGlError();
+    return ids[0];
+  }
+
+  /**
+   * Reads pixel data from the {@link GLES30#GL_COLOR_ATTACHMENT0} attachment of a framebuffer into
+   * the data store of a pixel buffer object.
+   *
+   * <p>The texture backing the color attachment of {@code readFboId} and the buffer store of {@code
+   * bufferId} must hold an image of the given {@code width} and {@code height} with format {@link
+   * GLES30#GL_RGBA} and type {@link GLES30#GL_UNSIGNED_BYTE}.
+   *
+   * <p>This a non-blocking call which reads the data asynchronously.
+   *
+   * <p>HDR support is not yet implemented.
+   *
+   * @param readFboId The framebuffer that holds pixel data.
+   * @param width The image width.
+   * @param height The image height.
+   * @param bufferId The pixel buffer object to read into.
+   */
+  public static void schedulePixelBufferRead(int readFboId, int width, int height, int bufferId)
+      throws GlException {
+    focusFramebufferUsingCurrentContext(readFboId, width, height);
+    GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, bufferId);
+    GlUtil.checkGlError();
+
+    GLES30.glReadBuffer(GLES30.GL_COLOR_ATTACHMENT0);
+    GLES30.glReadPixels(
+        /* x= */ 0,
+        /* y= */ 0,
+        width,
+        height,
+        GLES30.GL_RGBA,
+        GLES30.GL_UNSIGNED_BYTE,
+        /* offset= */ 0);
+    GlUtil.checkGlError();
+
+    GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, /* buffer= */ 0);
+    GlUtil.checkGlError();
+  }
+
+  /**
+   * Maps the pixel buffer object's data store of a given size and returns a {@link ByteBuffer} of
+   * OpenGL managed memory.
+   *
+   * <p>The application must not write into the returned {@link ByteBuffer}.
+   *
+   * <p>The pixel buffer object should have a {@linkplain #schedulePixelBufferRead previously
+   * scheduled pixel buffer read}.
+   *
+   * <p>When the application no longer needs to access the returned buffer, call {@link
+   * #unmapPixelBufferObject}.
+   *
+   * <p>This call blocks until the pixel buffer data from the last {@link #schedulePixelBufferRead}
+   * call is available.
+   *
+   * @param bufferId The pixel buffer object.
+   * @param size The size of the pixel buffer object's data store to be mapped.
+   * @return The {@link ByteBuffer} that holds pixel data.
+   */
+  public static ByteBuffer mapPixelBufferObject(int bufferId, int size) throws GlException {
+    GLES20.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, bufferId);
+    checkGlError();
+    ByteBuffer mappedPixelBuffer =
+        (ByteBuffer)
+            GLES30.glMapBufferRange(
+                GLES30.GL_PIXEL_PACK_BUFFER,
+                /* offset= */ 0,
+                /* length= */ size,
+                GLES30.GL_MAP_READ_BIT);
+    GlUtil.checkGlError();
+    GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, /* buffer= */ 0);
+    GlUtil.checkGlError();
+    return mappedPixelBuffer;
+  }
+
+  /**
+   * Unmaps the pixel buffer object {@code bufferId}'s data store.
+   *
+   * <p>The pixel buffer object should be previously {@linkplain #mapPixelBufferObject mapped}.
+   *
+   * <p>After this method returns, accessing data inside a previously {@linkplain
+   * #mapPixelBufferObject mapped} {@link ByteBuffer} results in undefined behaviour.
+   *
+   * <p>When this method returns, the pixel buffer object {@code bufferId} can be reused by {@link
+   * #schedulePixelBufferRead}.
+   *
+   * @param bufferId The pixel buffer object.
+   */
+  public static void unmapPixelBufferObject(int bufferId) throws GlException {
+    GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, bufferId);
+    GlUtil.checkGlError();
+    GLES30.glUnmapBuffer(GLES30.GL_PIXEL_PACK_BUFFER);
+    GlUtil.checkGlError();
+    GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, /* buffer= */ 0);
+    GlUtil.checkGlError();
+  }
+
+  /** Deletes a buffer object, or silently ignores the method call if {@code bufferId} is unused. */
+  public static void deleteBuffer(int bufferId) throws GlException {
+    GLES20.glDeleteBuffers(/* n= */ 1, new int[] {bufferId}, /* offset= */ 0);
+    checkGlError();
+  }
+
   /**
    * Throws a {@link GlException} with the given message if {@code expression} evaluates to {@code
    * false}.

libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java

@@ -177,5 +177,14 @@ public class QueuingGlShaderProgramTest {
       checkState(result == presentationTimeUs);
       events.add(Pair.create("finishProcessingAndBlend", presentationTimeUs));
     }
+
+    @Override
+    public void signalEndOfCurrentInputStream() {}
+
+    @Override
+    public void flush() {}
+
+    @Override
+    public void release() {}
   }
 }

libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java

@@ -15,18 +15,22 @@
  */
 package androidx.media3.effect;
 
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
 import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
 
 import android.graphics.Rect;
-import android.opengl.GLES20;
-import android.opengl.GLES30;
 import androidx.media3.common.C;
 import androidx.media3.common.GlObjectsProvider;
 import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.Size;
+import androidx.media3.common.util.Util;
+import com.google.common.util.concurrent.SettableFuture;
 import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Queue;
 import java.util.concurrent.Future;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
@@ -44,6 +48,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private static final int BYTES_PER_PIXEL = 4;
 
   private final ByteBufferGlEffect.Processor<T> processor;
+  private final int pendingPixelBufferQueueSize;
+  private final Queue<TexturePixelBuffer> unmappedPixelBuffers;
+  private final Queue<TexturePixelBuffer> mappedPixelBuffers;
+  private final PixelBufferObjectProvider pixelBufferObjectProvider;
 
   private int inputWidth;
   private int inputHeight;
@@ -52,10 +60,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   /**
    * Creates an instance.
    *
+   * @param pendingPixelBufferQueueSize The maximum number of scheduled but not yet completed
+   *     texture to {@linkplain ByteBuffer pixel buffer} transfers.
    * @param processor The {@linkplain ByteBufferGlEffect.Processor effect}.
    */
-  public ByteBufferConcurrentEffect(ByteBufferGlEffect.Processor<T> processor) {
+  public ByteBufferConcurrentEffect(
+      int pendingPixelBufferQueueSize, ByteBufferGlEffect.Processor<T> processor) {
     this.processor = processor;
+    this.pendingPixelBufferQueueSize = pendingPixelBufferQueueSize;
+    unmappedPixelBuffers = new ArrayDeque<>();
+    mappedPixelBuffers = new ArrayDeque<>();
+    pixelBufferObjectProvider = new PixelBufferObjectProvider();
     inputWidth = C.LENGTH_UNSET;
     inputHeight = C.LENGTH_UNSET;
   }
@@ -64,9 +79,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   public Future<T> queueInputFrame(
       GlObjectsProvider glObjectsProvider, GlTextureInfo textureInfo, long presentationTimeUs) {
     try {
+      while (unmappedPixelBuffers.size() >= pendingPixelBufferQueueSize) {
+        checkState(mapOnePixelBuffer());
+      }
+
       if (effectInputTexture == null
           || textureInfo.width != inputWidth
           || textureInfo.height != inputHeight) {
+        while (mapOnePixelBuffer()) {}
         inputWidth = textureInfo.width;
         inputHeight = textureInfo.height;
         Size effectInputSize = processor.configure(inputWidth, inputHeight);
@@ -90,20 +110,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
           new Rect(
               /* left= */ 0, /* top= */ 0, effectInputTexture.width, effectInputTexture.height));
 
-      GlUtil.focusFramebufferUsingCurrentContext(
-          effectInputTexture.fboId, effectInputTexture.width, effectInputTexture.height);
-      ByteBuffer pixelBuffer =
-          ByteBuffer.allocateDirect(texturePixelBufferSize(effectInputTexture));
-      GLES20.glReadPixels(
-          /* x= */ 0,
-          /* y= */ 0,
-          effectInputTexture.width,
-          effectInputTexture.height,
-          GLES30.GL_RGBA,
-          GLES30.GL_UNSIGNED_BYTE,
-          pixelBuffer);
-      GlUtil.checkGlError();
-      return processor.processPixelBuffer(pixelBuffer, presentationTimeUs);
+      TexturePixelBuffer texturePixelBuffer = new TexturePixelBuffer(effectInputTexture);
+      unmappedPixelBuffers.add(texturePixelBuffer);
+      return Util.transformFutureAsync(
+          texturePixelBuffer.byteBufferSettableFuture,
+          (pixelBuffer) -> processor.processPixelBuffer(pixelBuffer, presentationTimeUs));
     } catch (GlUtil.GlException | VideoFrameProcessingException e) {
       return immediateFailedFuture(e);
     }
@@ -112,10 +123,145 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   @Override
   public void finishProcessingAndBlend(GlTextureInfo textureInfo, long presentationTimeUs, T result)
       throws VideoFrameProcessingException {
+    try {
+      TexturePixelBuffer oldestRunningFrame = checkNotNull(mappedPixelBuffers.poll());
+      oldestRunningFrame.unmapAndRecycle();
+    } catch (GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e);
+    }
+
     processor.finishProcessingAndBlend(textureInfo, presentationTimeUs, result);
   }
 
+  @Override
+  public void signalEndOfCurrentInputStream() throws VideoFrameProcessingException {
+    try {
+      while (mapOnePixelBuffer()) {}
+    } catch (GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e);
+    }
+  }
+
+  @Override
+  public void flush() throws VideoFrameProcessingException {
+    try {
+      unmapAndRecyclePixelBuffers();
+    } catch (GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e);
+    }
+  }
+
+  @Override
+  public void release() throws VideoFrameProcessingException {
+    try {
+      unmapAndRecyclePixelBuffers();
+      pixelBufferObjectProvider.release();
+    } catch (GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e);
+    }
+  }
+
   private static int texturePixelBufferSize(GlTextureInfo textureInfo) {
     return textureInfo.width * textureInfo.height * BYTES_PER_PIXEL;
   }
+
+  private void unmapAndRecyclePixelBuffers() throws GlUtil.GlException {
+    TexturePixelBuffer texturePixelBuffer;
+    while ((texturePixelBuffer = unmappedPixelBuffers.poll()) != null) {
+      texturePixelBuffer.unmapAndRecycle();
+    }
+    while ((texturePixelBuffer = mappedPixelBuffers.poll()) != null) {
+      texturePixelBuffer.unmapAndRecycle();
+    }
+  }
+
+  private boolean mapOnePixelBuffer() throws GlUtil.GlException {
+    TexturePixelBuffer texturePixelBuffer = unmappedPixelBuffers.poll();
+    if (texturePixelBuffer == null) {
+      return false;
+    }
+    texturePixelBuffer.map();
+    mappedPixelBuffers.add(texturePixelBuffer);
+    return true;
+  }
+
+  /**
+   * Manages the lifecycle of a {@link PixelBufferObjectInfo} which is mapped to a {@link
+   * GlTextureInfo}.
+   */
+  private final class TexturePixelBuffer {
+    public final PixelBufferObjectInfo pixelBufferObjectInfo;
+    public final SettableFuture<ByteBuffer> byteBufferSettableFuture;
+
+    private boolean mapped;
+
+    public TexturePixelBuffer(GlTextureInfo textureInfo) throws GlUtil.GlException {
+      int pixelBufferSize = texturePixelBufferSize(textureInfo);
+      pixelBufferObjectInfo = pixelBufferObjectProvider.getPixelBufferObject(pixelBufferSize);
+      GlUtil.schedulePixelBufferRead(
+          textureInfo.fboId, textureInfo.width, textureInfo.height, pixelBufferObjectInfo.id);
+      byteBufferSettableFuture = SettableFuture.create();
+    }
+
+    public void map() throws GlUtil.GlException {
+      ByteBuffer byteBuffer =
+          GlUtil.mapPixelBufferObject(pixelBufferObjectInfo.id, pixelBufferObjectInfo.size);
+      byteBufferSettableFuture.set(byteBuffer);
+      mapped = true;
+    }
+
+    public void unmapAndRecycle() throws GlUtil.GlException {
+      if (mapped) {
+        GlUtil.unmapPixelBufferObject(pixelBufferObjectInfo.id);
+      }
+      pixelBufferObjectProvider.recycle(pixelBufferObjectInfo);
+    }
+  }
+
+  /** One pixel buffer object with a data store. */
+  private static final class PixelBufferObjectInfo {
+    public final int id;
+    public final int size;
+
+    public PixelBufferObjectInfo(int size) throws GlUtil.GlException {
+      this.size = size;
+      id = GlUtil.createPixelBufferObject(size);
+    }
+
+    public void release() throws GlUtil.GlException {
+      GlUtil.deleteBuffer(id);
+    }
+  }
+
+  /** Provider for {@link PixelBufferObjectInfo} objects. */
+  private static final class PixelBufferObjectProvider {
+    private final Queue<PixelBufferObjectInfo> availablePixelBufferObjects;
+
+    public PixelBufferObjectProvider() {
+      availablePixelBufferObjects = new ArrayDeque<>();
+    }
+
+    private PixelBufferObjectInfo getPixelBufferObject(int pixelBufferSize)
+        throws GlUtil.GlException {
+      PixelBufferObjectInfo pixelBufferObjectInfo;
+      while ((pixelBufferObjectInfo = availablePixelBufferObjects.poll()) != null) {
+        if (pixelBufferObjectInfo.size == pixelBufferSize) {
+          return pixelBufferObjectInfo;
+        }
+        GlUtil.deleteBuffer(pixelBufferObjectInfo.id);
+      }
+      return new PixelBufferObjectInfo(pixelBufferSize);
+    }
+
+    private void recycle(PixelBufferObjectInfo pixelBufferObjectInfo) {
+      availablePixelBufferObjects.add(pixelBufferObjectInfo);
+    }
+
+    public void release() throws GlUtil.GlException {
+      PixelBufferObjectInfo pixelBufferObjectInfo;
+      while ((pixelBufferObjectInfo = availablePixelBufferObjects.poll()) != null) {
+        pixelBufferObjectInfo.release();
+      }
+    }
+  }
 }

libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java

@@ -39,6 +39,7 @@ import java.util.concurrent.Future;
 /* package */ class ByteBufferGlEffect<T> implements GlEffect {
 
   private static final int DEFAULT_QUEUE_SIZE = 6;
+  private static final int DEFAULT_PENDING_PIXEL_BUFFER_QUEUE_SIZE = 1;
 
   /**
    * A processor that takes in {@link ByteBuffer ByteBuffers} that represent input image data, and
@@ -134,6 +135,7 @@ import java.util.concurrent.Future;
     return new QueuingGlShaderProgram<>(
         /* useHighPrecisionColorComponents= */ useHdr,
         /* queueSize= */ DEFAULT_QUEUE_SIZE,
-        new ByteBufferConcurrentEffect<>(processor));
+        new ByteBufferConcurrentEffect<>(
+            /* pendingPixelBufferQueueSize= */ DEFAULT_PENDING_PIXEL_BUFFER_QUEUE_SIZE, processor));
   }
 }

libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java

@@ -110,6 +110,32 @@ import java.util.concurrent.TimeUnit;
      */
     void finishProcessingAndBlend(GlTextureInfo outputFrame, long presentationTimeUs, T result)
         throws VideoFrameProcessingException;
+
+    /**
+     * Notifies the {@code ConcurrentEffect} that no further input frames belonging to the current
+     * input stream will be queued.
+     *
+     * <p>Can block until the {@code ConcurrentEffect} finishes processing pending frames.
+     *
+     * @throws VideoFrameProcessingException If an error occurs while processing pending frames.
+     */
+    void signalEndOfCurrentInputStream() throws VideoFrameProcessingException;
+
+    /**
+     * Flushes the {@code ConcurrentEffect}.
+     *
+     * <p>The {@code ConcurrentEffect} should reclaim the ownership of any allocated resources.
+     *
+     * @throws VideoFrameProcessingException If an error occurs while reclaiming resources.
+     */
+    void flush() throws VideoFrameProcessingException;
+
+    /**
+     * Releases all resources.
+     *
+     * @throws VideoFrameProcessingException If an error occurs while releasing resources.
+     */
+    void release() throws VideoFrameProcessingException;
   }
 
   private final ConcurrentEffect<T> concurrentEffect;
@@ -222,6 +248,11 @@ import java.util.concurrent.TimeUnit;
 
   @Override
   public void signalEndOfCurrentInputStream() {
+    try {
+      concurrentEffect.signalEndOfCurrentInputStream();
+    } catch (VideoFrameProcessingException e) {
+      onError(e);
+    }
     while (outputOneFrame()) {}
     outputListener.onCurrentOutputStreamEnded();
   }
@@ -229,6 +260,11 @@ import java.util.concurrent.TimeUnit;
   @Override
   @CallSuper
   public void flush() {
+    try {
+      concurrentEffect.flush();
+    } catch (VideoFrameProcessingException e) {
+      onError(e);
+    }
     cancelProcessingOfPendingFrames();
     outputTexturePool.freeAllTextures();
     inputListener.onFlush();
@@ -242,6 +278,7 @@ import java.util.concurrent.TimeUnit;
   public void release() throws VideoFrameProcessingException {
     try {
       cancelProcessingOfPendingFrames();
+      concurrentEffect.release();
       outputTexturePool.deleteAllTextures();
     } catch (GlUtil.GlException e) {
       throw new VideoFrameProcessingException(e);