Add QueuingGlShaderProgram for effects that run outside GL context

Implement a QueuingGlShaderProgram which queues up OpenGL frames and allows
asynchronous execution of effects that operate on video frames without a
performance penalty.

PiperOrigin-RevId: 666326611

libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java

@@ -23,6 +23,7 @@ import static androidx.media3.common.util.Assertions.checkState;
 import android.content.Context;
 import android.content.pm.PackageManager;
 import android.graphics.Bitmap;
+import android.graphics.Rect;
 import android.opengl.EGL14;
 import android.opengl.EGLConfig;
 import android.opengl.EGLContext;
@@ -824,6 +825,43 @@ public final class GlUtil {
     checkGlError();
   }
 
+  /**
+   * Copies the pixels from {@code readFboId} into {@code drawFboId}. Requires OpenGL ES 3.0.
+   *
+   * <p>When the input pixel region (given by {@code readRect}) doesn't have the same size as the
+   * output region (given by {@code drawRect}), this method uses {@link GLES20#GL_LINEAR} filtering
+   * to scale the image contents.
+   *
+   * @param readFboId The framebuffer object to read from.
+   * @param readRect The rectangular region of {@code readFboId} to read from.
+   * @param drawFboId The framebuffer object to draw into.
+   * @param drawRect The rectangular region of {@code drawFboId} to draw into.
+   */
+  public static void blitFrameBuffer(int readFboId, Rect readRect, int drawFboId, Rect drawRect)
+      throws GlException {
+    int[] boundFramebuffer = new int[1];
+    GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, boundFramebuffer, /* offset= */ 0);
+    checkGlError();
+    GLES30.glBindFramebuffer(GLES30.GL_READ_FRAMEBUFFER, readFboId);
+    checkGlError();
+    GLES30.glBindFramebuffer(GLES30.GL_DRAW_FRAMEBUFFER, drawFboId);
+    checkGlError();
+    GLES30.glBlitFramebuffer(
+        readRect.left,
+        readRect.top,
+        readRect.right,
+        readRect.bottom,
+        drawRect.left,
+        drawRect.top,
+        drawRect.right,
+        drawRect.bottom,
+        GLES30.GL_COLOR_BUFFER_BIT,
+        GLES30.GL_LINEAR);
+    checkGlError();
+    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, /* framebuffer= */ boundFramebuffer[0]);
+    checkGlError();
+  }
+
   /**
    * Throws a {@link GlException} with the given message if {@code expression} evaluates to {@code
    * false}.

libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames;
+import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.util.concurrent.Futures.immediateFuture;
+
+import android.content.Context;
+import android.graphics.Color;
+import android.text.Spannable;
+import android.text.SpannableString;
+import android.text.style.AbsoluteSizeSpan;
+import android.text.style.ForegroundColorSpan;
+import android.text.style.TypefaceSpan;
+import android.util.Pair;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.util.Consumer;
+import androidx.media3.test.utils.TextureBitmapReader;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+/** Tests for {@link QueuingGlShaderProgram}. */
+@RunWith(AndroidJUnit4.class)
+public class QueuingGlShaderProgramTest {
+  @Rule public final TestName testName = new TestName();
+
+  private static final String ASSET_PATH = "test-generated-goldens/QueuingGlShaderProgramTest";
+
+  private static final int BLANK_FRAME_WIDTH = 100;
+  private static final int BLANK_FRAME_HEIGHT = 50;
+  private static final Consumer<SpannableString> TEXT_SPAN_CONSUMER =
+      (text) -> {
+        text.setSpan(
+            new ForegroundColorSpan(Color.BLACK),
+            /* start= */ 0,
+            text.length(),
+            Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+        text.setSpan(
+            new AbsoluteSizeSpan(/* size= */ 24),
+            /* start= */ 0,
+            text.length(),
+            Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+        text.setSpan(
+            new TypefaceSpan(/* family= */ "sans-serif"),
+            /* start= */ 0,
+            text.length(),
+            Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+      };
+
+  private @MonotonicNonNull TextureBitmapReader textureBitmapReader;
+  private String testId;
+
+  @Before
+  public void setUp() {
+    textureBitmapReader = new TextureBitmapReader();
+    testId = testName.getMethodName();
+  }
+
+  @Test
+  public void queuingGlShaderProgram_withQueueSizeOne_outputsFramesInOrder() throws Exception {
+    List<Pair<String, Long>> events = new ArrayList<>();
+    ImmutableList<Long> frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L);
+    ImmutableList<Long> actualPresentationTimesUs =
+        generateAndProcessFrames(
+            BLANK_FRAME_WIDTH,
+            BLANK_FRAME_HEIGHT,
+            frameTimesUs,
+            new TestGlEffect(events, /* queueSize= */ 1),
+            textureBitmapReader,
+            TEXT_SPAN_CONSUMER);
+
+    assertThat(actualPresentationTimesUs).containsExactlyElementsIn(frameTimesUs).inOrder();
+    assertThat(events)
+        .containsExactly(
+            Pair.create("queueInputFrame", 0L),
+            Pair.create("finishProcessingAndBlend", 0L),
+            Pair.create("queueInputFrame", 333_333L),
+            Pair.create("finishProcessingAndBlend", 333_333L),
+            Pair.create("queueInputFrame", 666_667L),
+            Pair.create("finishProcessingAndBlend", 666_667L))
+        .inOrder();
+
+    getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH);
+  }
+
+  @Test
+  public void queuingGlShaderProgram_withQueueSizeTwo_outputsFramesInOrder() throws Exception {
+    List<Pair<String, Long>> events = new ArrayList<>();
+    ImmutableList<Long> frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L);
+    ImmutableList<Long> actualPresentationTimesUs =
+        generateAndProcessFrames(
+            BLANK_FRAME_WIDTH,
+            BLANK_FRAME_HEIGHT,
+            frameTimesUs,
+            new TestGlEffect(events, /* queueSize= */ 2),
+            textureBitmapReader,
+            TEXT_SPAN_CONSUMER);
+
+    assertThat(actualPresentationTimesUs).containsExactlyElementsIn(frameTimesUs).inOrder();
+    assertThat(events)
+        .containsExactly(
+            Pair.create("queueInputFrame", 0L),
+            Pair.create("queueInputFrame", 333_333L),
+            Pair.create("finishProcessingAndBlend", 0L),
+            Pair.create("queueInputFrame", 666_667L),
+            Pair.create("finishProcessingAndBlend", 333_333L),
+            Pair.create("finishProcessingAndBlend", 666_667L))
+        .inOrder();
+    getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH);
+  }
+
+  private static class TestGlEffect implements GlEffect {
+
+    private final List<Pair<String, Long>> events;
+    private final int queueSize;
+
+    TestGlEffect(List<Pair<String, Long>> events, int queueSize) {
+      this.events = events;
+      this.queueSize = queueSize;
+    }
+
+    @Override
+    public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
+      return new QueuingGlShaderProgram<>(
+          /* useHighPrecisionColorComponents= */ useHdr,
+          queueSize,
+          new NoOpConcurrentEffect(events));
+    }
+  }
+
+  private static class NoOpConcurrentEffect
+      implements QueuingGlShaderProgram.ConcurrentEffect<Long> {
+    private final List<Pair<String, Long>> events;
+
+    NoOpConcurrentEffect(List<Pair<String, Long>> events) {
+      this.events = events;
+    }
+
+    @Override
+    public Future<Long> queueInputFrame(GlTextureInfo textureInfo, long presentationTimeUs) {
+      checkState(textureInfo.width == BLANK_FRAME_WIDTH);
+      checkState(textureInfo.height == BLANK_FRAME_HEIGHT);
+      events.add(Pair.create("queueInputFrame", presentationTimeUs));
+      return immediateFuture(presentationTimeUs);
+    }
+
+    @Override
+    public void finishProcessingAndBlend(
+        GlTextureInfo outputFrame, long presentationTimeUs, Long result) {
+      checkState(result == presentationTimeUs);
+      events.add(Pair.create("finishProcessingAndBlend", presentationTimeUs));
+    }
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java

@@ -0,0 +1,302 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+import static androidx.media3.common.util.Assertions.checkState;
+
+import android.graphics.Rect;
+import androidx.annotation.CallSuper;
+import androidx.annotation.IntRange;
+import androidx.media3.common.C;
+import androidx.media3.common.GlObjectsProvider;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.UnstableApi;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.MoreExecutors;
+import java.util.ArrayDeque;
+import java.util.Queue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An implementation of {@link GlShaderProgram} that enables {@linkplain ConcurrentEffect
+ * asynchronous} processing of video frames outside the current OpenGL context without processor
+ * stalls.
+ *
+ * <h3>Data Dependencies and Processor Stalls</h3>
+ *
+ * Sharing image data between GPU and a {@link ConcurrentEffect} running on another processor
+ * creates a data dependency. The GPU must finish processing the frame before the data can be
+ * {@linkplain ConcurrentEffect#queueInputFrame submitted} to the other processor. And the other
+ * processor must finish processing the image data before any modifications can be {@linkplain
+ * ConcurrentEffect#finishProcessingAndBlend drawn} back to the main video stream.
+ *
+ * <p>If we force a synchronization and data transfer (e.g. via {@link
+ * android.opengl.GLES20#glReadPixels}) too early a processor would stall without any work
+ * available.
+ *
+ * <p>To keep multiple processors busy, {@code QueuingGlShaderProgram} maintains a queue of frames
+ * that are being processed by the provided {@link ConcurrentEffect}. The queue pipelines the
+ * processing stages and allows one frame to be processed on the GPU, while at the same time another
+ * frame is processed by the {@link ConcurrentEffect}. The size of the queue is configurable on
+ * construction, and should be large enough to compensate for the time required to execute the
+ * {@linkplain ConcurrentEffect asynchronous effect}, and any data transfer that is required between
+ * the processors.
+ *
+ * <p>The output frame {@link GlTextureInfo} produced by this class contains a copy of the
+ * {@linkplain #queueInputFrame input frame}, unless the frame contents were modified by the {@link
+ * ConcurrentEffect}.
+ *
+ * <p>All methods in this class must be called on the thread that owns the OpenGL context.
+ *
+ * @param <T> An intermediate type used by {@link ConcurrentEffect} implementations.
+ */
+@UnstableApi
+/* package */ final class QueuingGlShaderProgram<T> implements GlShaderProgram {
+
+  private static final long PROCESSING_TIMEOUT_MS = 500_000L;
+
+  /** A concurrent effect that is applied by the {@link QueuingGlShaderProgram}. */
+  public interface ConcurrentEffect<T> {
+    /**
+     * Submits a frame to be processed by the concurrent effect.
+     *
+     * <p>The {@linkplain GlTextureInfo textureInfo} will hold the image data corresponding to the
+     * frame at {@code presentationTimeUs}. The image data will not be modified until the returned
+     * {@link Future} {@linkplain Future#isDone() completes} or {@linkplain Future#isCancelled() is
+     * cancelled}.
+     *
+     * <p>The {@linkplain GlTextureInfo textureInfo} will have a valid {@linkplain
+     * GlTextureInfo#fboId framebuffer object}.
+     *
+     * <p>This method will be called on the thread that owns the OpenGL context.
+     *
+     * @param textureInfo The texture info of the current frame.
+     * @param presentationTimeUs The presentation timestamp of the input frame, in microseconds.
+     * @return A {@link Future} representing pending completion of the task.
+     */
+    Future<T> queueInputFrame(GlTextureInfo textureInfo, long presentationTimeUs);
+
+    /**
+     * Finishes processing the frame at {@code presentationTimeUs}. This method optionally allows
+     * the instance to draw an overlay or blend with the {@linkplain GlTextureInfo output frame}.
+     *
+     * <p>The {@linkplain GlTextureInfo outputFrame} contains the image data corresponding to the
+     * frame at {@code presentationTimeUs} when this method is invoked.
+     *
+     * <p>This method will be called on the thread that owns the OpenGL context.
+     *
+     * @param outputFrame The texture info of the frame.
+     * @param presentationTimeUs The presentation timestamp of the frame, in microseconds.
+     * @param result The result of the asynchronous computation in {@link #queueInputFrame}.
+     */
+    void finishProcessingAndBlend(GlTextureInfo outputFrame, long presentationTimeUs, T result)
+        throws VideoFrameProcessingException;
+  }
+
+  private final ConcurrentEffect<T> concurrentEffect;
+  private final TexturePool outputTexturePool;
+  private final Queue<TimedTextureInfo<T>> frameQueue;
+  private InputListener inputListener;
+  private OutputListener outputListener;
+  private ErrorListener errorListener;
+  private Executor errorListenerExecutor;
+  private int inputWidth;
+  private int inputHeight;
+
+  /**
+   * Creates a {@code QueuingGlShaderProgram} instance.
+   *
+   * @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes.
+   *     If {@code true}, use 16-bit (half-precision) floating-point.
+   * @param queueSize The number of frames to buffer before producing output, and also the capacity
+   *     of the texture pool.
+   * @param concurrentEffect The asynchronous effect to apply to each frame.
+   */
+  public QueuingGlShaderProgram(
+      boolean useHighPrecisionColorComponents,
+      @IntRange(from = 1) int queueSize,
+      ConcurrentEffect<T> concurrentEffect) {
+    checkArgument(queueSize > 0);
+    this.concurrentEffect = concurrentEffect;
+    frameQueue = new ArrayDeque<>(queueSize);
+    outputTexturePool = new TexturePool(useHighPrecisionColorComponents, queueSize);
+    inputListener = new InputListener() {};
+    outputListener = new OutputListener() {};
+    errorListener = (frameProcessingException) -> {};
+    errorListenerExecutor = MoreExecutors.directExecutor();
+    inputWidth = C.LENGTH_UNSET;
+    inputHeight = C.LENGTH_UNSET;
+  }
+
+  @Override
+  public void setInputListener(InputListener inputListener) {
+    this.inputListener = inputListener;
+    for (int i = 0; i < outputTexturePool.freeTextureCount(); i++) {
+      inputListener.onReadyToAcceptInputFrame();
+    }
+  }
+
+  @Override
+  public void setOutputListener(OutputListener outputListener) {
+    this.outputListener = outputListener;
+  }
+
+  @Override
+  public void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) {
+    this.errorListenerExecutor = errorListenerExecutor;
+    this.errorListener = errorListener;
+  }
+
+  @Override
+  public void queueInputFrame(
+      GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
+    try {
+      if (inputWidth != inputTexture.width
+          || inputHeight != inputTexture.height
+          || !outputTexturePool.isConfigured()) {
+        inputWidth = inputTexture.width;
+        inputHeight = inputTexture.height;
+        outputTexturePool.ensureConfigured(glObjectsProvider, inputWidth, inputHeight);
+      }
+
+      // Focus on the next free buffer.
+      GlTextureInfo outputTexture = outputTexturePool.useTexture();
+
+      // Copy frame from inputTexture fbo to outputTexture fbo.
+      checkState(inputTexture.fboId != C.INDEX_UNSET);
+      GlUtil.blitFrameBuffer(
+          inputTexture.fboId,
+          new Rect(/* left= */ 0, /* top= */ 0, /* right= */ inputWidth, /* bottom= */ inputHeight),
+          outputTexture.fboId,
+          new Rect(
+              /* left= */ 0, /* top= */ 0, /* right= */ inputWidth, /* bottom= */ inputHeight));
+
+      Future<T> task = concurrentEffect.queueInputFrame(outputTexture, presentationTimeUs);
+      frameQueue.add(new TimedTextureInfo<T>(outputTexture, presentationTimeUs, task));
+
+      inputListener.onInputFrameProcessed(inputTexture);
+
+      if (frameQueue.size() == outputTexturePool.capacity()) {
+        checkState(outputOneFrame());
+      }
+    } catch (GlUtil.GlException e) {
+      onError(e);
+    }
+  }
+
+  @Override
+  public void releaseOutputFrame(GlTextureInfo outputTexture) {
+    if (!outputTexturePool.isUsingTexture(outputTexture)) {
+      // This allows us to ignore outputTexture instances not associated with this
+      // GlShaderProgram instance. This may happen if a GlShaderProgram is introduced into
+      // the GlShaderProgram chain after frames already exist in the pipeline.
+      // TODO - b/320481157: Consider removing this if condition and disallowing disconnecting a
+      //  GlShaderProgram while it still has in-use frames.
+      return;
+    }
+    outputTexturePool.freeTexture(outputTexture);
+    inputListener.onReadyToAcceptInputFrame();
+  }
+
+  @Override
+  public void signalEndOfCurrentInputStream() {
+    while (outputOneFrame()) {}
+    outputListener.onCurrentOutputStreamEnded();
+  }
+
+  @Override
+  @CallSuper
+  public void flush() {
+    cancelProcessingOfPendingFrames();
+    outputTexturePool.freeAllTextures();
+    inputListener.onFlush();
+    for (int i = 0; i < outputTexturePool.capacity(); i++) {
+      inputListener.onReadyToAcceptInputFrame();
+    }
+  }
+
+  @Override
+  @CallSuper
+  public void release() throws VideoFrameProcessingException {
+    try {
+      cancelProcessingOfPendingFrames();
+      outputTexturePool.deleteAllTextures();
+    } catch (GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e);
+    }
+  }
+
+  /**
+   * Outputs one frame from {@link #frameQueue}.
+   *
+   * <p>Returns {@code false} if no more frames are available for output.
+   */
+  private boolean outputOneFrame() {
+    TimedTextureInfo<T> timedTextureInfo = frameQueue.poll();
+    if (timedTextureInfo == null) {
+      return false;
+    }
+    try {
+      T result =
+          Futures.getChecked(
+              timedTextureInfo.task,
+              VideoFrameProcessingException.class,
+              PROCESSING_TIMEOUT_MS,
+              TimeUnit.MILLISECONDS);
+      GlUtil.focusFramebufferUsingCurrentContext(
+          timedTextureInfo.textureInfo.fboId,
+          timedTextureInfo.textureInfo.width,
+          timedTextureInfo.textureInfo.height);
+      concurrentEffect.finishProcessingAndBlend(
+          timedTextureInfo.textureInfo, timedTextureInfo.presentationTimeUs, result);
+      outputListener.onOutputFrameAvailable(
+          timedTextureInfo.textureInfo, timedTextureInfo.presentationTimeUs);
+      return true;
+    } catch (GlUtil.GlException | VideoFrameProcessingException e) {
+      onError(e);
+      return false;
+    }
+  }
+
+  private void cancelProcessingOfPendingFrames() {
+    TimedTextureInfo<T> timedTextureInfo;
+    while ((timedTextureInfo = frameQueue.poll()) != null) {
+      timedTextureInfo.task.cancel(/* mayInterruptIfRunning= */ false);
+    }
+  }
+
+  private void onError(Exception e) {
+    errorListenerExecutor.execute(
+        () -> errorListener.onError(VideoFrameProcessingException.from(e)));
+  }
+
+  private static class TimedTextureInfo<T> {
+    final GlTextureInfo textureInfo;
+    final long presentationTimeUs;
+    final Future<T> task;
+
+    TimedTextureInfo(GlTextureInfo textureInfo, long presentationTimeUs, Future<T> task) {
+      this.textureInfo = textureInfo;
+      this.presentationTimeUs = presentationTimeUs;
+      this.task = task;
+    }
+  }
+}