Add ByteBufferGlEffect for easier non-GPU video effects

* new ByteBufferGlEffect GlEffect that enables API users to
implement an effect that accesses video frame data via CPU-mapped ByteBuffer
* ByteBufferConcurrentEffect responsible for reading
video frame data in CPU-accessible ByteBuffer

PiperOrigin-RevId: 666375594

libraries/effect/src/androidTest/java/androidx/media3/effect/ByteBufferGlEffectTest.java

@@ -0,0 +1,217 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames;
+import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.graphics.Bitmap;
+import android.graphics.Color;
+import android.graphics.Matrix;
+import android.graphics.Rect;
+import android.text.Spannable;
+import android.text.SpannableString;
+import android.text.style.AbsoluteSizeSpan;
+import android.text.style.ForegroundColorSpan;
+import android.text.style.TypefaceSpan;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.util.Consumer;
+import androidx.media3.common.util.Size;
+import androidx.media3.common.util.Util;
+import androidx.media3.test.utils.TextureBitmapReader;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+/** Tests for {@link ByteBufferGlEffect}. */
+@RunWith(AndroidJUnit4.class)
+public class ByteBufferGlEffectTest {
+
+  @Rule public final TestName testName = new TestName();
+
+  private static final String ASSET_PATH = "test-generated-goldens/ByteBufferGlEffectTest";
+
+  private static final int INPUT_FRAME_WIDTH = 100;
+  private static final int INPUT_FRAME_HEIGHT = 50;
+  private static final int EFFECT_INPUT_FRAME_WIDTH = 75;
+  private static final int EFFECT_INPUT_FRAME_HEIGHT = 30;
+  private static final int EFFECT_OUTPUT_FRAME_WIDTH = 50;
+  private static final int EFFECT_OUTPUT_FRAME_HEIGHT = 20;
+  private static final Consumer<SpannableString> TEXT_SPAN_CONSUMER =
+      (text) -> {
+        text.setSpan(
+            new ForegroundColorSpan(Color.BLACK),
+            /* start= */ 0,
+            text.length(),
+            Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+        text.setSpan(
+            new AbsoluteSizeSpan(/* size= */ 24),
+            /* start= */ 0,
+            text.length(),
+            Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+        text.setSpan(
+            new TypefaceSpan(/* family= */ "sans-serif"),
+            /* start= */ 0,
+            text.length(),
+            Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+      };
+
+  private @MonotonicNonNull TextureBitmapReader textureBitmapReader;
+  private String testId;
+
+  @Before
+  public void setUp() {
+    textureBitmapReader = new TextureBitmapReader();
+    testId = testName.getMethodName();
+  }
+
+  @Test
+  public void byteBufferEffectImplementation_receivesCorrectBitmapData() throws Exception {
+    List<Bitmap> effectInputBitmaps = new ArrayList<>();
+    List<Bitmap> effectOutputBitmaps = new ArrayList<>();
+    ImmutableList<Long> frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L);
+    ImmutableList<Long> actualPresentationTimesUs =
+        generateAndProcessFrames(
+            INPUT_FRAME_WIDTH,
+            INPUT_FRAME_HEIGHT,
+            frameTimesUs,
+            new ByteBufferGlEffect<>(
+                new TestByteBufferProcessor(effectInputBitmaps, effectOutputBitmaps)),
+            textureBitmapReader,
+            TEXT_SPAN_CONSUMER);
+
+    assertThat(actualPresentationTimesUs).containsExactlyElementsIn(frameTimesUs).inOrder();
+
+    getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH);
+    assertBitmapsMatchExpected(
+        effectInputBitmaps, actualPresentationTimesUs, testId, /* suffix= */ "_input");
+    assertBitmapsMatchExpected(
+        effectOutputBitmaps, actualPresentationTimesUs, testId, /* suffix= */ "_output");
+  }
+
+  private static class TestByteBufferProcessor implements ByteBufferGlEffect.Processor<Bitmap> {
+
+    private final List<Bitmap> inputBitmaps;
+    private final List<Bitmap> outputBitmaps;
+    private final ListeningExecutorService drawingService;
+
+    TestByteBufferProcessor(List<Bitmap> inputBitmaps, List<Bitmap> outputBitmaps) {
+      drawingService =
+          MoreExecutors.listeningDecorator(
+              Util.newSingleThreadExecutor(/* threadName= */ "TestByteBufferEffect"));
+      this.inputBitmaps = inputBitmaps;
+      this.outputBitmaps = outputBitmaps;
+    }
+
+    @Override
+    public Size configure(int inputWidth, int inputHeight) {
+      checkState(inputWidth == INPUT_FRAME_WIDTH);
+      checkState(inputHeight == INPUT_FRAME_HEIGHT);
+      return new Size(EFFECT_INPUT_FRAME_WIDTH, EFFECT_INPUT_FRAME_HEIGHT);
+    }
+
+    @Override
+    public Rect getScaledRegion(long presentationTimeUs) {
+      return new Rect(
+          /* left= */ 0,
+          /* top= */ 0,
+          /* right= */ INPUT_FRAME_WIDTH,
+          /* bottom= */ INPUT_FRAME_HEIGHT);
+    }
+
+    @Override
+    public ListenableFuture<Bitmap> processPixelBuffer(
+        ByteBuffer pixelBuffer, long presentationTimeUs) {
+      // TODO: b/361286064 - Add helper functions for easier conversion to Bitmap.
+      // The memory layout of pixels differs between OpenGL and Android Bitmap.
+      // The first pixel in OpenGL is in the lower left corner, and the first
+      // pixel in Android Bitmap is in the top left corner.
+      // Mirror the Bitmap's Y axis.
+      Bitmap bitmapInGlMemoryLayout =
+          Bitmap.createBitmap(
+              EFFECT_INPUT_FRAME_WIDTH, EFFECT_INPUT_FRAME_HEIGHT, Bitmap.Config.ARGB_8888);
+      bitmapInGlMemoryLayout.copyPixelsFromBuffer(pixelBuffer);
+      Matrix glToAndroidTransformation = new Matrix();
+      glToAndroidTransformation.setScale(/* sx= */ 1, /* sy= */ -1);
+      Bitmap inputBitmap =
+          Bitmap.createBitmap(
+              bitmapInGlMemoryLayout,
+              /* x= */ 0,
+              /* y= */ 0,
+              bitmapInGlMemoryLayout.getWidth(),
+              bitmapInGlMemoryLayout.getHeight(),
+              glToAndroidTransformation,
+              /* filter= */ true);
+      inputBitmaps.add(inputBitmap);
+      return drawingService.submit(
+          () ->
+              Bitmap.createScaledBitmap(
+                  inputBitmap,
+                  EFFECT_OUTPUT_FRAME_WIDTH,
+                  EFFECT_OUTPUT_FRAME_HEIGHT,
+                  /* filter= */ true));
+    }
+
+    @Override
+    public void finishProcessingAndBlend(
+        GlTextureInfo outputFrame, long presentationTimeUs, Bitmap result) {
+      outputBitmaps.add(result);
+    }
+
+    @Override
+    public void release() {}
+  }
+
+  private static void assertBitmapsMatchExpected(
+      List<Bitmap> bitmaps, List<Long> presentationTimesUs, String testId, String suffix)
+      throws IOException {
+    checkState(bitmaps.size() == presentationTimesUs.size());
+    for (int i = 0; i < presentationTimesUs.size(); i++) {
+      long presentationTimeUs = presentationTimesUs.get(i);
+      Bitmap actualBitmap = bitmaps.get(i);
+      maybeSaveTestBitmap(
+          testId, /* bitmapLabel= */ presentationTimeUs + suffix, actualBitmap, /* path= */ null);
+      Bitmap expectedBitmap =
+          readBitmap(
+              Util.formatInvariant("%s/pts_%d.png", ASSET_PATH + suffix, presentationTimeUs));
+      float averagePixelAbsoluteDifference =
+          getBitmapAveragePixelAbsoluteDifferenceArgb8888(
+              expectedBitmap, actualBitmap, testId + "_" + i);
+      // Golden bitmaps were generated with ffmpeg, use a higher threshold.
+      // TODO: b/361286064 - Use PSNR for quality computations.
+      assertThat(averagePixelAbsoluteDifference)
+          .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE);
+    }
+  }
+}

libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java

@@ -29,6 +29,7 @@ import android.text.style.AbsoluteSizeSpan;
 import android.text.style.ForegroundColorSpan;
 import android.text.style.TypefaceSpan;
 import android.util.Pair;
+import androidx.media3.common.GlObjectsProvider;
 import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.util.Consumer;
 import androidx.media3.test.utils.TextureBitmapReader;
@@ -162,7 +163,8 @@ public class QueuingGlShaderProgramTest {
     }
 
     @Override
-    public Future<Long> queueInputFrame(GlTextureInfo textureInfo, long presentationTimeUs) {
+    public Future<Long> queueInputFrame(
+        GlObjectsProvider glObjectsProvider, GlTextureInfo textureInfo, long presentationTimeUs) {
       checkState(textureInfo.width == BLANK_FRAME_WIDTH);
       checkState(textureInfo.height == BLANK_FRAME_HEIGHT);
       events.add(Pair.create("queueInputFrame", presentationTimeUs));

libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
+
+import android.graphics.Rect;
+import android.opengl.GLES20;
+import android.opengl.GLES30;
+import androidx.media3.common.C;
+import androidx.media3.common.GlObjectsProvider;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.Size;
+import java.nio.ByteBuffer;
+import java.util.concurrent.Future;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
+/**
+ * A {@link QueuingGlShaderProgram.ConcurrentEffect} implementation which wraps a {@link
+ * ByteBufferGlEffect.Processor}.
+ *
+ * <p>This class is responsible for asynchronously transferring texture frame data to a
+ * CPU-accessible {@link ByteBuffer} that will be used by the wrapped {@link
+ * ByteBufferGlEffect.Processor}.
+ */
+/* package */ class ByteBufferConcurrentEffect<T>
+    implements QueuingGlShaderProgram.ConcurrentEffect<T> {
+
+  private static final int BYTES_PER_PIXEL = 4;
+
+  private final ByteBufferGlEffect.Processor<T> processor;
+
+  private int inputWidth;
+  private int inputHeight;
+  private @MonotonicNonNull GlTextureInfo effectInputTexture;
+
+  /**
+   * Creates an instance.
+   *
+   * @param processor The {@linkplain ByteBufferGlEffect.Processor effect}.
+   */
+  public ByteBufferConcurrentEffect(ByteBufferGlEffect.Processor<T> processor) {
+    this.processor = processor;
+    inputWidth = C.LENGTH_UNSET;
+    inputHeight = C.LENGTH_UNSET;
+  }
+
+  @Override
+  public Future<T> queueInputFrame(
+      GlObjectsProvider glObjectsProvider, GlTextureInfo textureInfo, long presentationTimeUs) {
+    try {
+      if (effectInputTexture == null
+          || textureInfo.width != inputWidth
+          || textureInfo.height != inputHeight) {
+        inputWidth = textureInfo.width;
+        inputHeight = textureInfo.height;
+        Size effectInputSize = processor.configure(inputWidth, inputHeight);
+        if (effectInputTexture != null) {
+          effectInputTexture.release();
+        }
+        int texId =
+            GlUtil.createTexture(
+                effectInputSize.getWidth(),
+                effectInputSize.getHeight(),
+                /* useHighPrecisionColorComponents= */ false);
+        effectInputTexture =
+            glObjectsProvider.createBuffersForTexture(
+                texId, effectInputSize.getWidth(), effectInputSize.getHeight());
+      }
+
+      GlUtil.blitFrameBuffer(
+          textureInfo.fboId,
+          processor.getScaledRegion(presentationTimeUs),
+          effectInputTexture.fboId,
+          new Rect(
+              /* left= */ 0, /* top= */ 0, effectInputTexture.width, effectInputTexture.height));
+
+      GlUtil.focusFramebufferUsingCurrentContext(
+          effectInputTexture.fboId, effectInputTexture.width, effectInputTexture.height);
+      ByteBuffer pixelBuffer =
+          ByteBuffer.allocateDirect(texturePixelBufferSize(effectInputTexture));
+      GLES20.glReadPixels(
+          /* x= */ 0,
+          /* y= */ 0,
+          effectInputTexture.width,
+          effectInputTexture.height,
+          GLES30.GL_RGBA,
+          GLES30.GL_UNSIGNED_BYTE,
+          pixelBuffer);
+      GlUtil.checkGlError();
+      return processor.processPixelBuffer(pixelBuffer, presentationTimeUs);
+    } catch (GlUtil.GlException | VideoFrameProcessingException e) {
+      return immediateFailedFuture(e);
+    }
+  }
+
+  @Override
+  public void finishProcessingAndBlend(GlTextureInfo textureInfo, long presentationTimeUs, T result)
+      throws VideoFrameProcessingException {
+    processor.finishProcessingAndBlend(textureInfo, presentationTimeUs, result);
+  }
+
+  private static int texturePixelBufferSize(GlTextureInfo textureInfo) {
+    return textureInfo.width * textureInfo.height * BYTES_PER_PIXEL;
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+
+import android.content.Context;
+import android.graphics.Rect;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.Size;
+import androidx.media3.common.util.UnstableApi;
+import com.google.common.util.concurrent.ListenableFuture;
+import java.nio.ByteBuffer;
+import java.util.concurrent.Future;
+
+/**
+ * A {@link GlEffect} implementation that runs an asynchronous {@link Processor} on video frame data
+ * passed in as a {@link ByteBuffer}.
+ *
+ * <p>This effect can be used to apply CPU-based effects. Or the provided {@link ByteBuffer} can be
+ * passed to other heterogeneous compute components that are available such as another GPU context,
+ * FPGAs, or NPUs.
+ */
+@UnstableApi
+/* package */ class ByteBufferGlEffect<T> implements GlEffect {
+
+  private static final int DEFAULT_QUEUE_SIZE = 6;
+
+  /**
+   * A processor that takes in {@link ByteBuffer ByteBuffers} that represent input image data, and
+   * produces results of type {@code <T>}.
+   *
+   * <p>All methods are called on the GL thread.
+   *
+   * @param <T> The result type of running the processor.
+   */
+  public interface Processor<T> {
+
+    /**
+     * Configures the instance and returns the dimensions of the image required by {@link
+     * #processPixelBuffer}.
+     *
+     * <p>When the returned dimensions differ from {@code inputWidth} and {@code inputHeight}, the
+     * image will be scaled based on {@link #getScaledRegion}.
+     *
+     * @param inputWidth The input width in pixels.
+     * @param inputHeight The input height in pixels.
+     * @return The size in pixels of the image data accepted by {@link #processPixelBuffer}.
+     * @throws VideoFrameProcessingException On error.
+     */
+    Size configure(int inputWidth, int inputHeight) throws VideoFrameProcessingException;
+
+    /**
+     * Selects a region of the input texture that will be scaled to fill the image given that is
+     * given to {@link #processPixelBuffer}.
+     *
+     * <p>Called once per input frame.
+     *
+     * <p>The contents are scaled to fit the image dimensions returned by {@link #configure}.
+     *
+     * @param presentationTimeUs The presentation time in microseconds.
+     * @return The rectangular region of the input image that will be scaled to fill the effect
+     *     input image.
+     */
+    // TODO: b/b/361286064 - This method misuses android.graphics.Rect for OpenGL coordinates.
+    //   Implement a custom GlUtils.Rect to correctly label lower left corner as (0, 0).
+    Rect getScaledRegion(long presentationTimeUs);
+
+    /**
+     * Processing the image data in the {@code pixelBuffer}.
+     *
+     * <p>Accessing {@code pixelBuffer} after the returned future is {@linkplain Future#isDone()
+     * done} or {@linkplain Future#isCancelled() cancelled} can lead to undefined behaviour.
+     *
+     * @param pixelBuffer The image data.
+     * @param presentationTimeUs The presentation time in microseconds.
+     * @return A {@link ListenableFuture} of the result.
+     */
+    // TODO: b/361286064 - Add helper functions for easier conversion to Bitmap.
+    ListenableFuture<T> processPixelBuffer(ByteBuffer pixelBuffer, long presentationTimeUs);
+
+    /**
+     * Finishes processing the frame at {@code presentationTimeUs}. Use this method to perform
+     * custom drawing on the output frame.
+     *
+     * <p>The {@linkplain GlTextureInfo outputFrame} contains the image data corresponding to the
+     * frame at {@code presentationTimeUs} when this method is invoked.
+     *
+     * @param outputFrame The texture info of the frame.
+     * @param presentationTimeUs The presentation timestamp of the frame, in microseconds.
+     * @param result The result of the asynchronous computation in {@link #processPixelBuffer}.
+     */
+    void finishProcessingAndBlend(GlTextureInfo outputFrame, long presentationTimeUs, T result)
+        throws VideoFrameProcessingException;
+
+    /**
+     * Releases all resources.
+     *
+     * @throws VideoFrameProcessingException If an error occurs while releasing resources.
+     */
+    void release() throws VideoFrameProcessingException;
+  }
+
+  private final Processor<T> processor;
+
+  /**
+   * Creates an instance.
+   *
+   * @param processor The effect to apply.
+   */
+  public ByteBufferGlEffect(Processor<T> processor) {
+    this.processor = processor;
+  }
+
+  @Override
+  public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr)
+      throws VideoFrameProcessingException {
+    // TODO: b/361286064 - Implement HDR support.
+    checkArgument(!useHdr, "HDR support not yet implemented.");
+    return new QueuingGlShaderProgram<>(
+        /* useHighPrecisionColorComponents= */ useHdr,
+        /* queueSize= */ DEFAULT_QUEUE_SIZE,
+        new ByteBufferConcurrentEffect<>(processor));
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java

@@ -92,7 +92,8 @@ import java.util.concurrent.TimeUnit;
      * @param presentationTimeUs The presentation timestamp of the input frame, in microseconds.
      * @return A {@link Future} representing pending completion of the task.
      */
-    Future<T> queueInputFrame(GlTextureInfo textureInfo, long presentationTimeUs);
+    Future<T> queueInputFrame(
+        GlObjectsProvider glObjectsProvider, GlTextureInfo textureInfo, long presentationTimeUs);
 
     /**
      * Finishes processing the frame at {@code presentationTimeUs}. This method optionally allows
@@ -172,6 +173,8 @@ import java.util.concurrent.TimeUnit;
       if (inputWidth != inputTexture.width
           || inputHeight != inputTexture.height
           || !outputTexturePool.isConfigured()) {
+        // Output all pending frames before processing a format change.
+        while (outputOneFrame()) {}
         inputWidth = inputTexture.width;
         inputHeight = inputTexture.height;
         outputTexturePool.ensureConfigured(glObjectsProvider, inputWidth, inputHeight);
@@ -189,7 +192,8 @@ import java.util.concurrent.TimeUnit;
           new Rect(
               /* left= */ 0, /* top= */ 0, /* right= */ inputWidth, /* bottom= */ inputHeight));
 
-      Future<T> task = concurrentEffect.queueInputFrame(outputTexture, presentationTimeUs);
+      Future<T> task =
+          concurrentEffect.queueInputFrame(glObjectsProvider, outputTexture, presentationTimeUs);
       frameQueue.add(new TimedTextureInfo<T>(outputTexture, presentationTimeUs, task));
 
       inputListener.onInputFrameProcessed(inputTexture);