Effect: Make TexturePool and use in FinalWrapper.

Have the FinalShaderProgramWrapper / VideoFrameProcessor texture
output access textures provided through a texture pool, that
recycles used textures.

Also, add the TexturePool interface to generally re-use textures.

PiperOrigin-RevId: 532754377

libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java

@@ -20,6 +20,7 @@ import androidx.media3.common.util.UnstableApi;
 /** Contains information describing an OpenGL texture. */
 @UnstableApi
 public final class GlTextureInfo {
+  // TODO: b/262694346 - Add a release() method for GlTextureInfo.
 
   /** A {@link GlTextureInfo} instance with all fields unset. */
   public static final GlTextureInfo UNSET =

libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java

@@ -558,8 +558,8 @@ public final class GlUtil {
    *
    * @param width The width of the new texture in pixels.
    * @param height The height of the new texture in pixels.
-   * @param useHighPrecisionColorComponents If {@code false}, uses 8-bit unsigned bytes. If {@code
+   * @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes.
-   *     true}, use 16-bit (half-precision) floating-point.
+   *     If {@code true}, use 16-bit (half-precision) floating-point.
    * @throws GlException If the texture allocation fails.
    * @return The texture identifier for the newly-allocated texture.
    */

libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java

@@ -24,10 +24,7 @@ import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.Size;
 import androidx.media3.common.util.UnstableApi;
-import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.MoreExecutors;
-import java.util.ArrayDeque;
-import java.util.Iterator;
 import java.util.NoSuchElementException;
 import java.util.concurrent.Executor;
 
@@ -47,13 +44,7 @@ import java.util.concurrent.Executor;
  */
 @UnstableApi
 public abstract class BaseGlShaderProgram implements GlShaderProgram {
-
+  private final TexturePool outputTexturePool;
-  private final ArrayDeque<GlTextureInfo> freeOutputTextures;
-  private final ArrayDeque<GlTextureInfo> inUseOutputTextures;
-  private final int texturePoolCapacity;
-  private final boolean useHdr;
-
-  private GlObjectsProvider glObjectsProvider;
   protected InputListener inputListener;
   private OutputListener outputListener;
   private ErrorListener errorListener;
@@ -69,11 +60,8 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
    *     texture cache, the size should be the number of textures to cache.
    */
   public BaseGlShaderProgram(boolean useHdr, int texturePoolCapacity) {
-    freeOutputTextures = new ArrayDeque<>(texturePoolCapacity);
+    outputTexturePool =
-    inUseOutputTextures = new ArrayDeque<>(texturePoolCapacity);
+        new TexturePool(/* useHighPrecisionColorComponents= */ useHdr, texturePoolCapacity);
-    this.useHdr = useHdr;
-    this.texturePoolCapacity = texturePoolCapacity;
-    glObjectsProvider = GlObjectsProvider.DEFAULT;
     inputListener = new InputListener() {};
     outputListener = new OutputListener() {};
     errorListener = (frameProcessingException) -> {};
@@ -114,15 +102,7 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
   @Override
   public void setInputListener(InputListener inputListener) {
     this.inputListener = inputListener;
-    int numberOfFreeFramesToNotify;
+    for (int i = 0; i < outputTexturePool.freeTextureCount(); i++) {
-    if (getIteratorToAllTextures().hasNext()) {
-      // The frame buffers have already been allocated.
-      numberOfFreeFramesToNotify = freeOutputTextures.size();
-    } else {
-      // Defer frame buffer allocation to when queueing input frames.
-      numberOfFreeFramesToNotify = texturePoolCapacity;
-    }
-    for (int i = 0; i < numberOfFreeFramesToNotify; i++) {
       inputListener.onReadyToAcceptInputFrame();
     }
   }
@@ -143,22 +123,19 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
     checkState(
         !frameProcessingStarted,
         "The GlObjectsProvider cannot be set after frame processing has started.");
-    this.glObjectsProvider = glObjectsProvider;
+    outputTexturePool.setGlObjectsProvider(glObjectsProvider);
   }
 
   @Override
   public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
     try {
-      configureAllOutputTextures(inputTexture.width, inputTexture.height);
+      Size outputTextureSize = configure(inputTexture.width, inputTexture.height);
-      checkState(
+      outputTexturePool.ensureConfigured(
-          !freeOutputTextures.isEmpty(),
+          outputTextureSize.getWidth(), outputTextureSize.getHeight());
-          "The GlShaderProgram does not currently accept input frames. Release prior output frames"
-              + " first.");
       frameProcessingStarted = true;
 
       // Focus on the next free buffer.
-      GlTextureInfo outputTexture = freeOutputTextures.remove();
+      GlTextureInfo outputTexture = outputTexturePool.useTexture();
-      inUseOutputTextures.add(outputTexture);
 
       // Copy frame to fbo.
       GlUtil.focusFramebufferUsingCurrentContext(
@@ -176,9 +153,7 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
   @Override
   public void releaseOutputFrame(GlTextureInfo outputTexture) {
     frameProcessingStarted = true;
-    checkState(inUseOutputTextures.contains(outputTexture));
+    outputTexturePool.freeTexture(outputTexture);
-    inUseOutputTextures.remove(outputTexture);
-    freeOutputTextures.add(outputTexture);
     inputListener.onReadyToAcceptInputFrame();
   }
 
@@ -192,10 +167,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
   @CallSuper
   public void flush() {
     frameProcessingStarted = true;
-    freeOutputTextures.addAll(inUseOutputTextures);
+    outputTexturePool.freeAllTextures();
-    inUseOutputTextures.clear();
     inputListener.onFlush();
-    for (int i = 0; i < freeOutputTextures.size(); i++) {
+    for (int i = 0; i < outputTexturePool.capacity(); i++) {
       inputListener.onReadyToAcceptInputFrame();
     }
   }
@@ -205,52 +179,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
   public void release() throws VideoFrameProcessingException {
     frameProcessingStarted = true;
     try {
-      deleteAllOutputTextures();
+      outputTexturePool.deleteAllTextures();
     } catch (GlUtil.GlException e) {
       throw new VideoFrameProcessingException(e);
     }
   }
-
-  private void configureAllOutputTextures(int inputWidth, int inputHeight)
-      throws GlUtil.GlException, VideoFrameProcessingException {
-    Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
-    if (!allTextures.hasNext()) {
-      createAllOutputTextures(inputWidth, inputHeight);
-      return;
-    }
-    GlTextureInfo outputGlTextureInfo = allTextures.next();
-    if (outputGlTextureInfo.width != inputWidth || outputGlTextureInfo.height != inputHeight) {
-      deleteAllOutputTextures();
-      createAllOutputTextures(inputWidth, inputHeight);
-    }
-  }
-
-  private void createAllOutputTextures(int width, int height)
-      throws GlUtil.GlException, VideoFrameProcessingException {
-    checkState(freeOutputTextures.isEmpty());
-    checkState(inUseOutputTextures.isEmpty());
-    Size outputSize = configure(width, height);
-    for (int i = 0; i < texturePoolCapacity; i++) {
-      int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr);
-      GlTextureInfo outputTexture =
-          glObjectsProvider.createBuffersForTexture(
-              outputTexId, outputSize.getWidth(), outputSize.getHeight());
-      freeOutputTextures.add(outputTexture);
-    }
-  }
-
-  private void deleteAllOutputTextures() throws GlUtil.GlException {
-    Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
-    while (allTextures.hasNext()) {
-      GlTextureInfo textureInfo = allTextures.next();
-      GlUtil.deleteTexture(textureInfo.texId);
-      GlUtil.deleteFbo(textureInfo.fboId);
-    }
-    freeOutputTextures.clear();
-    inUseOutputTextures.clear();
-  }
-
-  private Iterator<GlTextureInfo> getIteratorToAllTextures() {
-    return Iterables.concat(freeOutputTextures, inUseOutputTextures).iterator();
-  }
 }

libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java

@@ -44,6 +44,7 @@ import androidx.media3.common.util.Log;
 import androidx.media3.common.util.Size;
 import androidx.media3.common.util.Util;
 import com.google.common.collect.ImmutableList;
+import java.util.ArrayDeque;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.Executor;
@@ -87,9 +88,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final Executor videoFrameProcessorListenerExecutor;
   private final VideoFrameProcessor.Listener videoFrameProcessorListener;
   private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
-  private final Queue<Pair<GlTextureInfo, Long>> outputTextures;
+  private final TexturePool outputTexturePool;
+  private final Queue<Long> outputTextureTimestamps; // Synchronized with outputTexturePool.
   @Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
-  private final int textureOutputCapacity;
 
   private int inputWidth;
   private int inputHeight;
@@ -143,11 +144,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.videoFrameProcessorListener = videoFrameProcessorListener;
     this.glObjectsProvider = glObjectsProvider;
     this.textureOutputListener = textureOutputListener;
-    this.textureOutputCapacity = textureOutputCapacity;
 
     inputListener = new InputListener() {};
     availableFrames = new ConcurrentLinkedQueue<>();
-    outputTextures = new ConcurrentLinkedQueue<>();
+
+    boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
+    outputTexturePool = new TexturePool(useHighPrecisionColorComponents, textureOutputCapacity);
+    outputTextureTimestamps = new ArrayDeque<>(textureOutputCapacity);
   }
 
   @Override
@@ -156,6 +159,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         !frameProcessingStarted,
         "The GlObjectsProvider cannot be set after frame processing has started.");
     this.glObjectsProvider = glObjectsProvider;
+    outputTexturePool.setGlObjectsProvider(glObjectsProvider);
   }
 
   @Override
@@ -207,7 +211,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
       }
     } else {
-      checkState(outputTextures.size() < textureOutputCapacity);
+      checkState(outputTexturePool.freeTextureCount() > 0);
       renderFrame(inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000);
     }
     maybeOnReadyToAcceptInputFrame();
@@ -219,16 +223,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     throw new UnsupportedOperationException();
   }
 
-  public void releaseOutputFrame(long presentationTimeUs) throws VideoFrameProcessingException {
+  public void releaseOutputFrame(long presentationTimeUs) {
-    while (!outputTextures.isEmpty()
+    while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
-        && checkNotNull(outputTextures.peek()).second <= presentationTimeUs) {
+        && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) {
-      GlTextureInfo outputTexture = outputTextures.remove().first;
+      outputTexturePool.freeTexture();
-      try {
+      outputTextureTimestamps.remove();
-        GlUtil.deleteTexture(outputTexture.texId);
-        GlUtil.deleteFbo(outputTexture.fboId);
-      } catch (GlUtil.GlException exception) {
-        throw new VideoFrameProcessingException(exception);
-      }
       maybeOnReadyToAcceptInputFrame();
     }
   }
@@ -261,11 +260,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       defaultShaderProgram.release();
     }
     try {
-      while (!outputTextures.isEmpty()) {
+      outputTexturePool.deleteAllTextures();
-        GlTextureInfo outputTexture = outputTextures.remove().first;
-        GlUtil.deleteTexture(outputTexture.texId);
-        GlUtil.deleteFbo(outputTexture.fboId);
-      }
       GlUtil.destroyEglSurface(eglDisplay, outputEglSurface);
     } catch (GlUtil.GlException e) {
       throw new VideoFrameProcessingException(e);
@@ -304,7 +299,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   private void maybeOnReadyToAcceptInputFrame() {
-    if (textureOutputListener == null || outputTextures.size() < textureOutputCapacity) {
+    if (textureOutputListener == null || outputTexturePool.freeTextureCount() > 0) {
       inputListener.onReadyToAcceptInputFrame();
     }
   }
@@ -363,15 +358,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
   private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs)
       throws GlUtil.GlException, VideoFrameProcessingException {
-    // TODO(b/262694346): Use a texture pool instead of creating a new texture on every frame.
+    GlTextureInfo outputTexture = outputTexturePool.useTexture();
-    int outputTexId =
+    outputTextureTimestamps.add(presentationTimeUs);
-        GlUtil.createTexture(
-            outputWidth,
-            outputHeight,
-            /* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
-    GlTextureInfo outputTexture =
-        glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
-
     GlUtil.focusFramebufferUsingCurrentContext(
         outputTexture.fboId, outputTexture.width, outputTexture.height);
     GlUtil.clearOutputFrame();
@@ -381,7 +369,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     //  glFinish. Consider removing glFinish and requiring onTextureRendered to handle
     //  synchronization.
     GLES20.glFinish();
-    outputTextures.add(Pair.create(outputTexture, presentationTimeUs));
     checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs);
   }
 
@@ -445,6 +432,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               // Frames are only rendered automatically when outputting to an encoder.
               /* isEncoderInputSurface= */ renderFramesAutomatically);
     }
+    if (textureOutputListener != null) {
+      outputTexturePool.ensureConfigured(outputWidth, outputHeight);
+    }
 
     @Nullable
     SurfaceView debugSurfaceView =

libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java

@@ -0,0 +1,159 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkState;
+
+import androidx.media3.common.GlObjectsProvider;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.util.GlUtil;
+import com.google.common.collect.Iterables;
+import java.util.ArrayDeque;
+import java.util.Iterator;
+import java.util.Queue;
+
+/** Holds {@code capacity} textures, to re-use textures. */
+/* package */ final class TexturePool {
+  private final Queue<GlTextureInfo> freeTextures;
+  private final Queue<GlTextureInfo> inUseTextures;
+  private final int capacity;
+  private final boolean useHighPrecisionColorComponents;
+
+  private GlObjectsProvider glObjectsProvider;
+
+  /**
+   * Creates a {@code TexturePool} instance.
+   *
+   * @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes.
+   *     If {@code true}, use 16-bit (half-precision) floating-point.
+   * @param capacity The capacity of the texture pool.
+   */
+  public TexturePool(boolean useHighPrecisionColorComponents, int capacity) {
+    this.capacity = capacity;
+    this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
+
+    freeTextures = new ArrayDeque<>(capacity);
+    inUseTextures = new ArrayDeque<>(capacity);
+
+    glObjectsProvider = new DefaultGlObjectsProvider(/* sharedEglContext= */ null);
+  }
+
+  /** Sets the {@link GlObjectsProvider}. */
+  public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
+    checkState(!isConfigured());
+    this.glObjectsProvider = glObjectsProvider;
+  }
+
+  /** Returns whether the instance has been {@linkplain #ensureConfigured configured}. */
+  public boolean isConfigured() {
+    return getIteratorToAllTextures().hasNext();
+  }
+
+  /** Returns the {@code capacity} of the instance. */
+  public int capacity() {
+    return capacity;
+  }
+
+  /** Returns the number of free textures available to {@link #useTexture}. */
+  public int freeTextureCount() {
+    if (!isConfigured()) {
+      return capacity;
+    }
+    return freeTextures.size();
+  }
+
+  /**
+   * Ensures that this instance is configured with the {@code width} and {@code height}.
+   *
+   * <p>Reconfigures backing textures as needed.
+   */
+  public void ensureConfigured(int width, int height) throws GlUtil.GlException {
+    if (!isConfigured()) {
+      createTextures(width, height);
+      return;
+    }
+    GlTextureInfo texture = getIteratorToAllTextures().next();
+    if (texture.width != width || texture.height != height) {
+      deleteAllTextures();
+      createTextures(width, height);
+    }
+  }
+
+  /** Returns a {@link GlTextureInfo} and marks it as in-use. */
+  public GlTextureInfo useTexture() {
+    if (freeTextures.isEmpty()) {
+      throw new IllegalStateException(
+          "Textures are all in use. Please release in-use textures before calling useTexture.");
+    }
+    GlTextureInfo texture = freeTextures.remove();
+    inUseTextures.add(texture);
+    return texture;
+  }
+
+  /**
+   * Frees the texture represented by {@code textureInfo}.
+   *
+   * <p>Throws {@link IllegalStateException} if {@code textureInfo} isn't in use.
+   */
+  public void freeTexture(GlTextureInfo textureInfo) {
+    checkState(inUseTextures.contains(textureInfo));
+    inUseTextures.remove(textureInfo);
+    freeTextures.add(textureInfo);
+  }
+
+  /**
+   * Frees the oldest in-use texture.
+   *
+   * <p>Throws {@link IllegalStateException} if there's no textures in use to free.
+   */
+  public void freeTexture() {
+    checkState(!inUseTextures.isEmpty());
+    GlTextureInfo texture = inUseTextures.remove();
+    freeTextures.add(texture);
+  }
+
+  /** Free all in-use textures. */
+  public void freeAllTextures() {
+    freeTextures.addAll(inUseTextures);
+    inUseTextures.clear();
+  }
+
+  /** Deletes all textures. */
+  public void deleteAllTextures() throws GlUtil.GlException {
+    Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
+    while (allTextures.hasNext()) {
+      GlTextureInfo textureInfo = allTextures.next();
+      GlUtil.deleteTexture(textureInfo.texId);
+      GlUtil.deleteFbo(textureInfo.fboId);
+    }
+    freeTextures.clear();
+    inUseTextures.clear();
+  }
+
+  private void createTextures(int width, int height) throws GlUtil.GlException {
+    checkState(freeTextures.isEmpty());
+    checkState(inUseTextures.isEmpty());
+    for (int i = 0; i < capacity; i++) {
+      int texId = GlUtil.createTexture(width, height, useHighPrecisionColorComponents);
+      GlTextureInfo texture = glObjectsProvider.createBuffersForTexture(texId, width, height);
+      freeTextures.add(texture);
+    }
+  }
+
+  private Iterator<GlTextureInfo> getIteratorToAllTextures() {
+    return Iterables.concat(freeTextures, inUseTextures).iterator();
+  }
+}