Effect: Use callback to release texture

This allows us to avoid needing a reference to the VideoFrameProcessor, which
can be especially difficult if an App only has a reference to the
VideoFrameProcessor.Factory it passes into Transformer/ExoPlayer.

PiperOrigin-RevId: 533205983

libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java

@@ -49,7 +49,7 @@ public interface VideoFrameProcessor {
 
   /** A listener for frame processing events. */
   @UnstableApi
-  public interface OnInputFrameProcessedListener {
+  interface OnInputFrameProcessedListener {
 
     /** Called when the given input frame has been processed. */
     void onInputFrameProcessed(int textureId) throws VideoFrameProcessingException;
@@ -291,18 +291,6 @@ public interface VideoFrameProcessor {
    */
   void renderOutputFrame(long renderTimeNs);
 
-  /**
-   * Releases resources associated with all output frames with presentation time less than or equal
-   * to {@code presentationTimeUs}.
-   *
-   * <p>Not needed for outputting to an {@linkplain #setOutputSurfaceInfo output surface}, but may
-   * be required for other outputs.
-   *
-   * @param presentationTimeUs The presentation time where all frames before and at this time should
-   *     be released, in microseconds.
-   */
-  void releaseOutputFrame(long presentationTimeUs);
-
   /**
    * Informs the {@code VideoFrameProcessor} that no further input frames should be accepted.
    *

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java

@@ -71,11 +71,24 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
   /** Listener interface for texture output. */
   @VisibleForTesting(otherwise = PACKAGE_PRIVATE)
   public interface TextureOutputListener {
-    /** Called when a texture has been rendered to. */
+    /**
-    void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs)
+     * Called when a texture has been rendered to. {@code releaseOutputTextureCallback} must be
+     * called to release the {@link GlTextureInfo}.
+     */
+    void onTextureRendered(
+        GlTextureInfo outputTexture,
+        long presentationTimeUs,
+        ReleaseOutputTextureCallback releaseOutputTextureCallback)
         throws VideoFrameProcessingException;
   }
 
+  /**
+   * Releases the output information stored for textures before and at {@code presentationTimeUs}.
+   */
+  public interface ReleaseOutputTextureCallback {
+    void release(long presentationTimeUs);
+  }
+
   /** A factory for {@link DefaultVideoFrameProcessor} instances. */
   public static final class Factory implements VideoFrameProcessor.Factory {
 
@@ -118,9 +131,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
        * Sets texture output settings.
        *
        * <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
-       * {@link TextureOutputListener#onTextureRendered}. Textures will stop being output when
+       * {@link TextureOutputListener#onTextureRendered}. Textures will stop being outputted when
-       * {@code textureOutputCapacity} is reached, until they're released via {@link
+       * the number of output textures available reaches the {@code textureOutputCapacity}. To
-       * #releaseOutputFrame}. Output textures must be released using {@link #releaseOutputFrame}.
+       * regain capacity, output textures must be released using {@link
+       * ReleaseOutputTextureCallback}.
        *
        * <p>If not set, there will be no texture output.
        *
@@ -455,21 +469,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
         () -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs));
   }
 
-  /**
-   * {@inheritDoc}
-   *
-   * <p>If a {@link TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set},
-   * this must be called to release the output information stored in the {@link GlTextureInfo}
-   * instances.
-   */
-  @Override
-  public void releaseOutputFrame(long presentationTimeUs) {
-    // TODO(b/262694346): Add Compositor system tests exercising this code path after GL texture
-    //  input is possible.
-    videoFrameProcessingTaskExecutor.submit(
-        () -> finalShaderProgramWrapper.releaseOutputFrame(presentationTimeUs));
-  }
-
   @Override
   public void signalEndOfInput() {
     DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos();
@@ -610,6 +609,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
             outputColorInfo,
             enableColorTransfers,
             renderFramesAutomatically,
+            videoFrameProcessingTaskExecutor,
             executor,
             listener,
             glObjectsProvider,
@@ -661,6 +661,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
       ColorInfo outputColorInfo,
       boolean enableColorTransfers,
       boolean renderFramesAutomatically,
+      VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
       Executor executor,
       Listener listener,
       GlObjectsProvider glObjectsProvider,
@@ -714,6 +715,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
             outputColorInfo,
             enableColorTransfers,
             renderFramesAutomatically,
+            videoFrameProcessingTaskExecutor,
             executor,
             listener,
             glObjectsProvider,

libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java

@@ -85,6 +85,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final ColorInfo outputColorInfo;
   private final boolean enableColorTransfers;
   private final boolean renderFramesAutomatically;
+  private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
   private final Executor videoFrameProcessorListenerExecutor;
   private final VideoFrameProcessor.Listener videoFrameProcessorListener;
   private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
@@ -126,6 +127,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       ColorInfo outputColorInfo,
       boolean enableColorTransfers,
       boolean renderFramesAutomatically,
+      VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
       Executor videoFrameProcessorListenerExecutor,
       VideoFrameProcessor.Listener videoFrameProcessorListener,
       GlObjectsProvider glObjectsProvider,
@@ -140,6 +142,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.outputColorInfo = outputColorInfo;
     this.enableColorTransfers = enableColorTransfers;
     this.renderFramesAutomatically = renderFramesAutomatically;
+    this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
     this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
     this.videoFrameProcessorListener = videoFrameProcessorListener;
     this.glObjectsProvider = glObjectsProvider;
@@ -224,6 +227,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   public void releaseOutputFrame(long presentationTimeUs) {
+    videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs));
+  }
+
+  private void releaseOutputFrameInternal(long presentationTimeUs) {
     while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
         && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) {
       outputTexturePool.freeTexture();
@@ -232,16 +239,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
   }
 
-  public void renderOutputFrame(long renderTimeNs) {
-    frameProcessingStarted = true;
-    checkState(!renderFramesAutomatically);
-    Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
-    renderFrame(
-        /* inputTexture= */ oldestAvailableFrame.first,
-        /* presentationTimeUs= */ oldestAvailableFrame.second,
-        renderTimeNs);
-  }
-
   @Override
   public void flush() {
     frameProcessingStarted = true;
@@ -267,6 +264,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
   }
 
+  public void renderOutputFrame(long renderTimeNs) {
+    frameProcessingStarted = true;
+    checkState(!renderFramesAutomatically);
+    Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
+    renderFrame(
+        /* inputTexture= */ oldestAvailableFrame.first,
+        /* presentationTimeUs= */ oldestAvailableFrame.second,
+        renderTimeNs);
+  }
+
   /**
    * Sets the output {@link SurfaceInfo}.
    *
@@ -369,7 +376,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     //  glFinish. Consider removing glFinish and requiring onTextureRendered to handle
     //  synchronization.
     GLES20.glFinish();
-    checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs);
+    checkNotNull(textureOutputListener)
+        .onTextureRendered(outputTexture, presentationTimeUs, this::releaseOutputFrame);
   }
 
   /**

libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java

@@ -109,6 +109,8 @@ import java.util.Queue;
    * <p>Throws {@link IllegalStateException} if {@code textureInfo} isn't in use.
    */
   public void freeTexture(GlTextureInfo textureInfo) {
+    // TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
+    // already.
     checkState(inUseTextures.contains(textureInfo));
     inUseTextures.remove(textureInfo);
     freeTextures.add(textureInfo);
@@ -120,6 +122,8 @@ import java.util.Queue;
    * <p>Throws {@link IllegalStateException} if there's no textures in use to free.
    */
   public void freeTexture() {
+    // TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
+    // already.
     checkState(!inUseTextures.isEmpty());
     GlTextureInfo texture = inUseTextures.remove();
     freeTextures.add(texture);
@@ -127,6 +131,8 @@ import java.util.Queue;
 
   /** Free all in-use textures. */
   public void freeAllTextures() {
+    // TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
+    // already.
     freeTextures.addAll(inUseTextures);
     inUseTextures.clear();
   }

libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java

@@ -288,11 +288,7 @@ public final class VideoFrameProcessorTestRunner {
                 boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
                 @Nullable
                 Surface outputSurface =
-                    bitmapReader.getSurface(
+                    bitmapReader.getSurface(width, height, useHighPrecisionColorComponents);
-                        width,
-                        height,
-                        useHighPrecisionColorComponents,
-                        checkNotNull(videoFrameProcessor)::releaseOutputFrame);
                 if (outputSurface != null) {
                   checkNotNull(videoFrameProcessor)
                       .setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
@@ -407,18 +403,10 @@ public final class VideoFrameProcessorTestRunner {
 
   /** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
   public interface BitmapReader {
-    /** Wraps a callback for {@link VideoFrameProcessor#releaseOutputFrame}. */
-    interface ReleaseOutputFrameListener {
-      void releaseOutputFrame(long releaseTimeUs);
-    }
 
     /** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
     @Nullable
-    Surface getSurface(
+    Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents);
-        int width,
-        int height,
-        boolean useHighPrecisionColorComponents,
-        ReleaseOutputFrameListener listener);
 
     /** Returns the output {@link Bitmap}. */
     Bitmap getBitmap();
@@ -438,11 +426,7 @@ public final class VideoFrameProcessorTestRunner {
     @Override
     @SuppressLint("WrongConstant")
     @Nullable
-    public Surface getSurface(
+    public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
-        int width,
-        int height,
-        boolean useHighPrecisionColorComponents,
-        ReleaseOutputFrameListener listener) {
       imageReader =
           ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
       return imageReader.getSurface();

libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java

@@ -507,14 +507,15 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
     DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
         new DefaultVideoFrameProcessor.Factory.Builder()
             .setTextureOutput(
-                (outputTexture, presentationTimeUs) ->
+                (outputTexture, presentationTimeUs, releaseOutputTextureCallback) ->
                     inputTextureIntoVideoFrameProcessor(
                         testId,
                         consumersBitmapReader,
                         colorInfo,
                         effects,
                         outputTexture,
-                        presentationTimeUs),
+                        presentationTimeUs,
+                        releaseOutputTextureCallback),
                 /* textureOutputCapacity= */ 1)
             .build();
     return new VideoFrameProcessorTestRunner.Builder()
@@ -533,7 +534,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
       ColorInfo colorInfo,
       List<Effect> effects,
       GlTextureInfo texture,
-      long presentationTimeUs)
+      long presentationTimeUs,
+      DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback)
       throws VideoFrameProcessingException {
     GlObjectsProvider contextSharingGlObjectsProvider =
         new DefaultGlObjectsProvider(GlUtil.getCurrentContext());
@@ -559,6 +561,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
     } catch (InterruptedException e) {
       throw new VideoFrameProcessingException(e);
     }
+    releaseOutputTextureCallback.release(presentationTimeUs);
   }
 
   private VideoFrameProcessorTestRunner.Builder getSurfaceInputFrameProcessorTestRunnerBuilder(
@@ -584,19 +587,12 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
   private static final class TextureBitmapReader implements BitmapReader {
     // TODO(b/239172735): This outputs an incorrect black output image on emulators.
     private boolean useHighPrecisionColorComponents;
-    private @MonotonicNonNull ReleaseOutputFrameListener releaseOutputFrameListener;
-
     private @MonotonicNonNull Bitmap outputBitmap;
 
     @Nullable
     @Override
-    public Surface getSurface(
+    public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
-        int width,
-        int height,
-        boolean useHighPrecisionColorComponents,
-        ReleaseOutputFrameListener releaseOutputFrameListener) {
       this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
-      this.releaseOutputFrameListener = releaseOutputFrameListener;
       return null;
     }
 
@@ -605,7 +601,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
       return checkStateNotNull(outputBitmap);
     }
 
-    public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs)
+    public void readBitmapFromTexture(
+        GlTextureInfo outputTexture,
+        long presentationTimeUs,
+        DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback)
         throws VideoFrameProcessingException {
       try {
         GlUtil.focusFramebufferUsingCurrentContext(
@@ -613,12 +612,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
         outputBitmap =
             createBitmapFromCurrentGlFrameBuffer(
                 outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
-        GlUtil.deleteTexture(outputTexture.texId);
-        GlUtil.deleteFbo(outputTexture.fboId);
       } catch (GlUtil.GlException e) {
         throw new VideoFrameProcessingException(e);
       }
-      checkNotNull(releaseOutputFrameListener).releaseOutputFrame(presentationTimeUs);
+      releaseOutputTextureCallback.release(presentationTimeUs);
     }
 
     private static Bitmap createBitmapFromCurrentGlFrameBuffer(