Effects: Output to texture without surface in VFP.

Allow the VideoFrameProcessor to output to a texture without an output surface.

Tested by updating texture output tests to no longer output to a surface.

PiperOrigin-RevId: 527244605

libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java

@@ -123,7 +123,8 @@ public interface VideoFrameProcessor {
     void onOutputSizeChanged(int width, int height);
 
     /**
-     * Called when an output frame with the given {@code presentationTimeUs} becomes available.
+     * Called when an output frame with the given {@code presentationTimeUs} becomes available for
+     * rendering.
      *
      * @param presentationTimeUs The presentation time of the frame, in microseconds.
      */

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java

@@ -114,6 +114,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
        *
        * <p>If set, the {@link VideoFrameProcessor} will output to an OpenGL texture, accessible via
        * {@link TextureOutputListener#onTextureRendered}. Otherwise, no texture will be rendered to.
+       *
+       * <p>If an {@linkplain #setOutputSurfaceInfo output surface} is set, the texture output will
+       * be be adjusted as needed, to match the output surface's output.
        */
       @VisibleForTesting
       @CanIgnoreReturnValue

libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java

@@ -48,12 +48,13 @@ import com.google.common.collect.ImmutableList;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.Executor;
-import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
- * Wrapper around a {@link DefaultShaderProgram} that writes to the provided output surface and if
+ * Wrapper around a {@link DefaultShaderProgram} that renders to the provided output surface or
- * provided, the optional debug surface view or output texture.
+ * texture.
+ *
+ * <p>Also renders to a debug surface, if provided.
  *
  * <p>The wrapped {@link DefaultShaderProgram} applies the {@link GlMatrixTransformation} and {@link
  * RgbMatrix} instances passed to the constructor, followed by any transformations needed to convert
@@ -92,15 +93,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private InputListener inputListener;
   private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
   @Nullable private SurfaceView debugSurfaceView;
-  private @MonotonicNonNull GlTextureInfo outputTexture;
+  @Nullable private GlTextureInfo outputTexture;
   private boolean frameProcessingStarted;
 
-  private volatile boolean outputChanged;
+  private volatile boolean outputSurfaceInfoChanged;
 
   @GuardedBy("this")
   @Nullable
   private SurfaceInfo outputSurfaceInfo;
 
+  /** Wraps the {@link Surface} in {@link #outputSurfaceInfo}. */
   @GuardedBy("this")
   @Nullable
   private EGLSurface outputEglSurface;
@@ -240,6 +242,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
     try {
       if (outputTexture != null) {
+        GlTextureInfo outputTexture = checkNotNull(this.outputTexture);
         GlUtil.deleteTexture(outputTexture.texId);
         GlUtil.deleteFbo(outputTexture.fboId);
       }
@@ -270,7 +273,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       }
       this.outputEglSurface = null;
     }
-    outputChanged =
+    outputSurfaceInfoChanged =
         this.outputSurfaceInfo == null
             || outputSurfaceInfo == null
             || this.outputSurfaceInfo.width != outputSurfaceInfo.width
@@ -279,14 +282,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.outputSurfaceInfo = outputSurfaceInfo;
   }
 
-  private void renderFrame(
+  private synchronized void renderFrame(
       GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) {
     try {
-      maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
+      if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
-      if (textureOutputListener != null && defaultShaderProgram != null) {
+          || !ensureConfigured(inputTexture.width, inputTexture.height)) {
+        inputListener.onInputFrameProcessed(inputTexture);
+        return; // Drop frames when requested, or there is no output surface.
+      }
+      if (outputSurfaceInfo != null) {
+        renderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
+      }
+      if (textureOutputListener != null) {
         renderFrameToOutputTexture(inputTexture, presentationTimeUs);
       }
-
     } catch (VideoFrameProcessingException | GlUtil.GlException e) {
       videoFrameProcessorListenerExecutor.execute(
           () ->
@@ -300,17 +309,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     inputListener.onInputFrameProcessed(inputTexture);
   }
 
-  private synchronized void maybeRenderFrameToOutputSurface(
+  private synchronized void renderFrameToOutputSurface(
       GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs)
       throws VideoFrameProcessingException, GlUtil.GlException {
-    if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
+    EGLSurface outputEglSurface = checkNotNull(this.outputEglSurface);
-        || !ensureConfigured(inputTexture.width, inputTexture.height)) {
+    SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo);
-      return; // Drop frames when requested, or there is no output surface.
+    DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram);
-    }
-
-    EGLSurface outputEglSurface = this.outputEglSurface;
-    SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
-    DefaultShaderProgram defaultShaderProgram = this.defaultShaderProgram;
 
     GlUtil.focusEglSurface(
         eglDisplay,
@@ -332,7 +336,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
   private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs)
       throws GlUtil.GlException, VideoFrameProcessingException {
-    checkNotNull(outputTexture);
+    GlTextureInfo outputTexture = checkNotNull(this.outputTexture);
     GlUtil.focusFramebufferUsingCurrentContext(
         outputTexture.fboId, outputTexture.width, outputTexture.height);
     GlUtil.clearOutputFrame();
@@ -345,12 +349,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    *
    * <p>Returns {@code false} if {@code outputSurfaceInfo} is unset.
    */
-  @EnsuresNonNullIf(
-      expression = {"outputSurfaceInfo", "outputEglSurface", "defaultShaderProgram"},
-      result = true)
   private synchronized boolean ensureConfigured(int inputWidth, int inputHeight)
       throws VideoFrameProcessingException, GlUtil.GlException {
-
+    // Clear extra or outdated resources.
     boolean inputSizeChanged =
         this.inputWidth != inputWidth
             || this.inputHeight != inputHeight
@@ -370,20 +371,31 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                     outputSizeBeforeSurfaceTransformation.getHeight()));
       }
     }
+    checkNotNull(outputSizeBeforeSurfaceTransformation);
 
     if (outputSurfaceInfo == null) {
+      GlUtil.destroyEglSurface(eglDisplay, outputEglSurface);
+      outputEglSurface = null;
+    }
+    if (outputSurfaceInfo == null && textureOutputListener == null) {
       if (defaultShaderProgram != null) {
         defaultShaderProgram.release();
         defaultShaderProgram = null;
       }
-      GlUtil.destroyEglSurface(eglDisplay, outputEglSurface);
-      outputEglSurface = null;
       return false;
     }
 
-    SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
+    int outputWidth =
-    @Nullable EGLSurface outputEglSurface = this.outputEglSurface;
+        outputSurfaceInfo == null
-    if (outputEglSurface == null) {
+            ? outputSizeBeforeSurfaceTransformation.getWidth()
+            : outputSurfaceInfo.width;
+    int outputHeight =
+        outputSurfaceInfo == null
+            ? outputSizeBeforeSurfaceTransformation.getHeight()
+            : outputSurfaceInfo.height;
+
+    // Allocate or update resources.
+    if (outputSurfaceInfo != null && outputEglSurface == null) {
       outputEglSurface =
           glObjectsProvider.createEglSurface(
               eglDisplay,
@@ -391,67 +403,58 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               outputColorInfo.colorTransfer,
               // Frames are only released automatically when outputting to an encoder.
               /* isEncoderInputSurface= */ releaseFramesAutomatically);
-
-      @Nullable
-      SurfaceView debugSurfaceView =
-          debugViewProvider.getDebugPreviewSurfaceView(
-              outputSurfaceInfo.width, outputSurfaceInfo.height);
-      if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
-        debugSurfaceViewWrapper =
-            new SurfaceViewWrapper(
-                eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer);
-      }
-      this.debugSurfaceView = debugSurfaceView;
     }
 
-    if (defaultShaderProgram != null && (outputChanged || inputSizeChanged)) {
+    @Nullable
+    SurfaceView debugSurfaceView =
+        debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
+    if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
+      debugSurfaceViewWrapper =
+          new SurfaceViewWrapper(
+              eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer);
+    }
+    this.debugSurfaceView = debugSurfaceView;
+
+    if (textureOutputListener != null) {
+      int outputTexId =
+          GlUtil.createTexture(
+              outputWidth,
+              outputHeight,
+              /* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
+      outputTexture =
+          glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
+    }
+
+    if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) {
       defaultShaderProgram.release();
       defaultShaderProgram = null;
-      outputChanged = false;
+      outputSurfaceInfoChanged = false;
     }
     if (defaultShaderProgram == null) {
-      DefaultShaderProgram defaultShaderProgram =
+      defaultShaderProgram =
-          createDefaultShaderProgramForOutputSurface(outputSurfaceInfo);
+          createDefaultShaderProgram(
-      if (textureOutputListener != null) {
+              outputSurfaceInfo == null ? 0 : outputSurfaceInfo.orientationDegrees,
-        configureOutputTexture(
+              outputWidth,
-            checkNotNull(outputSizeBeforeSurfaceTransformation).getWidth(),
+              outputHeight);
-            checkNotNull(outputSizeBeforeSurfaceTransformation).getHeight());
-      }
-      this.defaultShaderProgram = defaultShaderProgram;
     }
 
-    this.outputSurfaceInfo = outputSurfaceInfo;
-    this.outputEglSurface = outputEglSurface;
     return true;
   }
 
-  private void configureOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
+  private synchronized DefaultShaderProgram createDefaultShaderProgram(
-    if (outputTexture != null) {
+      int outputOrientationDegrees, int outputWidth, int outputHeight)
-      GlUtil.deleteTexture(outputTexture.texId);
+      throws VideoFrameProcessingException {
-      GlUtil.deleteFbo(outputTexture.fboId);
-    }
-    int outputTexId =
-        GlUtil.createTexture(
-            outputWidth,
-            outputHeight,
-            /* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
-    outputTexture =
-        glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
-  }
-
-  private DefaultShaderProgram createDefaultShaderProgramForOutputSurface(
-      SurfaceInfo outputSurfaceInfo) throws VideoFrameProcessingException {
     ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
         new ImmutableList.Builder<GlMatrixTransformation>().addAll(matrixTransformations);
-    if (outputSurfaceInfo.orientationDegrees != 0) {
+    if (outputOrientationDegrees != 0) {
       matrixTransformationListBuilder.add(
           new ScaleAndRotateTransformation.Builder()
-              .setRotationDegrees(outputSurfaceInfo.orientationDegrees)
+              .setRotationDegrees(outputOrientationDegrees)
               .build());
     }
     matrixTransformationListBuilder.add(
         Presentation.createForWidthAndHeight(
-            outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
+            outputWidth, outputHeight, Presentation.LAYOUT_SCALE_TO_FIT));
 
     DefaultShaderProgram defaultShaderProgram;
     ImmutableList<GlMatrixTransformation> expandedMatrixTransformations =
@@ -488,8 +491,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
     defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix);
     Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
-    checkState(outputSize.getWidth() == outputSurfaceInfo.width);
+    if (outputSurfaceInfo != null) {
-    checkState(outputSize.getHeight() == outputSurfaceInfo.height);
+      SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo);
+      checkState(outputSize.getWidth() == outputSurfaceInfo.width);
+      checkState(outputSize.getHeight() == outputSurfaceInfo.height);
+    }
     return defaultShaderProgram;
   }
 

libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java

@@ -281,14 +281,17 @@ public final class VideoFrameProcessorTestRunner {
             new VideoFrameProcessor.Listener() {
               @Override
               public void onOutputSizeChanged(int width, int height) {
+                @Nullable
                 Surface outputSurface =
                     bitmapReader.getSurface(
                         width,
                         height,
                         /* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(
                             outputColorInfo));
-                checkNotNull(videoFrameProcessor)
+                if (outputSurface != null) {
-                    .setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
+                  checkNotNull(videoFrameProcessor)
+                      .setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
+                }
               }
 
               @Override
@@ -368,7 +371,8 @@ public final class VideoFrameProcessorTestRunner {
   /** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
   public interface BitmapReader {
 
-    /** Returns the {@link VideoFrameProcessor} output {@link Surface}. */
+    /** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
+    @Nullable
     Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents);
 
     /** Returns the output {@link Bitmap}. */
@@ -388,6 +392,7 @@ public final class VideoFrameProcessorTestRunner {
 
     @Override
     @SuppressLint("WrongConstant")
+    @Nullable
     public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
       imageReader =
           ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);

libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java

@@ -31,7 +31,6 @@ import static com.google.common.truth.Truth.assertThat;
 
 import android.content.Context;
 import android.graphics.Bitmap;
-import android.graphics.SurfaceTexture;
 import android.view.Surface;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.Format;
@@ -52,6 +51,7 @@ import androidx.media3.transformer.EncoderUtil;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.common.collect.ImmutableList;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.junit.After;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -306,16 +306,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
     private @MonotonicNonNull Bitmap outputBitmap;
 
     @Override
+    @Nullable
     public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
       this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
-      int texId;
+      return null;
-      try {
-        texId = GlUtil.createExternalTexture();
-      } catch (GlUtil.GlException e) {
-        throw new RuntimeException(e);
-      }
-      SurfaceTexture surfaceTexture = new SurfaceTexture(texId);
-      return new Surface(surfaceTexture);
     }
 
     @Override