Texture Input: Add GLSyncToken to the OnInputFrameProcessedListener

PiperOrigin-RevId: 543867944

libraries/common/src/main/java/androidx/media3/common/OnInputFrameProcessedListener.java

@@ -21,6 +21,13 @@ import androidx.media3.common.util.UnstableApi;
 @UnstableApi
 public interface OnInputFrameProcessedListener {
 
-  /** Called when the given input frame has been processed. */
-  void onInputFrameProcessed(int textureId) throws VideoFrameProcessingException;
+  /**
+   * Called when the given input frame has been processed.
+   *
+   * @param textureId The identifier of the processed texture.
+   * @param syncObject A GL sync object that has been inserted into the GL command stream after the
+   *     last write of texture. Value is 0 if and only if the {@code GLES30#glFenceSync} failed or
+   *     the EGL context version is less than openGL 3.0.
+   */
+  void onInputFrameProcessed(int textureId, long syncObject) throws VideoFrameProcessingException;
 }

libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.common.util;
 
+import static android.opengl.EGL14.EGL_CONTEXT_CLIENT_VERSION;
 import static android.opengl.GLU.gluErrorString;
 import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkState;
@@ -370,6 +371,38 @@ public final class GlUtil {
     return eglSurface;
   }
 
+  /**
+   * Returns a newly created sync object and inserts it into the GL command stream.
+   *
+   * <p>Returns 0 if the operation failed or the {@link EGLContext} version is less than 3.0.
+   */
+  public static long createGlSyncFence() throws GlException {
+    int[] currentEglContextVersion = new int[1];
+    EGL14.eglQueryContext(
+        EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY),
+        EGL14.eglGetCurrentContext(),
+        EGL_CONTEXT_CLIENT_VERSION,
+        currentEglContextVersion,
+        /* offset= */ 0);
+    if (currentEglContextVersion[0] < 3) {
+      return 0;
+    }
+    long syncObject = GLES30.glFenceSync(GLES30.GL_SYNC_GPU_COMMANDS_COMPLETE, /* flags= */ 0);
+    checkGlError();
+    // Due to specifics of OpenGL, it might happen that the fence creation command is not yet
+    // sent into the GPU command queue, which can cause other threads to wait infinitely if
+    // the glSyncWait/glClientSyncWait command went into the GPU earlier. Hence, we have to
+    // call glFlush to ensure that glFenceSync is inside of the GPU command queue.
+    GLES20.glFlush();
+    return syncObject;
+  }
+
+  /** Releases the underlying native object. */
+  public static void deleteSyncObject(long syncObject) throws GlException {
+    GLES30.glDeleteSync(syncObject);
+    checkGlError();
+  }
+
   /** Gets the current {@link EGLContext context}. */
   public static EGLContext getCurrentContext() {
     return EGL14.eglGetCurrentContext();
@@ -697,7 +730,7 @@ public final class GlUtil {
     public static EGLContext createEglContext(
         EGLContext sharedContext, EGLDisplay eglDisplay, int version, int[] configAttributes)
         throws GlException {
-      int[] contextAttributes = {EGL14.EGL_CONTEXT_CLIENT_VERSION, version, EGL14.EGL_NONE};
+      int[] contextAttributes = {EGL_CONTEXT_CLIENT_VERSION, version, EGL14.EGL_NONE};
       EGLContext eglContext =
           EGL14.eglCreateContext(
               eglDisplay,

libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java

@@ -24,6 +24,8 @@ import androidx.media3.common.FrameInfo;
 import androidx.media3.common.GlObjectsProvider;
 import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.OnInputFrameProcessedListener;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlUtil;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
@@ -38,6 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
   private @MonotonicNonNull OnInputFrameProcessedListener frameProcessedListener;
   private @MonotonicNonNull FrameInfo inputFrameInfo;
+  private long glSyncObject;
 
   /**
    * Creates a new instance.
@@ -65,7 +68,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   @Override
   public void onInputFrameProcessed(GlTextureInfo inputTexture) {
     videoFrameProcessingTaskExecutor.submit(
-        () -> checkNotNull(frameProcessedListener).onInputFrameProcessed(inputTexture.getTexId()));
+        () -> {
+          glSyncObject = GlUtil.createGlSyncFence();
+          checkNotNull(frameProcessedListener)
+              .onInputFrameProcessed(inputTexture.getTexId(), glSyncObject);
+        });
   }
 
   @Override
@@ -116,7 +123,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   @Override
-  public void release() {
-    // Do nothing.
+  public void release() throws VideoFrameProcessingException {
+    try {
+      GlUtil.deleteSyncObject(glSyncObject);
+    } catch (GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e);
+    }
   }
 }

libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java

@@ -23,6 +23,7 @@ import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.media3.common.FrameInfo;
 import androidx.media3.common.OnInputFrameProcessedListener;
+import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 
 /** Handles {@code DefaultVideoFrameProcessor}'s input. */
@@ -106,5 +107,5 @@ import androidx.media3.common.VideoFrameProcessor;
    *
    * @see VideoFrameProcessor#release()
    */
-  void release();
+  void release() throws VideoFrameProcessingException;
 }

libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java

@@ -354,7 +354,7 @@ public final class VideoFrameProcessorTestRunner {
             .setPixelWidthHeightRatio(pixelWidthHeightRatio)
             .build());
     videoFrameProcessor.setOnInputFrameProcessedListener(
-        texId -> {
+        (texId, unused) -> {
           try {
             GlUtil.deleteTexture(texId);
           } catch (GlUtil.GlException e) {

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -503,7 +503,7 @@ public class TransformerEndToEndTest {
         EditedMediaItem editedMediaItem, Looper looper, AssetLoader.Listener listener) {
       Format format = new Format.Builder().setWidth(width).setHeight(height).build();
       OnInputFrameProcessedListener frameProcessedListener =
-          texId -> {
+          (texId, unused) -> {
             try {
               GlUtil.deleteTexture(texId);
             } catch (GlUtil.GlException e) {

libraries/transformer/src/test/java/androidx/media3/transformer/TextureAssetLoaderTest.java

@@ -130,7 +130,7 @@ public class TextureAssetLoaderTest {
             .setDurationUs(C.MICROS_PER_SECOND)
             .build();
     Format format = new Format.Builder().setWidth(10).setHeight(10).build();
-    OnInputFrameProcessedListener frameProcessedListener = unused -> {};
+    OnInputFrameProcessedListener frameProcessedListener = (unused, unused2) -> {};
     return new TextureAssetLoader(editedMediaItem, listener, format, frameProcessedListener);
   }