Loosen the requirement to register every input frame

VideoFrameProcessor requires every input frame to be registered before they are rendered to its input Surface. This CL adds the option to register the input frame only once. This is useful for camera inputs where not all input frames come with changing FrameInfo. PiperOrigin-RevId: 606294894 (cherry picked from commit 216f3fedb857d9a6e0bf846070784c1cc5b7e705)
2025-04-30 06:46:50 +08:00 · 2024-02-12 10:33:49 -08:00 · 2024-02-12 10:33:49 -08:00 · 5fd1f61c84
commit 5fd1f61c84
parent c4688fc557
3 changed files with 70 additions and 10 deletions
--- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
@ -91,10 +91,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      private @MonotonicNonNull GlObjectsProvider glObjectsProvider;
      private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener;
      private int textureOutputCapacity;
+      private boolean requireRegisteringAllInputFrames;

      /** Creates an instance. */
      public Builder() {
        enableColorTransfers = true;
+        requireRegisteringAllInputFrames = true;
      }

      private Builder(Factory factory) {
@ -116,6 +118,33 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
        return this;
      }

+      /**
+       * Sets whether {@link VideoFrameProcessor#registerInputFrame() registering} every input frame
+       * is required.
+       *
+       * <p>The default value is {@code true}, meaning that all frames input to the {@link
+       * VideoFrameProcessor}'s input {@link #getInputSurface Surface} must be {@linkplain
+       * #registerInputFrame() registered} before they are rendered. In this mode the input format
+       * change between input streams is handled frame-exactly. If {@code false}, {@link
+       * #registerInputFrame} can be called only once for each {@linkplain #registerInputStream
+       * registered input stream} before rendering the first frame to the input {@link
+       * #getInputSurface() Surface}. The same registered {@link FrameInfo} is repeated for the
+       * subsequent frames. To ensure the format change between input streams is applied on the
+       * right frame, the caller needs to {@linkplain #registerInputStream(int, List, FrameInfo)
+       * register} the new input stream strictly after rendering all frames from the previous input
+       * stream. This mode should be used in streams where users don't have direct control over
+       * rendering frames, like in a camera feed.
+       *
+       * <p>Regardless of the value set, {@link #registerInputStream(int, List, FrameInfo)} must be
+       * called for each input stream to specify the format for upcoming frames before calling
+       * {@link #registerInputFrame()}.
+       */
+      @CanIgnoreReturnValue
+      public Builder setRequireRegisteringAllInputFrames(boolean requireRegisteringAllInputFrames) {
+        this.requireRegisteringAllInputFrames = requireRegisteringAllInputFrames;
+        return this;
+      }
+
      /**
       * Sets the {@link GlObjectsProvider}.
       *
@ -178,6 +207,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      public DefaultVideoFrameProcessor.Factory build() {
        return new DefaultVideoFrameProcessor.Factory(
            enableColorTransfers,
+            /* repeatLastRegisteredFrame= */ !requireRegisteringAllInputFrames,
            glObjectsProvider == null ? new DefaultGlObjectsProvider() : glObjectsProvider,
            executorService,
            textureOutputListener,
@ -186,6 +216,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
    }

    private final boolean enableColorTransfers;
+    private final boolean repeatLastRegisteredFrame;
    private final GlObjectsProvider glObjectsProvider;
    @Nullable private final ExecutorService executorService;
    @Nullable private final GlTextureProducer.Listener textureOutputListener;
@ -193,11 +224,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {

    private Factory(
        boolean enableColorTransfers,
+        boolean repeatLastRegisteredFrame,
        GlObjectsProvider glObjectsProvider,
        @Nullable ExecutorService executorService,
        @Nullable GlTextureProducer.Listener textureOutputListener,
        int textureOutputCapacity) {
      this.enableColorTransfers = enableColorTransfers;
+      this.repeatLastRegisteredFrame = repeatLastRegisteredFrame;
      this.glObjectsProvider = glObjectsProvider;
      this.executorService = executorService;
      this.textureOutputListener = textureOutputListener;
@ -264,7 +297,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
                      listener,
                      glObjectsProvider,
                      textureOutputListener,
-                      textureOutputCapacity));
+                      textureOutputCapacity,
+                      repeatLastRegisteredFrame));

      try {
        return defaultVideoFrameProcessorFuture.get();
@ -623,10 +657,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      boolean renderFramesAutomatically,
      VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
      Executor videoFrameProcessorListenerExecutor,
-      VideoFrameProcessor.Listener listener,
+      Listener listener,
      GlObjectsProvider glObjectsProvider,
      @Nullable GlTextureProducer.Listener textureOutputListener,
-      int textureOutputCapacity)
+      int textureOutputCapacity,
+      boolean repeatLastRegisteredFrame)
      throws GlUtil.GlException, VideoFrameProcessingException {
    EGLDisplay eglDisplay = GlUtil.getDefaultEglDisplay();
    int[] configAttributes =
@ -662,7 +697,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
            videoFrameProcessingTaskExecutor,
            /* errorListenerExecutor= */ videoFrameProcessorListenerExecutor,
            /* samplingShaderProgramErrorListener= */ listener::onError,
-            enableColorTransfers);
+            enableColorTransfers,
+            repeatLastRegisteredFrame);

    FinalShaderProgramWrapper finalShaderProgramWrapper =
        new FinalShaderProgramWrapper(
--- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java
@ -68,6 +68,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
  private final Queue<FrameInfo> pendingFrames;
  private final ScheduledExecutorService forceEndOfStreamExecutorService;
  private final AtomicInteger externalShaderProgramInputCapacity;
+  private final boolean repeatLastRegisteredFrame;

  // Counts the frames that are registered before flush but are made available after flush.
  private int numberOfFramesToDropOnBecomingAvailable;
@ -76,6 +77,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

  // The frame that is sent downstream and is not done processing yet.
  @Nullable private FrameInfo currentFrame;
+  @Nullable private FrameInfo lastRegisteredFrame;

  @Nullable private Future<?> forceSignalEndOfStreamFuture;
  private boolean shouldRejectIncomingFrames;
@ -85,16 +87,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   *
   * @param glObjectsProvider The {@link GlObjectsProvider} for using EGL and GLES.
   * @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor}.
+   * @param repeatLastRegisteredFrame If {@code true}, the last {@linkplain
+   *     #registerInputFrame(FrameInfo) registered frame} is repeated for subsequent input textures
+   *     made available on the {@linkplain #getInputSurface() input Surface}. This means the user
+   *     can call {@link #registerInputFrame(FrameInfo)} only once. Else, every input frame needs to
+   *     be {@linkplain #registerInputFrame(FrameInfo) registered} before they are made available on
+   *     the {@linkplain #getInputSurface() input Surface}.
   * @throws VideoFrameProcessingException If a problem occurs while creating the external texture.
   */
  // The onFrameAvailableListener will not be invoked until the constructor returns.
  @SuppressWarnings("nullness:method.invocation.invalid")
  public ExternalTextureManager(
      GlObjectsProvider glObjectsProvider,
-      VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor)
+      VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
+      boolean repeatLastRegisteredFrame)
      throws VideoFrameProcessingException {
    super(videoFrameProcessingTaskExecutor);
    this.glObjectsProvider = glObjectsProvider;
+    this.repeatLastRegisteredFrame = repeatLastRegisteredFrame;
    try {
      externalTexId = GlUtil.createExternalTexture();
    } catch (GlUtil.GlException e) {
@ -190,7 +200,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   */
  @Override
  public void registerInputFrame(FrameInfo frame) {
-    pendingFrames.add(frame);
+    lastRegisteredFrame = frame;
+    if (!repeatLastRegisteredFrame) {
+      pendingFrames.add(frame);
+    }
    videoFrameProcessingTaskExecutor.submit(() -> shouldRejectIncomingFrames = false);
  }

@ -198,6 +211,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   * Returns the number of {@linkplain #registerInputFrame(FrameInfo) registered} frames that have
   * not been sent to the downstream {@link ExternalShaderProgram} yet.
   *
+   * <p>This method always returns 0 if {@code ExternalTextureManager} is built with {@code
+   * repeatLastRegisteredFrame} equal to {@code true}.
+   *
   * <p>Can be called on any thread.
   */
  @Override
@ -236,6 +252,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    externalShaderProgramInputCapacity.set(0);
    currentFrame = null;
    pendingFrames.clear();
+    lastRegisteredFrame = null;
    maybeExecuteAfterFlushTask();
  }

@ -299,7 +316,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

    surfaceTexture.updateTexImage();
    availableFrameCount--;
-    FrameInfo currentFrame = pendingFrames.element();
+
+    FrameInfo currentFrame =
+        repeatLastRegisteredFrame ? checkNotNull(lastRegisteredFrame) : pendingFrames.element();
    this.currentFrame = currentFrame;

    externalShaderProgramInputCapacity.decrementAndGet();
@ -319,7 +338,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                currentFrame.width,
                currentFrame.height),
            presentationTimeUs);
-    checkStateNotNull(pendingFrames.remove());
+    if (!repeatLastRegisteredFrame) {
+      checkStateNotNull(pendingFrames.remove());
+    }
    DebugTraceUtil.logEvent(DebugTraceUtil.EVENT_VFP_QUEUE_FRAME, presentationTimeUs);
    // If the queued frame is the last frame, end of stream will be signaled onInputFrameProcessed.
  }
--- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java
@ -64,7 +64,8 @@ import org.checkerframework.checker.nullness.qual.Nullable;
      VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
      Executor errorListenerExecutor,
      GlShaderProgram.ErrorListener samplingShaderProgramErrorListener,
-      boolean enableColorTransfers)
+      boolean enableColorTransfers,
+      boolean repeatLastRegisteredFrame)
      throws VideoFrameProcessingException {
    this.context = context;
    this.outputColorInfo = outputColorInfo;
@ -78,7 +79,9 @@ import org.checkerframework.checker.nullness.qual.Nullable;
    // TODO(b/274109008): Investigate lazy instantiating the texture managers.
    inputs.put(
        INPUT_TYPE_SURFACE,
-        new Input(new ExternalTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor)));
+        new Input(
+            new ExternalTextureManager(
+                glObjectsProvider, videoFrameProcessingTaskExecutor, repeatLastRegisteredFrame)));
    inputs.put(
        INPUT_TYPE_BITMAP,
        new Input(new BitmapTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor)));