Abstract the interface of DefaultVideoFrameProcessor's input.

PiperOrigin-RevId: 526642898

libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java

@@ -208,11 +208,8 @@ public interface VideoFrameProcessor {
   void registerInputFrame();
 
   /**
-   * Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
+   * returns the number of input frames that have been made available to the {@code
-   * but not processed off the {@linkplain #getInputSurface() input surface} yet.
+   * VideoFrameProcessor} but have not been processed yet.
-   *
-   * <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
-   * Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
    *
    * <p>Can be called on any thread.
    */

libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java

@@ -21,10 +21,12 @@ import static java.lang.Math.round;
 import android.graphics.Bitmap;
 import android.opengl.GLES20;
 import android.opengl.GLUtils;
+import android.view.Surface;
+import androidx.annotation.Nullable;
 import androidx.media3.common.C;
+import androidx.media3.common.FrameInfo;
 import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.VideoFrameProcessingException;
-import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.UnstableApi;
 import java.util.Queue;
@@ -38,7 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
  * <p>Public methods in this class can be called from any thread.
  */
 @UnstableApi
-/* package */ final class BitmapTextureManager implements GlShaderProgram.InputListener {
+/* package */ final class BitmapTextureManager implements InputHandler {
   private final GlShaderProgram shaderProgram;
   private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
   // The queue holds all bitmaps with one or more frames pending to be sent downstream.
@@ -77,22 +79,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         });
   }
 
-  /**
+  @Override
-   * Provides an input {@link Bitmap} to put into the video frames.
+  public void setDefaultBufferSize(int width, int height) {
-   *
+    throw new UnsupportedOperationException();
-   * @see VideoFrameProcessor#queueInputBitmap
+  }
-   */
+
+  @Override
   public void queueInputBitmap(
       Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
     videoFrameProcessingTaskExecutor.submit(
         () -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr));
   }
 
-  /**
+  @Override
-   * Signals the end of the input.
+  public Surface getInputSurface() {
-   *
+    throw new UnsupportedOperationException();
-   * @see VideoFrameProcessor#signalEndOfInput()
+  }
-   */
+
+  @Override
+  public void registerInputFrame(FrameInfo frameInfo) {
+    // Do nothing.
+  }
+
+  @Override
+  public int getPendingFrameCount() {
+    // Always treat all queued bitmaps as immediately processed.
+    return 0;
+  }
+
+  @Override
   public void signalEndOfInput() {
     videoFrameProcessingTaskExecutor.submit(
         () -> {
@@ -101,11 +116,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         });
   }
 
-  /**
+  @Override
-   * Releases all resources.
+  public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) {
-   *
+    // Do nothing.
-   * @see VideoFrameProcessor#release()
+  }
-   */
+
+  @Override
   public void release() {
     videoFrameProcessingTaskExecutor.submit(
         () -> {

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java

@@ -17,7 +17,6 @@ package androidx.media3.effect;
 
 import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
 import static androidx.media3.common.util.Assertions.checkArgument;
-import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
 import static com.google.common.collect.Iterables.getLast;
@@ -247,8 +246,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
   private final EGLDisplay eglDisplay;
   private final EGLContext eglContext;
   private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
-  private @MonotonicNonNull BitmapTextureManager inputBitmapTextureManager;
+  private final InputHandler inputHandler;
-  private @MonotonicNonNull ExternalTextureManager inputExternalTextureManager;
   private final boolean releaseFramesAutomatically;
   private final FinalShaderProgramWrapper finalShaderProgramWrapper;
   private final ImmutableList<GlShaderProgram> allShaderPrograms;
@@ -278,20 +276,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
     switch (inputType) {
       case VideoFrameProcessor.INPUT_TYPE_SURFACE:
         checkState(inputShaderProgram instanceof ExternalShaderProgram);
-        inputExternalTextureManager =
+        inputHandler =
             new ExternalTextureManager(
                 (ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor);
-        inputShaderProgram.setInputListener(inputExternalTextureManager);
         break;
       case VideoFrameProcessor.INPUT_TYPE_BITMAP:
-        inputBitmapTextureManager =
+        inputHandler =
             new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor);
-        inputShaderProgram.setInputListener(inputBitmapTextureManager);
         break;
       case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through
       default:
         throw new VideoFrameProcessingException("Input type not supported yet");
     }
+    inputShaderProgram.setInputListener(inputHandler);
 
     finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(shaderPrograms);
     allShaderPrograms = shaderPrograms;
@@ -319,18 +316,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
    * @param height The default height for input buffers, in pixels.
    */
   public void setInputDefaultBufferSize(int width, int height) {
-    checkNotNull(inputExternalTextureManager).setDefaultBufferSize(width, height);
+    inputHandler.setDefaultBufferSize(width, height);
   }
 
   @Override
   public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
-    checkNotNull(inputBitmapTextureManager)
+    inputHandler.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
-        .queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
   }
 
   @Override
   public Surface getInputSurface() {
-    return checkNotNull(inputExternalTextureManager).getInputSurface();
+    return inputHandler.getInputSurface();
   }
 
   @Override
@@ -344,12 +340,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
     checkStateNotNull(
         nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
 
-    checkNotNull(inputExternalTextureManager).registerInputFrame(nextInputFrameInfo);
+    inputHandler.registerInputFrame(nextInputFrameInfo);
   }
 
   @Override
   public int getPendingInputFrameCount() {
-    return checkNotNull(inputExternalTextureManager).getPendingFrameCount();
+    return inputHandler.getPendingFrameCount();
   }
 
   @Override
@@ -370,12 +366,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
   public void signalEndOfInput() {
     checkState(!inputStreamEnded);
     inputStreamEnded = true;
-    if (inputBitmapTextureManager != null) {
+    videoFrameProcessingTaskExecutor.submit(inputHandler::signalEndOfInput);
-      videoFrameProcessingTaskExecutor.submit(inputBitmapTextureManager::signalEndOfInput);
-    }
-    if (inputExternalTextureManager != null) {
-      videoFrameProcessingTaskExecutor.submit(inputExternalTextureManager::signalEndOfInput);
-    }
   }
 
   @Override
@@ -383,10 +374,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
     try {
       videoFrameProcessingTaskExecutor.flush();
       CountDownLatch latch = new CountDownLatch(1);
-      checkNotNull(inputExternalTextureManager).setOnFlushCompleteListener(latch::countDown);
+      inputHandler.setOnFlushCompleteListener(latch::countDown);
       videoFrameProcessingTaskExecutor.submit(finalShaderProgramWrapper::flush);
       latch.await();
-      inputExternalTextureManager.setOnFlushCompleteListener(null);
+      inputHandler.setOnFlushCompleteListener(null);
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
     }
@@ -401,12 +392,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
       Thread.currentThread().interrupt();
       throw new IllegalStateException(unexpected);
     }
-    if (inputExternalTextureManager != null) {
+    inputHandler.release();
-      inputExternalTextureManager.release();
-    }
-    if (inputBitmapTextureManager != null) {
-      inputBitmapTextureManager.release();
-    }
   }
 
   /**

libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java

@@ -17,6 +17,7 @@ package androidx.media3.effect;
 
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
 
+import android.graphics.Bitmap;
 import android.graphics.SurfaceTexture;
 import android.view.Surface;
 import androidx.annotation.Nullable;
@@ -24,7 +25,6 @@ import androidx.media3.common.C;
 import androidx.media3.common.FrameInfo;
 import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.VideoFrameProcessingException;
-import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.effect.GlShaderProgram.InputListener;
 import java.util.Queue;
@@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger;
  * Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
  * {@link ExternalShaderProgram} for consumption.
  */
-/* package */ final class ExternalTextureManager implements InputListener {
+/* package */ final class ExternalTextureManager implements InputHandler {
 
   private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
   private final ExternalShaderProgram externalShaderProgram;
@@ -105,12 +105,18 @@ import java.util.concurrent.atomic.AtomicInteger;
     surface = new Surface(surfaceTexture);
   }
 
-  /** See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. */
+  @Override
   public void setDefaultBufferSize(int width, int height) {
     surfaceTexture.setDefaultBufferSize(width, height);
   }
 
-  /** Returns the {@linkplain Surface input surface} that wraps the external texture. */
+  @Override
+  public void queueInputBitmap(
+      Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
   public Surface getInputSurface() {
     return surface;
   }
@@ -137,7 +143,7 @@ import java.util.concurrent.atomic.AtomicInteger;
         });
   }
 
-  /** Sets the task to run on completing flushing, or {@code null} to clear any task. */
+  @Override
   public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) {
     onFlushCompleteTask = task;
   }
@@ -154,6 +160,7 @@ import java.util.concurrent.atomic.AtomicInteger;
    * <p>Can be called on any thread. The caller must ensure that frames are registered in the
    * correct order.
    */
+  @Override
   public void registerInputFrame(FrameInfo frame) {
     pendingFrames.add(frame);
   }
@@ -164,15 +171,12 @@ import java.util.concurrent.atomic.AtomicInteger;
    *
    * <p>Can be called on any thread.
    */
+  @Override
   public int getPendingFrameCount() {
     return pendingFrames.size();
   }
 
-  /**
+  @Override
-   * Signals the end of the input.
-   *
-   * @see VideoFrameProcessor#signalEndOfInput()
-   */
   public void signalEndOfInput() {
     videoFrameProcessingTaskExecutor.submit(
         () -> {
@@ -183,11 +187,7 @@ import java.util.concurrent.atomic.AtomicInteger;
         });
   }
 
-  /**
+  @Override
-   * Releases all resources.
-   *
-   * @see VideoFrameProcessor#release()
-   */
   public void release() {
     surfaceTexture.release();
     surface.release();

libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package androidx.media3.effect;
+
+import android.graphics.Bitmap;
+import android.graphics.SurfaceTexture;
+import android.view.Surface;
+import androidx.annotation.Nullable;
+import androidx.media3.common.FrameInfo;
+import androidx.media3.common.VideoFrameProcessor;
+
+/** A component that handles {@code DefaultVideoFrameProcessor}'s input. */
+/* package */ interface InputHandler extends GlShaderProgram.InputListener {
+
+  /**
+   * See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}.
+   *
+   * <p>Only works when the input is received on a {@link SurfaceTexture}.
+   */
+  void setDefaultBufferSize(int width, int height);
+
+  /**
+   * Provides an input {@link Bitmap} to put into the video frames.
+   *
+   * @see VideoFrameProcessor#queueInputBitmap
+   */
+  void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr);
+
+  /**
+   * See {@link VideoFrameProcessor#getInputSurface}.
+   *
+   * <p>Only works when the input is received on a {@link SurfaceTexture}.
+   */
+  Surface getInputSurface();
+
+  /** Informs the {@code InputHandler} that a frame will be queued. */
+  void registerInputFrame(FrameInfo frameInfo);
+
+  /** See {@link VideoFrameProcessor#getPendingInputFrameCount}. */
+  int getPendingFrameCount();
+
+  /**
+   * Signals the end of the input.
+   *
+   * @see VideoFrameProcessor#signalEndOfInput()
+   */
+  void signalEndOfInput();
+
+  /** Sets the task to run on completing flushing, or {@code null} to clear any task. */
+  void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task);
+
+  /**
+   * Releases all resources.
+   *
+   * @see VideoFrameProcessor#release()
+   */
+  void release();
+}