Move Single/MultiVideoGraph impl to effect

PiperOrigin-RevId: 569188658

libraries/common/src/main/java/androidx/media3/common/VideoGraph.java

@@ -24,6 +24,7 @@ import androidx.media3.common.util.UnstableApi;
 public interface VideoGraph {
 
   /** Listener for video frame processing events. */
+  @UnstableApi
   interface Listener {
     /**
      * Called when the output size changes.
@@ -58,6 +59,26 @@ public interface VideoGraph {
    */
   void initialize() throws VideoFrameProcessingException;
 
+  /**
+   * Registers a new input to the {@code VideoGraph}.
+   *
+   * <p>A underlying processing {@link VideoFrameProcessor} is created every time this method is
+   * called.
+   *
+   * <p>If the method throws, the caller must call {@link #release}.
+   *
+   * @return The id of the registered input, which can be used to get the underlying {@link
+   *     VideoFrameProcessor} via {@link #getProcessor(int)}.
+   */
+  int registerInput() throws VideoFrameProcessingException;
+
+  /**
+   * Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via
+   * {@link #registerInput()}. If the {@code inputId} is not {@linkplain #registerInput()
+   * registered} before, this method will throw an {@link IllegalStateException}.
+   */
+  VideoFrameProcessor getProcessor(int inputId);
+
   /**
    * Sets the output surface and supporting information.
    *

libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package androidx.media3.transformer;
+package androidx.media3.effect;
 
 import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID;
 import static androidx.media3.common.util.Assertions.checkNotNull;
@@ -45,12 +45,7 @@ import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.VideoGraph;
 import androidx.media3.common.util.GlUtil;
-import androidx.media3.effect.DefaultGlObjectsProvider;
-import androidx.media3.effect.DefaultVideoCompositor;
-import androidx.media3.effect.DefaultVideoFrameProcessor;
-import androidx.media3.effect.GlTextureProducer;
-import androidx.media3.effect.VideoCompositor;
-import androidx.media3.effect.VideoCompositorSettings;
+import androidx.media3.common.util.UnstableApi;
 import com.google.common.util.concurrent.MoreExecutors;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -61,32 +56,8 @@ import java.util.concurrent.ExecutorService;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /** A {@link VideoGraph} that handles multiple input streams. */
-/* package */ final class MultipleInputVideoGraph implements TransformerVideoGraph {
-
-  public static final class Factory implements TransformerVideoGraph.Factory {
-    @Override
-    public MultipleInputVideoGraph create(
-        Context context,
-        ColorInfo inputColorInfo,
-        ColorInfo outputColorInfo,
-        DebugViewProvider debugViewProvider,
-        Listener listener,
-        Executor listenerExecutor,
-        VideoCompositorSettings videoCompositorSettings,
-        List<Effect> compositionEffects,
-        long initialTimestampOffsetUs) {
-      return new MultipleInputVideoGraph(
-          context,
-          inputColorInfo,
-          outputColorInfo,
-          debugViewProvider,
-          listener,
-          listenerExecutor,
-          videoCompositorSettings,
-          compositionEffects,
-          initialTimestampOffsetUs);
-    }
-  }
+@UnstableApi
+public abstract class MultipleInputVideoGraph implements VideoGraph {
 
   private static final String SHARED_EXECUTOR_NAME = "Transformer:MultipleInputVideoGraph:Thread";
 
@@ -95,15 +66,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private static final int COMPOSITOR_TEXTURE_OUTPUT_CAPACITY = 1;
 
   private final Context context;
+
   private final ColorInfo inputColorInfo;
   private final ColorInfo outputColorInfo;
   private final GlObjectsProvider glObjectsProvider;
   private final DebugViewProvider debugViewProvider;
-  private final Listener listener;
+  private final VideoGraph.Listener listener;
   private final Executor listenerExecutor;
   private final VideoCompositorSettings videoCompositorSettings;
   private final List<Effect> compositionEffects;
-  private final List<VideoFrameProcessingWrapper> preProcessingWrappers;
+  private final List<VideoFrameProcessor> preProcessors;
 
   private final ExecutorService sharedExecutorService;
 
@@ -124,12 +96,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
   private volatile boolean hasProducedFrameWithTimestampZero;
 
-  private MultipleInputVideoGraph(
+  protected MultipleInputVideoGraph(
       Context context,
       ColorInfo inputColorInfo,
       ColorInfo outputColorInfo,
       DebugViewProvider debugViewProvider,
-      Listener listener,
+      VideoGraph.Listener listener,
       Executor listenerExecutor,
       VideoCompositorSettings videoCompositorSettings,
       List<Effect> compositionEffects,
@@ -144,7 +116,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.compositionEffects = new ArrayList<>(compositionEffects);
     this.initialTimestampOffsetUs = initialTimestampOffsetUs;
     lastRenderedPresentationTimeUs = C.TIME_UNSET;
-    preProcessingWrappers = new ArrayList<>();
+    preProcessors = new ArrayList<>();
     sharedExecutorService = newSingleThreadScheduledExecutor(SHARED_EXECUTOR_NAME);
     glObjectsProvider = new SingleContextGlObjectsProvider();
     // TODO - b/289986435: Support injecting VideoFrameProcessor.Factory.
@@ -165,7 +137,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   @Override
   public void initialize() throws VideoFrameProcessingException {
     checkState(
-        preProcessingWrappers.isEmpty()
+        preProcessors.isEmpty()
             && videoCompositor == null
             && compositionVideoFrameProcessor == null
             && !released);
@@ -243,14 +215,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   @Override
-  public GraphInput createInput() throws VideoFrameProcessingException {
+  public int registerInput() throws VideoFrameProcessingException {
     checkStateNotNull(videoCompositor);
 
     int videoCompositorInputId = videoCompositor.registerInputSource();
     // Creating a new VideoFrameProcessor for the input.
-    VideoFrameProcessingWrapper preProcessingVideoFrameProcessorWrapper =
-        new VideoFrameProcessingWrapper(
-            context,
+    VideoFrameProcessor preProcessor =
         videoFrameProcessorFactory
             .buildUpon()
             .setTextureOutput(
@@ -259,10 +229,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                     queuePreProcessingOutputToCompositor(
                         videoCompositorInputId, textureProducer, texture, presentationTimeUs),
                 PRE_COMPOSITOR_TEXTURE_OUTPUT_CAPACITY)
-                .build(),
+            .build()
+            .create(
+                context,
+                DebugViewProvider.NONE,
                 inputColorInfo,
                 outputColorInfo,
-            DebugViewProvider.NONE,
+                // Pre-processors render frames as soon as available, to VideoCompositor.
+                /* renderFramesAutomatically= */ true,
                 listenerExecutor,
                 new VideoFrameProcessor.Listener() {
                   // All of this listener's methods are called on the sharedExecutorService.
@@ -287,12 +261,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                   public void onEnded() {
                     onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId);
                   }
-            },
-            /* renderFramesAutomatically= */ true,
-            /* presentation= */ null,
-            initialTimestampOffsetUs);
-    preProcessingWrappers.add(preProcessingVideoFrameProcessorWrapper);
-    return preProcessingVideoFrameProcessorWrapper;
+                });
+    preProcessors.add(preProcessor);
+    return videoCompositorInputId;
+  }
+
+  @Override
+  public VideoFrameProcessor getProcessor(int inputId) {
+    checkState(inputId < preProcessors.size());
+    return preProcessors.get(inputId);
   }
 
   @Override
@@ -312,10 +289,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
 
     // Needs to release the frame processors before their internal executor services are released.
-    for (int i = 0; i < preProcessingWrappers.size(); i++) {
-      preProcessingWrappers.get(i).release();
+    for (int i = 0; i < preProcessors.size(); i++) {
+      preProcessors.get(i).release();
     }
-    preProcessingWrappers.clear();
+    preProcessors.clear();
 
     if (videoCompositor != null) {
       videoCompositor.release();
@@ -338,6 +315,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     released = true;
   }
 
+  protected ColorInfo getInputColorInfo() {
+    return inputColorInfo;
+  }
+
+  protected long getInitialTimestampOffsetUs() {
+    return initialTimestampOffsetUs;
+  }
+
   // This method is called on the sharedExecutorService.
   private void queuePreProcessingOutputToCompositor(
       int videoCompositorInputId,

libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package androidx.media3.transformer;
+package androidx.media3.effect;
 
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
@@ -30,13 +30,16 @@ import androidx.media3.common.SurfaceInfo;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.VideoGraph;
-import androidx.media3.effect.Presentation;
-import androidx.media3.effect.VideoCompositorSettings;
+import androidx.media3.common.util.UnstableApi;
 import java.util.List;
 import java.util.concurrent.Executor;
 
 /** A {@link VideoGraph} that handles one input stream. */
-/* package */ abstract class SingleInputVideoGraph implements VideoGraph {
+@UnstableApi
+public abstract class SingleInputVideoGraph implements VideoGraph {
+
+  /** The ID {@link #registerInput()} returns. */
+  public static final int SINGLE_INPUT_INDEX = 0;
 
   private final Context context;
   private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
@@ -46,10 +49,11 @@ import java.util.concurrent.Executor;
   private final DebugViewProvider debugViewProvider;
   private final Executor listenerExecutor;
   private final boolean renderFramesAutomatically;
+
   private final long initialTimestampOffsetUs;
   @Nullable private final Presentation presentation;
 
-  @Nullable private VideoFrameProcessingWrapper videoFrameProcessingWrapper;
+  @Nullable private VideoFrameProcessor videoFrameProcessor;
 
   private boolean released;
   private volatile boolean hasProducedFrameWithTimestampZero;
@@ -93,16 +97,21 @@ import java.util.concurrent.Executor;
    * <p>This method must be called at most once.
    */
   @Override
-  public void initialize() throws VideoFrameProcessingException {
-    checkStateNotNull(videoFrameProcessingWrapper == null && !released);
+  public void initialize() {
+    // Initialization is deferred to registerInput();
+  }
 
-    videoFrameProcessingWrapper =
-        new VideoFrameProcessingWrapper(
+  @Override
+  public int registerInput() throws VideoFrameProcessingException {
+    checkStateNotNull(videoFrameProcessor == null && !released);
+
+    videoFrameProcessor =
+        videoFrameProcessorFactory.create(
             context,
-            videoFrameProcessorFactory,
+            debugViewProvider,
             inputColorInfo,
             outputColorInfo,
-            debugViewProvider,
+            renderFramesAutomatically,
             listenerExecutor,
             new VideoFrameProcessor.Listener() {
               private long lastProcessedFramePresentationTimeUs;
@@ -136,15 +145,18 @@ import java.util.concurrent.Executor;
               public void onEnded() {
                 listener.onEnded(lastProcessedFramePresentationTimeUs);
               }
-            },
-            renderFramesAutomatically,
-            presentation,
-            initialTimestampOffsetUs);
+            });
+    return SINGLE_INPUT_INDEX;
+  }
+
+  @Override
+  public VideoFrameProcessor getProcessor(int inputId) {
+    return checkStateNotNull(videoFrameProcessor);
   }
 
   @Override
   public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
-    checkNotNull(videoFrameProcessingWrapper).setOutputSurfaceInfo(outputSurfaceInfo);
+    checkNotNull(videoFrameProcessor).setOutputSurfaceInfo(outputSurfaceInfo);
   }
 
   @Override
@@ -158,14 +170,23 @@ import java.util.concurrent.Executor;
       return;
     }
 
-    if (videoFrameProcessingWrapper != null) {
-      videoFrameProcessingWrapper.release();
-      videoFrameProcessingWrapper = null;
+    if (videoFrameProcessor != null) {
+      videoFrameProcessor.release();
+      videoFrameProcessor = null;
     }
     released = true;
   }
 
-  protected VideoFrameProcessingWrapper getVideoFrameProcessingWrapper() {
-    return checkNotNull(videoFrameProcessingWrapper);
+  protected ColorInfo getInputColorInfo() {
+    return inputColorInfo;
+  }
+
+  protected long getInitialTimestampOffsetUs() {
+    return initialTimestampOffsetUs;
+  }
+
+  @Nullable
+  protected Presentation getPresentation() {
+    return presentation;
   }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.transformer;
+
+import android.content.Context;
+import androidx.media3.common.ColorInfo;
+import androidx.media3.common.DebugViewProvider;
+import androidx.media3.common.Effect;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.VideoGraph;
+import androidx.media3.effect.MultipleInputVideoGraph;
+import androidx.media3.effect.VideoCompositorSettings;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+/**
+ * A {@link TransformerVideoGraph Transformer}-specific implementation of {@link
+ * MultipleInputVideoGraph}.
+ */
+/* package */ final class TransformerMultipleInputVideoGraph extends MultipleInputVideoGraph
+    implements TransformerVideoGraph {
+
+  /** A factory for creating {@link TransformerMultipleInputVideoGraph} instances. */
+  public static final class Factory implements TransformerVideoGraph.Factory {
+    @Override
+    public TransformerMultipleInputVideoGraph create(
+        Context context,
+        ColorInfo inputColorInfo,
+        ColorInfo outputColorInfo,
+        DebugViewProvider debugViewProvider,
+        VideoGraph.Listener listener,
+        Executor listenerExecutor,
+        VideoCompositorSettings videoCompositorSettings,
+        List<Effect> compositionEffects,
+        long initialTimestampOffsetUs) {
+      return new TransformerMultipleInputVideoGraph(
+          context,
+          inputColorInfo,
+          outputColorInfo,
+          debugViewProvider,
+          listener,
+          listenerExecutor,
+          videoCompositorSettings,
+          compositionEffects,
+          initialTimestampOffsetUs);
+    }
+  }
+
+  private TransformerMultipleInputVideoGraph(
+      Context context,
+      ColorInfo inputColorInfo,
+      ColorInfo outputColorInfo,
+      DebugViewProvider debugViewProvider,
+      Listener listener,
+      Executor listenerExecutor,
+      VideoCompositorSettings videoCompositorSettings,
+      List<Effect> compositionEffects,
+      long initialTimestampOffsetUs) {
+    super(
+        context,
+        inputColorInfo,
+        outputColorInfo,
+        debugViewProvider,
+        listener,
+        listenerExecutor,
+        videoCompositorSettings,
+        compositionEffects,
+        initialTimestampOffsetUs);
+  }
+
+  @Override
+  public GraphInput createInput() throws VideoFrameProcessingException {
+    int inputId = registerInput();
+    return new VideoFrameProcessingWrapper(
+        getProcessor(inputId),
+        getInputColorInfo(),
+        /* presentation= */ null,
+        getInitialTimestampOffsetUs());
+  }
+}

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java

@@ -16,21 +16,30 @@
 
 package androidx.media3.transformer;
 
+import static androidx.media3.common.util.Assertions.checkState;
+
 import android.content.Context;
 import androidx.annotation.Nullable;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.DebugViewProvider;
 import androidx.media3.common.Effect;
+import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.effect.Presentation;
+import androidx.media3.effect.SingleInputVideoGraph;
 import androidx.media3.effect.VideoCompositorSettings;
 import java.util.List;
 import java.util.concurrent.Executor;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
+/**
+ * A {@link TransformerVideoGraph Transformer}-specific implementation of {@link
+ * SingleInputVideoGraph}.
+ */
 /* package */ final class TransformerSingleInputVideoGraph extends SingleInputVideoGraph
     implements TransformerVideoGraph {
 
-  /** A factory for creating a {@link SingleInputVideoGraph}. */
+  /** A factory for creating {@link TransformerSingleInputVideoGraph} instances. */
   public static final class Factory implements TransformerVideoGraph.Factory {
 
     private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
@@ -40,7 +49,7 @@ import java.util.concurrent.Executor;
     }
 
     @Override
-    public TransformerVideoGraph create(
+    public TransformerSingleInputVideoGraph create(
         Context context,
         ColorInfo inputColorInfo,
         ColorInfo outputColorInfo,
@@ -72,6 +81,8 @@ import java.util.concurrent.Executor;
     }
   }
 
+  private @MonotonicNonNull VideoFrameProcessingWrapper videoFrameProcessingWrapper;
+
   private TransformerSingleInputVideoGraph(
       Context context,
       VideoFrameProcessor.Factory videoFrameProcessorFactory,
@@ -99,7 +110,15 @@ import java.util.concurrent.Executor;
   }
 
   @Override
-  public GraphInput createInput() {
-    return getVideoFrameProcessingWrapper();
+  public GraphInput createInput() throws VideoFrameProcessingException {
+    checkState(videoFrameProcessingWrapper == null);
+    int inputId = registerInput();
+    videoFrameProcessingWrapper =
+        new VideoFrameProcessingWrapper(
+            getProcessor(inputId),
+            getInputColorInfo(),
+            getPresentation(),
+            getInitialTimestampOffsetUs());
+    return videoFrameProcessingWrapper;
   }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java

@@ -21,26 +21,22 @@ import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
 import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID;
 import static androidx.media3.common.util.Assertions.checkNotNull;
 
-import android.content.Context;
 import android.graphics.Bitmap;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.media3.common.ColorInfo;
-import androidx.media3.common.DebugViewProvider;
 import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
 import androidx.media3.common.FrameInfo;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.OnInputFrameProcessedListener;
 import androidx.media3.common.SurfaceInfo;
-import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.util.Size;
 import androidx.media3.common.util.TimestampIterator;
 import androidx.media3.effect.Presentation;
 import com.google.common.collect.ImmutableList;
 import java.util.List;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicLong;
 
 /** A wrapper for {@link VideoFrameProcessor} that handles {@link GraphInput} events. */
@@ -52,31 +48,15 @@ import java.util.concurrent.atomic.AtomicLong;
   @Nullable final Presentation presentation;
 
   public VideoFrameProcessingWrapper(
-      Context context,
-      VideoFrameProcessor.Factory videoFrameProcessorFactory,
+      VideoFrameProcessor videoFrameProcessor,
       ColorInfo inputColorInfo,
-      ColorInfo outputColorInfo,
-      DebugViewProvider debugViewProvider,
-      Executor listenerExecutor,
-      VideoFrameProcessor.Listener listener,
-      boolean renderFramesAutomatically,
       @Nullable Presentation presentation,
-      long initialTimestampOffsetUs)
-      throws VideoFrameProcessingException {
+      long initialTimestampOffsetUs) {
+    this.videoFrameProcessor = videoFrameProcessor;
     this.mediaItemOffsetUs = new AtomicLong();
     this.inputColorInfo = inputColorInfo;
     this.initialTimestampOffsetUs = initialTimestampOffsetUs;
     this.presentation = presentation;
-
-    videoFrameProcessor =
-        videoFrameProcessorFactory.create(
-            context,
-            debugViewProvider,
-            inputColorInfo,
-            outputColorInfo,
-            renderFramesAutomatically,
-            listenerExecutor,
-            listener);
   }
 
   @Override

libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java

@@ -147,7 +147,7 @@ import org.checkerframework.dataflow.qual.Pure;
           new VideoGraphWrapper(
               context,
               hasMultipleInputs
-                  ? new MultipleInputVideoGraph.Factory()
+                  ? new TransformerMultipleInputVideoGraph.Factory()
                   : new TransformerSingleInputVideoGraph.Factory(videoFrameProcessorFactory),
               videoGraphInputColor,
               videoGraphOutputColor,
@@ -533,6 +533,16 @@ import org.checkerframework.dataflow.qual.Pure;
       videoGraph.initialize();
     }
 
+    @Override
+    public int registerInput() throws VideoFrameProcessingException {
+      return videoGraph.registerInput();
+    }
+
+    @Override
+    public VideoFrameProcessor getProcessor(int inputId) {
+      return videoGraph.getProcessor(inputId);
+    }
+
     @Override
     public GraphInput createInput() throws VideoFrameProcessingException {
       return videoGraph.createInput();