Fix indeterminate z-order of EditedMediaItemSequences by passing sequenceIndex when registeringInput

libraries/common/src/main/java/androidx/media3/common/VideoGraph.java

@@ -75,14 +75,16 @@ public interface VideoGraph {
    *
    * <p>If the method throws, the caller must call {@link #release}.
    *
+   * @param sequenceIndex The sequence index of the input which can aid ordering of the inputs.
+   *
    * @return The id of the registered input, which can be used to get the underlying {@link
    *     VideoFrameProcessor} via {@link #getProcessor(int)}.
    */
-  int registerInput() throws VideoFrameProcessingException;
+  int registerInput(int sequenceIndex) throws VideoFrameProcessingException;
 
   /**
    * Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via
-   * {@link #registerInput()}. If the {@code inputId} is not {@linkplain #registerInput()
+   * {@link #registerInput(int)}. If the {@code inputId} is not {@linkplain #registerInput(int)
    * registered} before, this method will throw an {@link IllegalStateException}.
    */
   VideoFrameProcessor getProcessor(int inputId);

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java

@@ -26,6 +26,8 @@ import android.opengl.EGLContext;
 import android.opengl.EGLDisplay;
 import android.opengl.EGLSurface;
 import android.opengl.GLES20;
+import android.util.SparseArray;
+
 import androidx.annotation.GuardedBy;
 import androidx.annotation.IntRange;
 import androidx.annotation.Nullable;
@@ -41,16 +43,18 @@ import androidx.media3.common.util.LongArrayQueue;
 import androidx.media3.common.util.Size;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
+
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
 import java.io.IOException;
 import java.util.ArrayDeque;
-import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.ExecutorService;
-import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
  * A basic {@link VideoCompositor} implementation that takes in frames from input sources' streams
@@ -88,7 +92,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
   private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
 
   @GuardedBy("this")
-  private final List<InputSource> inputSources;
+  private final SparseArray<InputSource> inputSources;
 
   @GuardedBy("this")
   private boolean allInputsEnded; // Whether all inputSources have signaled end of input.
@@ -124,7 +128,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
     this.settings = settings;
     this.compositorGlProgram = new CompositorGlProgram(context);
 
-    inputSources = new ArrayList<>();
+    inputSources = new SparseArray<>();
     outputTexturePool =
         new TexturePool(/* useHighPrecisionColorComponents= */ false, textureOutputCapacity);
     outputTextureTimestamps = new LongArrayQueue(textureOutputCapacity);
@@ -142,9 +146,9 @@ public final class DefaultVideoCompositor implements VideoCompositor {
   }
 
   @Override
-  public synchronized int registerInputSource() {
+  public synchronized int registerInputSource(int sequenceId) {
-    inputSources.add(new InputSource());
+    inputSources.put(sequenceId, new InputSource());
-    return inputSources.size() - 1;
+    return sequenceId;
   }
 
   @Override
@@ -152,7 +156,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
     inputSources.get(inputId).isInputEnded = true;
     boolean allInputsEnded = true;
     for (int i = 0; i < inputSources.size(); i++) {
-      if (!inputSources.get(i).isInputEnded) {
+      if (!inputSources.get(inputSources.keyAt(i)).isInputEnded) {
         allInputsEnded = false;
         break;
       }
@@ -229,7 +233,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
       if (i == PRIMARY_INPUT_ID) {
         continue;
       }
-      releaseExcessFramesInSecondaryStream(inputSources.get(i));
+      releaseExcessFramesInSecondaryStream(inputSources.get(inputSources.keyAt(i)));
     }
   }
 
@@ -334,7 +338,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
       return ImmutableList.of();
     }
     for (int inputId = 0; inputId < inputSources.size(); inputId++) {
-      if (inputSources.get(inputId).frameInfos.isEmpty()) {
+      if (inputSources.get(inputSources.keyAt(inputId)).frameInfos.isEmpty()) {
         return ImmutableList.of();
       }
     }
@@ -353,7 +357,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
       //   2. Two or more frames, and at least one frame has timestamp greater than the target
       //      timestamp.
       // The smaller timestamp is taken if two timestamps have the same distance from the primary.
-      InputSource secondaryInputSource = inputSources.get(inputId);
+      InputSource secondaryInputSource = inputSources.get(inputSources.keyAt(inputId));
       if (secondaryInputSource.frameInfos.size() == 1 && !secondaryInputSource.isInputEnded) {
         return ImmutableList.of();
       }

libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java

@@ -46,6 +46,7 @@ import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.VideoGraph;
 import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.NullableType;
 import androidx.media3.common.util.UnstableApi;
 import com.google.common.util.concurrent.MoreExecutors;
 import java.util.ArrayDeque;
@@ -75,7 +76,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
   private final Executor listenerExecutor;
   private final VideoCompositorSettings videoCompositorSettings;
   private final List<Effect> compositionEffects;
-  private final List<VideoFrameProcessor> preProcessors;
+  private final List<@NullableType VideoFrameProcessor> preProcessors;
 
   private final ExecutorService sharedExecutorService;
 
@@ -211,10 +212,11 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
   }
 
   @Override
-  public int registerInput() throws VideoFrameProcessingException {
+  public int registerInput(int forceId) throws VideoFrameProcessingException {
     checkStateNotNull(videoCompositor);
 
-    int videoCompositorInputId = videoCompositor.registerInputSource();
+    int videoCompositorInputId;
+    videoCompositorInputId = videoCompositor.registerInputSource(forceId);
     // Creating a new VideoFrameProcessor for the input.
     VideoFrameProcessor preProcessor =
         videoFrameProcessorFactory
@@ -257,7 +259,12 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
                     onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId);
                   }
                 });
-    preProcessors.add(preProcessor);
+
+    while (preProcessors.size() <= videoCompositorInputId) {
+      //noinspection DataFlowIssue
+      preProcessors.add(null);
+    }
+    preProcessors.set(videoCompositorInputId, preProcessor);
     return videoCompositorInputId;
   }
 

libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java

@@ -38,7 +38,7 @@ import java.util.concurrent.Executor;
 @UnstableApi
 public abstract class SingleInputVideoGraph implements VideoGraph {
 
-  /** The ID {@link #registerInput()} returns. */
+  /** The ID {@link #registerInput(int)} returns. */
   public static final int SINGLE_INPUT_INDEX = 0;
 
   private final Context context;
@@ -99,7 +99,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph {
   }
 
   @Override
-  public int registerInput() throws VideoFrameProcessingException {
+  public int registerInput(int sequenceIndex) throws VideoFrameProcessingException {
     checkStateNotNull(videoFrameProcessor == null && !released);
 
     videoFrameProcessor =

libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java

@@ -48,9 +48,11 @@ public interface VideoCompositor extends GlTextureProducer {
   /**
    * Registers a new input source, and returns a unique {@code inputId} corresponding to this
    * source, to be used in {@link #queueInputTexture}.
+   *
+   * @param sequenceId The sequence ID of the input source, which is can be used to determine the
+   * order of the input sources.
    */
-  int registerInputSource();
+  int registerInputSource(int sequenceId);
-
   /**
    * Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}.
    *

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java

@@ -550,6 +550,8 @@ public final class CompositingVideoSinkProvider
       //  reduces decoder timeouts, and consider restoring.
       videoFrameProcessorMaxPendingFrameCount =
           Util.getMaxPendingFramesCountForMediaCodecDecoders(context);
+      int videoGraphInputId = videoGraph.registerInput(0);
+      videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId);
 
       videoEffects = new ArrayList<>();
       finalBufferPresentationTimeUs = C.TIME_UNSET;

libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java

@@ -856,7 +856,7 @@ public final class DefaultVideoCompositorPixelTest {
         VideoCompositor videoCompositor,
         @Nullable ExecutorService executorService,
         GlObjectsProvider glObjectsProvider) {
-      int inputId = videoCompositor.registerInputSource();
+      int inputId = videoCompositor.registerInputSource(0);
       DefaultVideoFrameProcessor.Factory.Builder defaultVideoFrameProcessorFactoryBuilder =
           new DefaultVideoFrameProcessor.Factory.Builder()
               .setGlObjectsProvider(glObjectsProvider)

libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java

@@ -97,7 +97,7 @@ import org.checkerframework.dataflow.qual.Pure;
   }
 
   @Override
-  public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format)
+  public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex)
       throws ExportException {
     if (!returnedFirstInput) {
       // First input initialized in constructor because output AudioFormat is needed.

libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java

@@ -129,7 +129,7 @@ import java.util.concurrent.atomic.AtomicLong;
   }
 
   @Override
-  public GraphInput getInput(EditedMediaItem item, Format format) {
+  public GraphInput getInput(EditedMediaItem item, Format format, int sequenceIndex) {
     return this;
   }
 

libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java

@@ -64,10 +64,10 @@ import java.util.List;
    *
    * @param editedMediaItem The initial {@link EditedMediaItem} of the input.
    * @param format The initial {@link Format} of the input.
+   * @param sequenceIndex The sequence index of the input.
    * @throws ExportException If an error occurs getting the input.
    */
-  public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format)
+  public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException;
-      throws ExportException;
 
   /**
    * Processes the input data and returns whether it may be possible to process more data by calling

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java

@@ -637,7 +637,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         }
 
         GraphInput sampleExporterInput =
-            sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat);
+          sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat, sequenceIndex);
         OnMediaItemChangedListener onMediaItemChangedListener =
             (editedMediaItem, durationUs, decodedFormat, isLast) -> {
               onMediaItemChanged(trackType, durationUs, isLast);

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java

@@ -22,6 +22,7 @@ import androidx.media3.common.DebugViewProvider;
 import androidx.media3.common.Effect;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoGraph;
+import androidx.media3.common.util.Log;
 import androidx.media3.effect.MultipleInputVideoGraph;
 import androidx.media3.effect.VideoCompositorSettings;
 import java.util.List;
@@ -79,8 +80,8 @@ import java.util.concurrent.Executor;
   }
 
   @Override
-  public GraphInput createInput() throws VideoFrameProcessingException {
+  public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException {
-    int inputId = registerInput();
+    int inputId = registerInput(sequenceIndex);
     return new VideoFrameProcessingWrapper(
         getProcessor(inputId), /* presentation= */ null, getInitialTimestampOffsetUs());
   }

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java

@@ -106,12 +106,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   @Override
-  public GraphInput createInput() throws VideoFrameProcessingException {
+  public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException {
     checkState(videoFrameProcessingWrapper == null);
-    int inputId = registerInput();
+    int inputId = registerInput(sequenceIndex);
     videoFrameProcessingWrapper =
             new VideoFrameProcessingWrapper(
-            getProcessor(inputId), getPresentation(), getInitialTimestampOffsetUs());
+                    getProcessor(inputId),
+                    getInputColorInfo(),
+                    getPresentation(),
+                    getInitialTimestampOffsetUs());
     return videoFrameProcessingWrapper;
   }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java

@@ -69,6 +69,8 @@ import java.util.concurrent.Executor;
    * <p>This method must called exactly once for every input stream.
    *
    * <p>If the method throws any {@link Exception}, the caller must call {@link #release}.
+   *
+   * @param sequenceIndex The sequence index of the input, which can aid ordering of the inputs.
    */
-  GraphInput createInput() throws VideoFrameProcessingException;
+  GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException;
 }

libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java

@@ -50,6 +50,7 @@ import androidx.media3.common.util.Util;
 import androidx.media3.decoder.DecoderInputBuffer;
 import androidx.media3.effect.DebugTraceUtil;
 import androidx.media3.effect.VideoCompositorSettings;
+import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.MoreExecutors;
 import java.nio.ByteBuffer;
@@ -58,6 +59,8 @@ import java.util.Objects;
 import org.checkerframework.checker.initialization.qual.Initialized;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.checkerframework.dataflow.qual.Pure;
+import java.nio.ByteBuffer;
+import java.util.List;
 
 /** Processes, encodes and muxes raw video frames. */
 /* package */ final class VideoSampleExporter extends SampleExporter {
@@ -153,10 +156,9 @@ import org.checkerframework.dataflow.qual.Pure;
   }
 
   @Override
-  public GraphInput getInput(EditedMediaItem editedMediaItem, Format format)
+  public GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException {
-      throws ExportException {
     try {
-      return videoGraph.createInput();
+      return videoGraph.createInput(sequenceIndex);
     } catch (VideoFrameProcessingException e) {
       throw ExportException.createForVideoFrameProcessingException(e);
     }
@@ -540,8 +542,8 @@ import org.checkerframework.dataflow.qual.Pure;
     }
 
     @Override
-    public int registerInput() throws VideoFrameProcessingException {
+    public int registerInput(int sequenceIndex) throws VideoFrameProcessingException {
-      return videoGraph.registerInput();
+      return videoGraph.registerInput(sequenceIndex);
     }
 
     @Override
@@ -550,8 +552,8 @@ import org.checkerframework.dataflow.qual.Pure;
     }
 
     @Override
-    public GraphInput createInput() throws VideoFrameProcessingException {
+    public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException {
-      return videoGraph.createInput();
+      return videoGraph.createInput(sequenceIndex);
     }
 
     @Override