diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 496e16f7d4..3221caa6be 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -75,14 +75,16 @@ public interface VideoGraph { * *

If the method throws, the caller must call {@link #release}. * + * @param sequenceIndex The sequence index of the input which can aid ordering of the inputs. + * * @return The id of the registered input, which can be used to get the underlying {@link * VideoFrameProcessor} via {@link #getProcessor(int)}. */ - int registerInput() throws VideoFrameProcessingException; + int registerInput(int sequenceIndex) throws VideoFrameProcessingException; /** * Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via - * {@link #registerInput()}. If the {@code inputId} is not {@linkplain #registerInput() + * {@link #registerInput(int)}. If the {@code inputId} is not {@linkplain #registerInput(int) * registered} before, this method will throw an {@link IllegalStateException}. */ VideoFrameProcessor getProcessor(int inputId); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index 591e7d8e2a..70f0f76a71 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -26,6 +26,8 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLSurface; import android.opengl.GLES20; +import android.util.SparseArray; + import androidx.annotation.GuardedBy; import androidx.annotation.IntRange; import androidx.annotation.Nullable; @@ -41,16 +43,18 @@ import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; + import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; + +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + import java.io.IOException; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Queue; import java.util.concurrent.ExecutorService; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A basic {@link VideoCompositor} implementation that takes in frames from input sources' streams @@ -88,7 +92,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; @GuardedBy("this") - private final List inputSources; + private final SparseArray inputSources; @GuardedBy("this") private boolean allInputsEnded; // Whether all inputSources have signaled end of input. @@ -124,7 +128,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { this.settings = settings; this.compositorGlProgram = new CompositorGlProgram(context); - inputSources = new ArrayList<>(); + inputSources = new SparseArray<>(); outputTexturePool = new TexturePool(/* useHighPrecisionColorComponents= */ false, textureOutputCapacity); outputTextureTimestamps = new LongArrayQueue(textureOutputCapacity); @@ -142,9 +146,9 @@ public final class DefaultVideoCompositor implements VideoCompositor { } @Override - public synchronized int registerInputSource() { - inputSources.add(new InputSource()); - return inputSources.size() - 1; + public synchronized int registerInputSource(int sequenceId) { + inputSources.put(sequenceId, new InputSource()); + return sequenceId; } @Override @@ -152,7 +156,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { inputSources.get(inputId).isInputEnded = true; boolean allInputsEnded = true; for (int i = 0; i < inputSources.size(); i++) { - if (!inputSources.get(i).isInputEnded) { + if (!inputSources.get(inputSources.keyAt(i)).isInputEnded) { allInputsEnded = false; break; } @@ -229,7 +233,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { if (i == PRIMARY_INPUT_ID) { continue; } - releaseExcessFramesInSecondaryStream(inputSources.get(i)); + releaseExcessFramesInSecondaryStream(inputSources.get(inputSources.keyAt(i))); } } @@ -334,7 +338,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { return ImmutableList.of(); } for (int inputId = 0; inputId < inputSources.size(); inputId++) { - if (inputSources.get(inputId).frameInfos.isEmpty()) { + if (inputSources.get(inputSources.keyAt(inputId)).frameInfos.isEmpty()) { return ImmutableList.of(); } } @@ -353,7 +357,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { // 2. Two or more frames, and at least one frame has timestamp greater than the target // timestamp. // The smaller timestamp is taken if two timestamps have the same distance from the primary. - InputSource secondaryInputSource = inputSources.get(inputId); + InputSource secondaryInputSource = inputSources.get(inputSources.keyAt(inputId)); if (secondaryInputSource.frameInfos.size() == 1 && !secondaryInputSource.isInputEnded) { return ImmutableList.of(); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index e503ca997d..6dac599b0b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -46,6 +46,7 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.NullableType; import androidx.media3.common.util.UnstableApi; import com.google.common.util.concurrent.MoreExecutors; import java.util.ArrayDeque; @@ -75,7 +76,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { private final Executor listenerExecutor; private final VideoCompositorSettings videoCompositorSettings; private final List compositionEffects; - private final List preProcessors; + private final List<@NullableType VideoFrameProcessor> preProcessors; private final ExecutorService sharedExecutorService; @@ -211,10 +212,11 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { } @Override - public int registerInput() throws VideoFrameProcessingException { + public int registerInput(int forceId) throws VideoFrameProcessingException { checkStateNotNull(videoCompositor); - int videoCompositorInputId = videoCompositor.registerInputSource(); + int videoCompositorInputId; + videoCompositorInputId = videoCompositor.registerInputSource(forceId); // Creating a new VideoFrameProcessor for the input. VideoFrameProcessor preProcessor = videoFrameProcessorFactory @@ -257,7 +259,12 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId); } }); - preProcessors.add(preProcessor); + + while (preProcessors.size() <= videoCompositorInputId) { + //noinspection DataFlowIssue + preProcessors.add(null); + } + preProcessors.set(videoCompositorInputId, preProcessor); return videoCompositorInputId; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 33a4536481..17f1293562 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -38,7 +38,7 @@ import java.util.concurrent.Executor; @UnstableApi public abstract class SingleInputVideoGraph implements VideoGraph { - /** The ID {@link #registerInput()} returns. */ + /** The ID {@link #registerInput(int)} returns. */ public static final int SINGLE_INPUT_INDEX = 0; private final Context context; @@ -99,7 +99,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph { } @Override - public int registerInput() throws VideoFrameProcessingException { + public int registerInput(int sequenceIndex) throws VideoFrameProcessingException { checkStateNotNull(videoFrameProcessor == null && !released); videoFrameProcessor = diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 5ec3e68047..13ed072bb7 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -48,9 +48,11 @@ public interface VideoCompositor extends GlTextureProducer { /** * Registers a new input source, and returns a unique {@code inputId} corresponding to this * source, to be used in {@link #queueInputTexture}. + * + * @param sequenceId The sequence ID of the input source, which is can be used to determine the + * order of the input sources. */ - int registerInputSource(); - + int registerInputSource(int sequenceId); /** * Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 7ba81a4389..f9da1c3789 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -550,6 +550,8 @@ public final class CompositingVideoSinkProvider // reduces decoder timeouts, and consider restoring. videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); + int videoGraphInputId = videoGraph.registerInput(0); + videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId); videoEffects = new ArrayList<>(); finalBufferPresentationTimeUs = C.TIME_UNSET; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java index 2d584fdf30..bf85f8a757 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java @@ -856,7 +856,7 @@ public final class DefaultVideoCompositorPixelTest { VideoCompositor videoCompositor, @Nullable ExecutorService executorService, GlObjectsProvider glObjectsProvider) { - int inputId = videoCompositor.registerInputSource(); + int inputId = videoCompositor.registerInputSource(0); DefaultVideoFrameProcessor.Factory.Builder defaultVideoFrameProcessorFactoryBuilder = new DefaultVideoFrameProcessor.Factory.Builder() .setGlObjectsProvider(glObjectsProvider) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java index 6d434bd74c..951f8165f8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java @@ -97,7 +97,7 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format) + public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException { if (!returnedFirstInput) { // First input initialized in constructor because output AudioFormat is needed. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java index 03097e1119..30e1f81207 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java @@ -129,7 +129,7 @@ import java.util.concurrent.atomic.AtomicLong; } @Override - public GraphInput getInput(EditedMediaItem item, Format format) { + public GraphInput getInput(EditedMediaItem item, Format format, int sequenceIndex) { return this; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java index b430639161..276caef733 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java @@ -64,10 +64,10 @@ import java.util.List; * * @param editedMediaItem The initial {@link EditedMediaItem} of the input. * @param format The initial {@link Format} of the input. + * @param sequenceIndex The sequence index of the input. * @throws ExportException If an error occurs getting the input. */ - public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format) - throws ExportException; + public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException; /** * Processes the input data and returns whether it may be possible to process more data by calling diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 1adf5c1de6..d40df2a4ca 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -637,7 +637,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } GraphInput sampleExporterInput = - sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat); + sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat, sequenceIndex); OnMediaItemChangedListener onMediaItemChangedListener = (editedMediaItem, durationUs, decodedFormat, isLast) -> { onMediaItemChanged(trackType, durationUs, isLast); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java index 27a122bdda..a334379efb 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java @@ -22,6 +22,7 @@ import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoGraph; +import androidx.media3.common.util.Log; import androidx.media3.effect.MultipleInputVideoGraph; import androidx.media3.effect.VideoCompositorSettings; import java.util.List; @@ -79,8 +80,8 @@ import java.util.concurrent.Executor; } @Override - public GraphInput createInput() throws VideoFrameProcessingException { - int inputId = registerInput(); + public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { + int inputId = registerInput(sequenceIndex); return new VideoFrameProcessingWrapper( getProcessor(inputId), /* presentation= */ null, getInitialTimestampOffsetUs()); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java index 5118e7fe47..588ccc0601 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -106,12 +106,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public GraphInput createInput() throws VideoFrameProcessingException { + public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { checkState(videoFrameProcessingWrapper == null); - int inputId = registerInput(); + int inputId = registerInput(sequenceIndex); videoFrameProcessingWrapper = - new VideoFrameProcessingWrapper( - getProcessor(inputId), getPresentation(), getInitialTimestampOffsetUs()); + new VideoFrameProcessingWrapper( + getProcessor(inputId), + getInputColorInfo(), + getPresentation(), + getInitialTimestampOffsetUs()); return videoFrameProcessingWrapper; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java index eb9e836e45..5883e8ab6f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java @@ -69,6 +69,8 @@ import java.util.concurrent.Executor; *

This method must called exactly once for every input stream. * *

If the method throws any {@link Exception}, the caller must call {@link #release}. + * + * @param sequenceIndex The sequence index of the input, which can aid ordering of the inputs. */ - GraphInput createInput() throws VideoFrameProcessingException; + GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 929d4727db..0556bdf65f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -50,6 +50,7 @@ import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.effect.DebugTraceUtil; import androidx.media3.effect.VideoCompositorSettings; +import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; @@ -58,6 +59,8 @@ import java.util.Objects; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; +import java.nio.ByteBuffer; +import java.util.List; /** Processes, encodes and muxes raw video frames. */ /* package */ final class VideoSampleExporter extends SampleExporter { @@ -153,10 +156,9 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public GraphInput getInput(EditedMediaItem editedMediaItem, Format format) - throws ExportException { + public GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException { try { - return videoGraph.createInput(); + return videoGraph.createInput(sequenceIndex); } catch (VideoFrameProcessingException e) { throw ExportException.createForVideoFrameProcessingException(e); } @@ -540,8 +542,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public int registerInput() throws VideoFrameProcessingException { - return videoGraph.registerInput(); + public int registerInput(int sequenceIndex) throws VideoFrameProcessingException { + return videoGraph.registerInput(sequenceIndex); } @Override @@ -550,8 +552,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public GraphInput createInput() throws VideoFrameProcessingException { - return videoGraph.createInput(); + public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { + return videoGraph.createInput(sequenceIndex); } @Override