From 177f1f33d0dc67c2a7b65466a8d16cf926006113 Mon Sep 17 00:00:00 2001 From: Patrik Aradi Date: Wed, 31 Jan 2024 09:55:40 +0800 Subject: [PATCH 1/8] Fix indeterminate z-order of EditedMediaItemSequences by passing sequenceIndex when registeringInput --- .../androidx/media3/common/VideoGraph.java | 6 +++-- .../media3/effect/DefaultVideoCompositor.java | 26 +++++++++++-------- .../effect/MultipleInputVideoGraph.java | 15 ++++++++--- .../media3/effect/SingleInputVideoGraph.java | 4 +-- .../media3/effect/VideoCompositor.java | 6 +++-- .../video/CompositingVideoSinkProvider.java | 2 ++ .../DefaultVideoCompositorPixelTest.java | 2 +- .../transformer/AudioSampleExporter.java | 2 +- .../transformer/EncodedSampleExporter.java | 2 +- .../media3/transformer/SampleExporter.java | 4 +-- .../transformer/TransformerInternal.java | 2 +- .../TransformerMultipleInputVideoGraph.java | 5 ++-- .../TransformerSingleInputVideoGraph.java | 11 +++++--- .../transformer/TransformerVideoGraph.java | 4 ++- .../transformer/VideoSampleExporter.java | 16 +++++++----- 15 files changed, 66 insertions(+), 41 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 496e16f7d4..3221caa6be 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -75,14 +75,16 @@ public interface VideoGraph { * *

If the method throws, the caller must call {@link #release}. * + * @param sequenceIndex The sequence index of the input which can aid ordering of the inputs. + * * @return The id of the registered input, which can be used to get the underlying {@link * VideoFrameProcessor} via {@link #getProcessor(int)}. */ - int registerInput() throws VideoFrameProcessingException; + int registerInput(int sequenceIndex) throws VideoFrameProcessingException; /** * Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via - * {@link #registerInput()}. If the {@code inputId} is not {@linkplain #registerInput() + * {@link #registerInput(int)}. If the {@code inputId} is not {@linkplain #registerInput(int) * registered} before, this method will throw an {@link IllegalStateException}. */ VideoFrameProcessor getProcessor(int inputId); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index 591e7d8e2a..70f0f76a71 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -26,6 +26,8 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLSurface; import android.opengl.GLES20; +import android.util.SparseArray; + import androidx.annotation.GuardedBy; import androidx.annotation.IntRange; import androidx.annotation.Nullable; @@ -41,16 +43,18 @@ import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; + import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; + +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + import java.io.IOException; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Queue; import java.util.concurrent.ExecutorService; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A basic {@link VideoCompositor} implementation that takes in frames from input sources' streams @@ -88,7 +92,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; @GuardedBy("this") - private final List inputSources; + private final SparseArray inputSources; @GuardedBy("this") private boolean allInputsEnded; // Whether all inputSources have signaled end of input. @@ -124,7 +128,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { this.settings = settings; this.compositorGlProgram = new CompositorGlProgram(context); - inputSources = new ArrayList<>(); + inputSources = new SparseArray<>(); outputTexturePool = new TexturePool(/* useHighPrecisionColorComponents= */ false, textureOutputCapacity); outputTextureTimestamps = new LongArrayQueue(textureOutputCapacity); @@ -142,9 +146,9 @@ public final class DefaultVideoCompositor implements VideoCompositor { } @Override - public synchronized int registerInputSource() { - inputSources.add(new InputSource()); - return inputSources.size() - 1; + public synchronized int registerInputSource(int sequenceId) { + inputSources.put(sequenceId, new InputSource()); + return sequenceId; } @Override @@ -152,7 +156,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { inputSources.get(inputId).isInputEnded = true; boolean allInputsEnded = true; for (int i = 0; i < inputSources.size(); i++) { - if (!inputSources.get(i).isInputEnded) { + if (!inputSources.get(inputSources.keyAt(i)).isInputEnded) { allInputsEnded = false; break; } @@ -229,7 +233,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { if (i == PRIMARY_INPUT_ID) { continue; } - releaseExcessFramesInSecondaryStream(inputSources.get(i)); + releaseExcessFramesInSecondaryStream(inputSources.get(inputSources.keyAt(i))); } } @@ -334,7 +338,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { return ImmutableList.of(); } for (int inputId = 0; inputId < inputSources.size(); inputId++) { - if (inputSources.get(inputId).frameInfos.isEmpty()) { + if (inputSources.get(inputSources.keyAt(inputId)).frameInfos.isEmpty()) { return ImmutableList.of(); } } @@ -353,7 +357,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { // 2. Two or more frames, and at least one frame has timestamp greater than the target // timestamp. // The smaller timestamp is taken if two timestamps have the same distance from the primary. - InputSource secondaryInputSource = inputSources.get(inputId); + InputSource secondaryInputSource = inputSources.get(inputSources.keyAt(inputId)); if (secondaryInputSource.frameInfos.size() == 1 && !secondaryInputSource.isInputEnded) { return ImmutableList.of(); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index e503ca997d..6dac599b0b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -46,6 +46,7 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.NullableType; import androidx.media3.common.util.UnstableApi; import com.google.common.util.concurrent.MoreExecutors; import java.util.ArrayDeque; @@ -75,7 +76,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { private final Executor listenerExecutor; private final VideoCompositorSettings videoCompositorSettings; private final List compositionEffects; - private final List preProcessors; + private final List<@NullableType VideoFrameProcessor> preProcessors; private final ExecutorService sharedExecutorService; @@ -211,10 +212,11 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { } @Override - public int registerInput() throws VideoFrameProcessingException { + public int registerInput(int forceId) throws VideoFrameProcessingException { checkStateNotNull(videoCompositor); - int videoCompositorInputId = videoCompositor.registerInputSource(); + int videoCompositorInputId; + videoCompositorInputId = videoCompositor.registerInputSource(forceId); // Creating a new VideoFrameProcessor for the input. VideoFrameProcessor preProcessor = videoFrameProcessorFactory @@ -257,7 +259,12 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId); } }); - preProcessors.add(preProcessor); + + while (preProcessors.size() <= videoCompositorInputId) { + //noinspection DataFlowIssue + preProcessors.add(null); + } + preProcessors.set(videoCompositorInputId, preProcessor); return videoCompositorInputId; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 33a4536481..17f1293562 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -38,7 +38,7 @@ import java.util.concurrent.Executor; @UnstableApi public abstract class SingleInputVideoGraph implements VideoGraph { - /** The ID {@link #registerInput()} returns. */ + /** The ID {@link #registerInput(int)} returns. */ public static final int SINGLE_INPUT_INDEX = 0; private final Context context; @@ -99,7 +99,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph { } @Override - public int registerInput() throws VideoFrameProcessingException { + public int registerInput(int sequenceIndex) throws VideoFrameProcessingException { checkStateNotNull(videoFrameProcessor == null && !released); videoFrameProcessor = diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 5ec3e68047..13ed072bb7 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -48,9 +48,11 @@ public interface VideoCompositor extends GlTextureProducer { /** * Registers a new input source, and returns a unique {@code inputId} corresponding to this * source, to be used in {@link #queueInputTexture}. + * + * @param sequenceId The sequence ID of the input source, which is can be used to determine the + * order of the input sources. */ - int registerInputSource(); - + int registerInputSource(int sequenceId); /** * Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 7ba81a4389..f9da1c3789 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -550,6 +550,8 @@ public final class CompositingVideoSinkProvider // reduces decoder timeouts, and consider restoring. videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); + int videoGraphInputId = videoGraph.registerInput(0); + videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId); videoEffects = new ArrayList<>(); finalBufferPresentationTimeUs = C.TIME_UNSET; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java index 2d584fdf30..bf85f8a757 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java @@ -856,7 +856,7 @@ public final class DefaultVideoCompositorPixelTest { VideoCompositor videoCompositor, @Nullable ExecutorService executorService, GlObjectsProvider glObjectsProvider) { - int inputId = videoCompositor.registerInputSource(); + int inputId = videoCompositor.registerInputSource(0); DefaultVideoFrameProcessor.Factory.Builder defaultVideoFrameProcessorFactoryBuilder = new DefaultVideoFrameProcessor.Factory.Builder() .setGlObjectsProvider(glObjectsProvider) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java index 6d434bd74c..951f8165f8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java @@ -97,7 +97,7 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format) + public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException { if (!returnedFirstInput) { // First input initialized in constructor because output AudioFormat is needed. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java index 03097e1119..30e1f81207 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java @@ -129,7 +129,7 @@ import java.util.concurrent.atomic.AtomicLong; } @Override - public GraphInput getInput(EditedMediaItem item, Format format) { + public GraphInput getInput(EditedMediaItem item, Format format, int sequenceIndex) { return this; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java index b430639161..276caef733 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java @@ -64,10 +64,10 @@ import java.util.List; * * @param editedMediaItem The initial {@link EditedMediaItem} of the input. * @param format The initial {@link Format} of the input. + * @param sequenceIndex The sequence index of the input. * @throws ExportException If an error occurs getting the input. */ - public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format) - throws ExportException; + public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException; /** * Processes the input data and returns whether it may be possible to process more data by calling diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 1adf5c1de6..d40df2a4ca 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -637,7 +637,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } GraphInput sampleExporterInput = - sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat); + sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat, sequenceIndex); OnMediaItemChangedListener onMediaItemChangedListener = (editedMediaItem, durationUs, decodedFormat, isLast) -> { onMediaItemChanged(trackType, durationUs, isLast); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java index 27a122bdda..a334379efb 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java @@ -22,6 +22,7 @@ import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoGraph; +import androidx.media3.common.util.Log; import androidx.media3.effect.MultipleInputVideoGraph; import androidx.media3.effect.VideoCompositorSettings; import java.util.List; @@ -79,8 +80,8 @@ import java.util.concurrent.Executor; } @Override - public GraphInput createInput() throws VideoFrameProcessingException { - int inputId = registerInput(); + public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { + int inputId = registerInput(sequenceIndex); return new VideoFrameProcessingWrapper( getProcessor(inputId), /* presentation= */ null, getInitialTimestampOffsetUs()); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java index 5118e7fe47..588ccc0601 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -106,12 +106,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public GraphInput createInput() throws VideoFrameProcessingException { + public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { checkState(videoFrameProcessingWrapper == null); - int inputId = registerInput(); + int inputId = registerInput(sequenceIndex); videoFrameProcessingWrapper = - new VideoFrameProcessingWrapper( - getProcessor(inputId), getPresentation(), getInitialTimestampOffsetUs()); + new VideoFrameProcessingWrapper( + getProcessor(inputId), + getInputColorInfo(), + getPresentation(), + getInitialTimestampOffsetUs()); return videoFrameProcessingWrapper; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java index eb9e836e45..5883e8ab6f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java @@ -69,6 +69,8 @@ import java.util.concurrent.Executor; *

This method must called exactly once for every input stream. * *

If the method throws any {@link Exception}, the caller must call {@link #release}. + * + * @param sequenceIndex The sequence index of the input, which can aid ordering of the inputs. */ - GraphInput createInput() throws VideoFrameProcessingException; + GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 929d4727db..0556bdf65f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -50,6 +50,7 @@ import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.effect.DebugTraceUtil; import androidx.media3.effect.VideoCompositorSettings; +import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; @@ -58,6 +59,8 @@ import java.util.Objects; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; +import java.nio.ByteBuffer; +import java.util.List; /** Processes, encodes and muxes raw video frames. */ /* package */ final class VideoSampleExporter extends SampleExporter { @@ -153,10 +156,9 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public GraphInput getInput(EditedMediaItem editedMediaItem, Format format) - throws ExportException { + public GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException { try { - return videoGraph.createInput(); + return videoGraph.createInput(sequenceIndex); } catch (VideoFrameProcessingException e) { throw ExportException.createForVideoFrameProcessingException(e); } @@ -540,8 +542,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public int registerInput() throws VideoFrameProcessingException { - return videoGraph.registerInput(); + public int registerInput(int sequenceIndex) throws VideoFrameProcessingException { + return videoGraph.registerInput(sequenceIndex); } @Override @@ -550,8 +552,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public GraphInput createInput() throws VideoFrameProcessingException { - return videoGraph.createInput(); + public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { + return videoGraph.createInput(sequenceIndex); } @Override From b909162daaf69097e575ad7d14039bfaacd3e73e Mon Sep 17 00:00:00 2001 From: Patrik Aradi Date: Wed, 31 Jan 2024 13:48:24 +0800 Subject: [PATCH 2/8] Use sparse array in MultipleInputVideoGraph, fix formatting --- .../effect/MultipleInputVideoGraph.java | 22 +++++++------------ .../TransformerMultipleInputVideoGraph.java | 1 - .../TransformerSingleInputVideoGraph.java | 10 ++++----- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index 6dac599b0b..8e76ee146d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -46,7 +46,6 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.GlUtil; -import androidx.media3.common.util.NullableType; import androidx.media3.common.util.UnstableApi; import com.google.common.util.concurrent.MoreExecutors; import java.util.ArrayDeque; @@ -76,7 +75,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { private final Executor listenerExecutor; private final VideoCompositorSettings videoCompositorSettings; private final List compositionEffects; - private final List<@NullableType VideoFrameProcessor> preProcessors; + private final SparseArray preProcessors; private final ExecutorService sharedExecutorService; @@ -115,7 +114,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { this.compositionEffects = new ArrayList<>(compositionEffects); this.initialTimestampOffsetUs = initialTimestampOffsetUs; lastRenderedPresentationTimeUs = C.TIME_UNSET; - preProcessors = new ArrayList<>(); + preProcessors = new SparseArray<>(); sharedExecutorService = newSingleThreadScheduledExecutor(SHARED_EXECUTOR_NAME); glObjectsProvider = new SingleContextGlObjectsProvider(); // TODO - b/289986435: Support injecting VideoFrameProcessor.Factory. @@ -137,7 +136,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { @Override public void initialize() throws VideoFrameProcessingException { checkState( - preProcessors.isEmpty() + preProcessors.size() == 0 && videoCompositor == null && compositionVideoFrameProcessor == null && !released); @@ -212,11 +211,10 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { } @Override - public int registerInput(int forceId) throws VideoFrameProcessingException { + public int registerInput(int sequenceIndex) throws VideoFrameProcessingException { checkStateNotNull(videoCompositor); - int videoCompositorInputId; - videoCompositorInputId = videoCompositor.registerInputSource(forceId); + int videoCompositorInputId = videoCompositor.registerInputSource(sequenceIndex); // Creating a new VideoFrameProcessor for the input. VideoFrameProcessor preProcessor = videoFrameProcessorFactory @@ -260,17 +258,13 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { } }); - while (preProcessors.size() <= videoCompositorInputId) { - //noinspection DataFlowIssue - preProcessors.add(null); - } - preProcessors.set(videoCompositorInputId, preProcessor); + preProcessors.put(videoCompositorInputId, preProcessor); return videoCompositorInputId; } @Override public VideoFrameProcessor getProcessor(int inputId) { - checkState(inputId < preProcessors.size()); + checkState(preProcessors.indexOfKey(inputId) >= 0); return preProcessors.get(inputId); } @@ -292,7 +286,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { // Needs to release the frame processors before their internal executor services are released. for (int i = 0; i < preProcessors.size(); i++) { - preProcessors.get(i).release(); + preProcessors.get(preProcessors.keyAt(i)).release(); } preProcessors.clear(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java index a334379efb..174911ea90 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java @@ -22,7 +22,6 @@ import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoGraph; -import androidx.media3.common.util.Log; import androidx.media3.effect.MultipleInputVideoGraph; import androidx.media3.effect.VideoCompositorSettings; import java.util.List; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java index 588ccc0601..3f6da227b1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -110,11 +110,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkState(videoFrameProcessingWrapper == null); int inputId = registerInput(sequenceIndex); videoFrameProcessingWrapper = - new VideoFrameProcessingWrapper( - getProcessor(inputId), - getInputColorInfo(), - getPresentation(), - getInitialTimestampOffsetUs()); + new VideoFrameProcessingWrapper( + getProcessor(inputId), + getInputColorInfo(), + getPresentation(), + getInitialTimestampOffsetUs()); return videoFrameProcessingWrapper; } } From 84df55c6e82171d07286f6d2405693e6d3b0518f Mon Sep 17 00:00:00 2001 From: Patrik Aradi Date: Tue, 6 Feb 2024 11:14:14 +0800 Subject: [PATCH 3/8] fix naming and formatting --- .../media3/effect/DefaultVideoCompositor.java | 12 ++++-------- .../media3/effect/MultipleInputVideoGraph.java | 1 - .../androidx/media3/effect/VideoCompositor.java | 6 +++--- .../media3/transformer/SampleExporter.java | 3 ++- .../media3/transformer/TransformerInternal.java | 2 +- .../media3/transformer/VideoSampleExporter.java | 14 +++++++------- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index 70f0f76a71..778e612d4a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -27,7 +27,6 @@ import android.opengl.EGLDisplay; import android.opengl.EGLSurface; import android.opengl.GLES20; import android.util.SparseArray; - import androidx.annotation.GuardedBy; import androidx.annotation.IntRange; import androidx.annotation.Nullable; @@ -43,18 +42,15 @@ import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; - import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; - -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - import java.io.IOException; import java.util.ArrayDeque; import java.util.Iterator; import java.util.List; import java.util.Queue; import java.util.concurrent.ExecutorService; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A basic {@link VideoCompositor} implementation that takes in frames from input sources' streams @@ -146,9 +142,9 @@ public final class DefaultVideoCompositor implements VideoCompositor { } @Override - public synchronized int registerInputSource(int sequenceId) { - inputSources.put(sequenceId, new InputSource()); - return sequenceId; + public synchronized int registerInputSource(int sequenceIndex) { + inputSources.put(sequenceIndex, new InputSource()); + return sequenceIndex; } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index 8e76ee146d..20542cb4f6 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -257,7 +257,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId); } }); - preProcessors.put(videoCompositorInputId, preProcessor); return videoCompositorInputId; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 13ed072bb7..198a7d566d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -49,10 +49,10 @@ public interface VideoCompositor extends GlTextureProducer { * Registers a new input source, and returns a unique {@code inputId} corresponding to this * source, to be used in {@link #queueInputTexture}. * - * @param sequenceId The sequence ID of the input source, which is can be used to determine the - * order of the input sources. + * @param sequenceIndex The sequence index of the input source, which is can be used to determine + * the order of the input sources. */ - int registerInputSource(int sequenceId); + int registerInputSource(int sequenceIndex); /** * Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java index 276caef733..e11f0fcd02 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java @@ -67,7 +67,8 @@ import java.util.List; * @param sequenceIndex The sequence index of the input. * @throws ExportException If an error occurs getting the input. */ - public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException; + public abstract GraphInput getInput( + EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException; /** * Processes the input data and returns whether it may be possible to process more data by calling diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index d40df2a4ca..f840801b26 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -637,7 +637,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } GraphInput sampleExporterInput = - sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat, sequenceIndex); + sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat, sequenceIndex); OnMediaItemChangedListener onMediaItemChangedListener = (editedMediaItem, durationUs, decodedFormat, isLast) -> { onMediaItemChanged(trackType, durationUs, isLast); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 0556bdf65f..9784af9cbe 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -59,8 +59,6 @@ import java.util.Objects; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; -import java.nio.ByteBuffer; -import java.util.List; /** Processes, encodes and muxes raw video frames. */ /* package */ final class VideoSampleExporter extends SampleExporter { @@ -276,14 +274,16 @@ import java.util.List; ImmutableList hdrEncoders = getSupportedEncodersForHdrEditing(requestedOutputMimeType, inputFormat.colorInfo); if (hdrEncoders.isEmpty()) { - // Fallback H.265/HEVC codecs for HDR content to avoid tonemapping. - hdrEncoders = - getSupportedEncodersForHdrEditing(MimeTypes.VIDEO_H265, inputFormat.colorInfo); + @Nullable + String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(inputFormat); + if (alternativeMimeType != null) { + requestedOutputMimeType = alternativeMimeType; + hdrEncoders = + getSupportedEncodersForHdrEditing(alternativeMimeType, inputFormat.colorInfo); + } } if (hdrEncoders.isEmpty()) { hdrMode = HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL; - } else { - requestedOutputMimeType = MimeTypes.VIDEO_H265; } } From 3cccecf36866ed0b749bdf90100b2c53fc88b276 Mon Sep 17 00:00:00 2001 From: Luyuan Chen Date: Wed, 7 Feb 2024 20:56:42 +0000 Subject: [PATCH 4/8] Format with google-java-format --- .../src/main/java/androidx/media3/common/VideoGraph.java | 1 - .../src/main/java/androidx/media3/effect/VideoCompositor.java | 3 ++- .../main/java/androidx/media3/transformer/SampleExporter.java | 2 +- .../java/androidx/media3/transformer/VideoSampleExporter.java | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 3221caa6be..9ac269b8f5 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -76,7 +76,6 @@ public interface VideoGraph { *

If the method throws, the caller must call {@link #release}. * * @param sequenceIndex The sequence index of the input which can aid ordering of the inputs. - * * @return The id of the registered input, which can be used to get the underlying {@link * VideoFrameProcessor} via {@link #getProcessor(int)}. */ diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 198a7d566d..e8835c46b2 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -50,9 +50,10 @@ public interface VideoCompositor extends GlTextureProducer { * source, to be used in {@link #queueInputTexture}. * * @param sequenceIndex The sequence index of the input source, which is can be used to determine - * the order of the input sources. + * the order of the input sources. */ int registerInputSource(int sequenceIndex); + /** * Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java index e11f0fcd02..478923af76 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java @@ -68,7 +68,7 @@ import java.util.List; * @throws ExportException If an error occurs getting the input. */ public abstract GraphInput getInput( - EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException; + EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException; /** * Processes the input data and returns whether it may be possible to process more data by calling diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 9784af9cbe..2960b49015 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -154,7 +154,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException { + public GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) + throws ExportException { try { return videoGraph.createInput(sequenceIndex); } catch (VideoFrameProcessingException e) { From 52adaf8d263737cc76f5839f66af028a1e440e0a Mon Sep 17 00:00:00 2001 From: Patrik Aradi Date: Tue, 13 Feb 2024 10:41:16 +0800 Subject: [PATCH 5/8] implement top down approach for passing input source id --- .../main/java/androidx/media3/common/VideoGraph.java | 4 +--- .../media3/effect/DefaultVideoCompositor.java | 3 +-- .../media3/effect/MultipleInputVideoGraph.java | 12 +++++------- .../media3/effect/SingleInputVideoGraph.java | 3 +-- .../java/androidx/media3/effect/VideoCompositor.java | 2 +- .../video/CompositingVideoSinkProvider.java | 4 ++-- .../transformer/DefaultVideoCompositorPixelTest.java | 7 ++++--- .../TransformerMultipleInputVideoGraph.java | 4 ++-- .../TransformerSingleInputVideoGraph.java | 7 ++----- .../media3/transformer/VideoSampleExporter.java | 4 ++-- 10 files changed, 21 insertions(+), 29 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 9ac269b8f5..89142f922f 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -76,10 +76,8 @@ public interface VideoGraph { *

If the method throws, the caller must call {@link #release}. * * @param sequenceIndex The sequence index of the input which can aid ordering of the inputs. - * @return The id of the registered input, which can be used to get the underlying {@link - * VideoFrameProcessor} via {@link #getProcessor(int)}. */ - int registerInput(int sequenceIndex) throws VideoFrameProcessingException; + void registerInput(int sequenceIndex) throws VideoFrameProcessingException; /** * Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index 778e612d4a..7a47951572 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -142,9 +142,8 @@ public final class DefaultVideoCompositor implements VideoCompositor { } @Override - public synchronized int registerInputSource(int sequenceIndex) { + public synchronized void registerInputSource(int sequenceIndex) { inputSources.put(sequenceIndex, new InputSource()); - return sequenceIndex; } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index 20542cb4f6..851375ffa1 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -211,10 +211,9 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { } @Override - public int registerInput(int sequenceIndex) throws VideoFrameProcessingException { + public void registerInput(int sequenceIndex) throws VideoFrameProcessingException { checkStateNotNull(videoCompositor); - - int videoCompositorInputId = videoCompositor.registerInputSource(sequenceIndex); + videoCompositor.registerInputSource(sequenceIndex); // Creating a new VideoFrameProcessor for the input. VideoFrameProcessor preProcessor = videoFrameProcessorFactory @@ -223,7 +222,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { // Texture output to compositor. (textureProducer, texture, presentationTimeUs, syncObject) -> queuePreProcessingOutputToCompositor( - videoCompositorInputId, textureProducer, texture, presentationTimeUs), + sequenceIndex, textureProducer, texture, presentationTimeUs), PRE_COMPOSITOR_TEXTURE_OUTPUT_CAPACITY) .build() .create( @@ -254,11 +253,10 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { @Override public void onEnded() { - onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId); + onPreProcessingVideoFrameProcessorEnded(sequenceIndex); } }); - preProcessors.put(videoCompositorInputId, preProcessor); - return videoCompositorInputId; + preProcessors.put(sequenceIndex, preProcessor); } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 17f1293562..789e3674e1 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -99,7 +99,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph { } @Override - public int registerInput(int sequenceIndex) throws VideoFrameProcessingException { + public void registerInput(int sequenceIndex) throws VideoFrameProcessingException { checkStateNotNull(videoFrameProcessor == null && !released); videoFrameProcessor = @@ -159,7 +159,6 @@ public abstract class SingleInputVideoGraph implements VideoGraph { if (outputSurfaceInfo != null) { videoFrameProcessor.setOutputSurfaceInfo(outputSurfaceInfo); } - return SINGLE_INPUT_INDEX; } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index e8835c46b2..348b9c6fad 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -52,7 +52,7 @@ public interface VideoCompositor extends GlTextureProducer { * @param sequenceIndex The sequence index of the input source, which is can be used to determine * the order of the input sources. */ - int registerInputSource(int sequenceIndex); + void registerInputSource(int sequenceIndex); /** * Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index f9da1c3789..99d0a76777 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -550,8 +550,8 @@ public final class CompositingVideoSinkProvider // reduces decoder timeouts, and consider restoring. videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); - int videoGraphInputId = videoGraph.registerInput(0); - videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId); + videoGraph.registerInput(0); + videoFrameProcessor = videoGraph.getProcessor(0); videoEffects = new ArrayList<>(); finalBufferPresentationTimeUs = C.TIME_UNSET; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java index bf85f8a757..8c9442235d 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java @@ -856,7 +856,8 @@ public final class DefaultVideoCompositorPixelTest { VideoCompositor videoCompositor, @Nullable ExecutorService executorService, GlObjectsProvider glObjectsProvider) { - int inputId = videoCompositor.registerInputSource(0); + int sequenceIndex = 0; + videoCompositor.registerInputSource(sequenceIndex); DefaultVideoFrameProcessor.Factory.Builder defaultVideoFrameProcessorFactoryBuilder = new DefaultVideoFrameProcessor.Factory.Builder() .setGlObjectsProvider(glObjectsProvider) @@ -870,7 +871,7 @@ public final class DefaultVideoCompositorPixelTest { textureBitmapReader.readBitmapUnpremultipliedAlpha( outputTexture, presentationTimeUs); videoCompositor.queueInputTexture( - inputId, + sequenceIndex, outputTextureProducer, outputTexture, ColorInfo.SRGB_BT709_FULL, @@ -884,7 +885,7 @@ public final class DefaultVideoCompositorPixelTest { .setTestId(testId) .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactoryBuilder.build()) .setBitmapReader(textureBitmapReader) - .setOnEndedListener(() -> videoCompositor.signalEndOfInputSource(inputId)); + .setOnEndedListener(() -> videoCompositor.signalEndOfInputSource(sequenceIndex)); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java index 174911ea90..c05ee46c7a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java @@ -80,8 +80,8 @@ import java.util.concurrent.Executor; @Override public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { - int inputId = registerInput(sequenceIndex); + registerInput(sequenceIndex); return new VideoFrameProcessingWrapper( - getProcessor(inputId), /* presentation= */ null, getInitialTimestampOffsetUs()); + getProcessor(sequenceIndex), /* presentation= */ null, getInitialTimestampOffsetUs()); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java index 3f6da227b1..69c05cb0c9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -108,13 +108,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { checkState(videoFrameProcessingWrapper == null); - int inputId = registerInput(sequenceIndex); + registerInput(sequenceIndex); videoFrameProcessingWrapper = new VideoFrameProcessingWrapper( - getProcessor(inputId), - getInputColorInfo(), - getPresentation(), - getInitialTimestampOffsetUs()); + getProcessor(sequenceIndex), getPresentation(), getInitialTimestampOffsetUs()); return videoFrameProcessingWrapper; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 2960b49015..8c15528fc2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -543,8 +543,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public int registerInput(int sequenceIndex) throws VideoFrameProcessingException { - return videoGraph.registerInput(sequenceIndex); + public void registerInput(int sequenceIndex) throws VideoFrameProcessingException { + videoGraph.registerInput(sequenceIndex); } @Override From 51a9bcca616bd492c17ac6d5c4a11c37ff01978b Mon Sep 17 00:00:00 2001 From: Patrik Aradi Date: Mon, 26 Feb 2024 09:34:32 +0800 Subject: [PATCH 6/8] Refactor according to reviews. --- .../src/main/java/androidx/media3/common/VideoGraph.java | 3 ++- .../java/androidx/media3/effect/MultipleInputVideoGraph.java | 4 +++- .../java/androidx/media3/effect/SingleInputVideoGraph.java | 2 +- .../src/main/java/androidx/media3/effect/VideoCompositor.java | 4 ++-- .../main/java/androidx/media3/transformer/SampleExporter.java | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 89142f922f..d542d22c0f 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -75,7 +75,8 @@ public interface VideoGraph { * *

If the method throws, the caller must call {@link #release}. * - * @param sequenceIndex The sequence index of the input which can aid ordering of the inputs. + * @param sequenceIndex The sequence index of the input which can aid ordering of the inputs. The + * index must start from 0. */ void registerInput(int sequenceIndex) throws VideoFrameProcessingException; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index 851375ffa1..e4dbd5433b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -47,6 +47,8 @@ import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; + import com.google.common.util.concurrent.MoreExecutors; import java.util.ArrayDeque; import java.util.ArrayList; @@ -261,7 +263,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { @Override public VideoFrameProcessor getProcessor(int inputId) { - checkState(preProcessors.indexOfKey(inputId) >= 0); + checkState(Util.contains(preProcessors, inputId)); return preProcessors.get(inputId); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 789e3674e1..02433043c6 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -38,7 +38,7 @@ import java.util.concurrent.Executor; @UnstableApi public abstract class SingleInputVideoGraph implements VideoGraph { - /** The ID {@link #registerInput(int)} returns. */ + /** The index of the only {@linkplain #registerInput(int) registered} input. */ public static final int SINGLE_INPUT_INDEX = 0; private final Context context; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 348b9c6fad..783a3f33bf 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -49,8 +49,8 @@ public interface VideoCompositor extends GlTextureProducer { * Registers a new input source, and returns a unique {@code inputId} corresponding to this * source, to be used in {@link #queueInputTexture}. * - * @param sequenceIndex The sequence index of the input source, which is can be used to determine - * the order of the input sources. + * @param sequenceIndex The sequence index of the input source which is used to determine the + * order of the input sources. */ void registerInputSource(int sequenceIndex); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java index 478923af76..eb5a5fba51 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java @@ -64,7 +64,7 @@ import java.util.List; * * @param editedMediaItem The initial {@link EditedMediaItem} of the input. * @param format The initial {@link Format} of the input. - * @param sequenceIndex The sequence index of the input. + * @param sequenceIndex The index of the input sequence. * @throws ExportException If an error occurs getting the input. */ public abstract GraphInput getInput( From 0403e5881d4045eb2b56675c70e1b1a2da51961b Mon Sep 17 00:00:00 2001 From: Luyuan Chen Date: Mon, 4 Mar 2024 11:57:36 +0000 Subject: [PATCH 7/8] Format with google-java-format --- .../src/main/java/androidx/media3/common/VideoGraph.java | 7 ++++--- .../androidx/media3/effect/MultipleInputVideoGraph.java | 8 +++----- .../androidx/media3/effect/SingleInputVideoGraph.java | 2 +- .../main/java/androidx/media3/effect/VideoCompositor.java | 5 ++--- .../exoplayer/video/CompositingVideoSinkProvider.java | 4 ++-- .../media3/transformer/TransformerVideoGraph.java | 2 +- .../androidx/media3/transformer/VideoSampleExporter.java | 4 ++-- 7 files changed, 15 insertions(+), 17 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index d542d22c0f..537c9f0cb6 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -82,10 +82,11 @@ public interface VideoGraph { /** * Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via - * {@link #registerInput(int)}. If the {@code inputId} is not {@linkplain #registerInput(int) - * registered} before, this method will throw an {@link IllegalStateException}. + * {@link #registerInput(int)}. If the {@code sequenceIndex} is not {@linkplain + * #registerInput(int) registered} before, this method will throw an {@link + * IllegalStateException}. */ - VideoFrameProcessor getProcessor(int inputId); + VideoFrameProcessor getProcessor(int sequenceIndex); /** * Sets the output surface and supporting information. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index e4dbd5433b..f4f86653ba 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -47,8 +47,6 @@ import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; -import androidx.media3.common.util.Util; - import com.google.common.util.concurrent.MoreExecutors; import java.util.ArrayDeque; import java.util.ArrayList; @@ -262,9 +260,9 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { } @Override - public VideoFrameProcessor getProcessor(int inputId) { - checkState(Util.contains(preProcessors, inputId)); - return preProcessors.get(inputId); + public VideoFrameProcessor getProcessor(int sequenceIndex) { + checkState(contains(preProcessors, sequenceIndex)); + return preProcessors.get(sequenceIndex); } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 02433043c6..31520c47ae 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -162,7 +162,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph { } @Override - public VideoFrameProcessor getProcessor(int inputId) { + public VideoFrameProcessor getProcessor(int sequenceIndex) { return checkStateNotNull(videoFrameProcessor); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 783a3f33bf..365f41446e 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -46,11 +46,10 @@ public interface VideoCompositor extends GlTextureProducer { } /** - * Registers a new input source, and returns a unique {@code inputId} corresponding to this - * source, to be used in {@link #queueInputTexture}. + * Registers a new input source. * * @param sequenceIndex The sequence index of the input source which is used to determine the - * order of the input sources. + * order of the input sources. The same index should to be used in {@link #queueInputTexture}. */ void registerInputSource(int sequenceIndex); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 99d0a76777..b9a043136a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -550,8 +550,8 @@ public final class CompositingVideoSinkProvider // reduces decoder timeouts, and consider restoring. videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); - videoGraph.registerInput(0); - videoFrameProcessor = videoGraph.getProcessor(0); + videoGraph.registerInput(/* sequenceIndex= */ 0); + videoFrameProcessor = videoGraph.getProcessor(/* sequenceIndex= */ 0); videoEffects = new ArrayList<>(); finalBufferPresentationTimeUs = C.TIME_UNSET; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java index 5883e8ab6f..59f5727a09 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java @@ -70,7 +70,7 @@ import java.util.concurrent.Executor; * *

If the method throws any {@link Exception}, the caller must call {@link #release}. * - * @param sequenceIndex The sequence index of the input, which can aid ordering of the inputs. + * @param sequenceIndex The index of the input sequence, which is used to order the inputs. */ GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 8c15528fc2..c1a5d4d011 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -548,8 +548,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public VideoFrameProcessor getProcessor(int inputId) { - return videoGraph.getProcessor(inputId); + public VideoFrameProcessor getProcessor(int sequenceIndex) { + return videoGraph.getProcessor(sequenceIndex); } @Override From 766ff44a2ca3ada05dd633c53038b11f354e69a9 Mon Sep 17 00:00:00 2001 From: Luyuan Chen Date: Tue, 5 Mar 2024 14:05:27 +0000 Subject: [PATCH 8/8] Fix review comment --- .../androidx/media3/common/VideoGraph.java | 14 +++-- .../media3/effect/DefaultVideoCompositor.java | 56 ++++++++++--------- .../effect/MultipleInputVideoGraph.java | 20 ++++--- .../PreviewingSingleInputVideoGraph.java | 2 +- .../media3/effect/SingleInputVideoGraph.java | 18 ++++-- .../media3/effect/VideoCompositor.java | 19 ++++--- .../video/CompositingVideoSinkProvider.java | 7 +-- .../DefaultVideoCompositorPixelTest.java | 13 +++-- .../transformer/AudioSampleExporter.java | 2 +- .../transformer/EncodedSampleExporter.java | 2 +- .../media3/transformer/SampleExporter.java | 4 +- .../TransformerMultipleInputVideoGraph.java | 6 +- .../TransformerSingleInputVideoGraph.java | 6 +- .../transformer/TransformerVideoGraph.java | 4 +- .../transformer/VideoSampleExporter.java | 18 +++--- 15 files changed, 108 insertions(+), 83 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 537c9f0cb6..440a526352 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -16,6 +16,7 @@ package androidx.media3.common; +import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.media3.common.util.UnstableApi; @@ -73,20 +74,23 @@ public interface VideoGraph { *

A underlying processing {@link VideoFrameProcessor} is created every time this method is * called. * + *

All inputs must be registered before rendering frames to the underlying + * {@link #getProcessor(int) VideoFrameProcessor}. + * *

If the method throws, the caller must call {@link #release}. * - * @param sequenceIndex The sequence index of the input which can aid ordering of the inputs. The - * index must start from 0. + * @param inputIndex The index of the input which could be used to order the inputs. + * The index must start from 0. */ - void registerInput(int sequenceIndex) throws VideoFrameProcessingException; + void registerInput(@IntRange(from = 0) int inputIndex) throws VideoFrameProcessingException; /** * Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via - * {@link #registerInput(int)}. If the {@code sequenceIndex} is not {@linkplain + * {@link #registerInput(int)}. If the {@code inputIndex} is not {@linkplain * #registerInput(int) registered} before, this method will throw an {@link * IllegalStateException}. */ - VideoFrameProcessor getProcessor(int sequenceIndex); + VideoFrameProcessor getProcessor(int inputIndex); /** * Sets the output surface and supporting information. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index 7a47951572..d390b8e887 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -18,6 +18,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.contains; import static java.lang.Math.abs; import static java.lang.Math.max; @@ -78,7 +79,8 @@ public final class DefaultVideoCompositor implements VideoCompositor { private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread"; private static final String TAG = "DefaultVideoCompositor"; - private static final int PRIMARY_INPUT_ID = 0; + // TODO: b/338579287: Use the first registered index instead of a constant value. + private static final int PRIMARY_INPUT_INDEX = 0; private final VideoCompositor.Listener listener; private final GlTextureProducer.Listener textureOutputListener; @@ -142,24 +144,26 @@ public final class DefaultVideoCompositor implements VideoCompositor { } @Override - public synchronized void registerInputSource(int sequenceIndex) { - inputSources.put(sequenceIndex, new InputSource()); + public synchronized void registerInputSource(@IntRange(from = 0) int inputIndex) { + checkState(!contains(inputSources, inputIndex)); + inputSources.put(inputIndex, new InputSource()); } @Override - public synchronized void signalEndOfInputSource(int inputId) { - inputSources.get(inputId).isInputEnded = true; + public synchronized void signalEndOfInputSource(int inputIndex) { + checkState(contains(inputSources, inputIndex)); + inputSources.get(inputIndex).isInputEnded = true; boolean allInputsEnded = true; for (int i = 0; i < inputSources.size(); i++) { - if (!inputSources.get(inputSources.keyAt(i)).isInputEnded) { + if (!inputSources.valueAt(i).isInputEnded) { allInputsEnded = false; break; } } this.allInputsEnded = allInputsEnded; - if (inputSources.get(PRIMARY_INPUT_ID).frameInfos.isEmpty()) { - if (inputId == PRIMARY_INPUT_ID) { + if (inputSources.get(PRIMARY_INPUT_INDEX).frameInfos.isEmpty()) { + if (inputIndex == PRIMARY_INPUT_INDEX) { releaseExcessFramesInAllSecondaryStreams(); } if (allInputsEnded) { @@ -167,7 +171,8 @@ public final class DefaultVideoCompositor implements VideoCompositor { return; } } - if (inputId != PRIMARY_INPUT_ID && inputSources.get(inputId).frameInfos.size() == 1) { + if (inputIndex != PRIMARY_INPUT_INDEX + && inputSources.get(inputIndex).frameInfos.size() == 1) { // When a secondary stream ends input, composite if there was only one pending frame in the // stream. videoFrameProcessingTaskExecutor.submit(this::maybeComposite); @@ -176,12 +181,13 @@ public final class DefaultVideoCompositor implements VideoCompositor { @Override public synchronized void queueInputTexture( - int inputId, + int inputIndex, GlTextureProducer textureProducer, GlTextureInfo inputTexture, ColorInfo colorInfo, long presentationTimeUs) { - InputSource inputSource = inputSources.get(inputId); + checkState(contains(inputSources, inputIndex)); + InputSource inputSource = inputSources.get(inputIndex); checkState(!inputSource.isInputEnded); checkStateNotNull(!ColorInfo.isTransferHdr(colorInfo), "HDR input is not supported."); if (configuredColorInfo == null) { @@ -195,10 +201,10 @@ public final class DefaultVideoCompositor implements VideoCompositor { textureProducer, inputTexture, presentationTimeUs, - settings.getOverlaySettings(inputId, presentationTimeUs)); + settings.getOverlaySettings(inputIndex, presentationTimeUs)); inputSource.frameInfos.add(inputFrameInfo); - if (inputId == PRIMARY_INPUT_ID) { + if (inputIndex == PRIMARY_INPUT_INDEX) { releaseExcessFramesInAllSecondaryStreams(); } else { releaseExcessFramesInSecondaryStream(inputSource); @@ -224,11 +230,11 @@ public final class DefaultVideoCompositor implements VideoCompositor { } private synchronized void releaseExcessFramesInAllSecondaryStreams() { - for (int i = 0; i < inputSources.size(); i++) { - if (i == PRIMARY_INPUT_ID) { + for (int inputIndex = 0; inputIndex < inputSources.size(); inputIndex++) { + if (inputIndex == PRIMARY_INPUT_INDEX) { continue; } - releaseExcessFramesInSecondaryStream(inputSources.get(inputSources.keyAt(i))); + releaseExcessFramesInSecondaryStream(inputSources.valueAt(inputIndex)); } } @@ -240,7 +246,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { * began. */ private synchronized void releaseExcessFramesInSecondaryStream(InputSource secondaryInputSource) { - InputSource primaryInputSource = inputSources.get(PRIMARY_INPUT_ID); + InputSource primaryInputSource = inputSources.get(PRIMARY_INPUT_INDEX); // If the primary stream output is ended, all secondary frames can be released. if (primaryInputSource.frameInfos.isEmpty() && primaryInputSource.isInputEnded) { releaseFrames( @@ -291,7 +297,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { return; } - InputFrameInfo primaryInputFrame = framesToComposite.get(PRIMARY_INPUT_ID); + InputFrameInfo primaryInputFrame = framesToComposite.get(PRIMARY_INPUT_INDEX); ImmutableList.Builder inputSizes = new ImmutableList.Builder<>(); for (int i = 0; i < framesToComposite.size(); i++) { @@ -312,7 +318,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { textureOutputListener.onTextureRendered( /* textureProducer= */ this, outputTexture, outputPresentationTimestampUs, syncObject); - InputSource primaryInputSource = inputSources.get(PRIMARY_INPUT_ID); + InputSource primaryInputSource = inputSources.get(PRIMARY_INPUT_INDEX); releaseFrames(primaryInputSource, /* numberOfFramesToRelease= */ 1); releaseExcessFramesInAllSecondaryStreams(); @@ -332,18 +338,18 @@ public final class DefaultVideoCompositor implements VideoCompositor { if (outputTexturePool.freeTextureCount() == 0) { return ImmutableList.of(); } - for (int inputId = 0; inputId < inputSources.size(); inputId++) { - if (inputSources.get(inputSources.keyAt(inputId)).frameInfos.isEmpty()) { + for (int i = 0; i < inputSources.size(); i++) { + if (inputSources.valueAt(i).frameInfos.isEmpty()) { return ImmutableList.of(); } } ImmutableList.Builder framesToComposite = new ImmutableList.Builder<>(); InputFrameInfo primaryFrameToComposite = - inputSources.get(PRIMARY_INPUT_ID).frameInfos.element(); + inputSources.get(PRIMARY_INPUT_INDEX).frameInfos.element(); framesToComposite.add(primaryFrameToComposite); - for (int inputId = 0; inputId < inputSources.size(); inputId++) { - if (inputId == PRIMARY_INPUT_ID) { + for (int i = 0; i < inputSources.size(); i++) { + if (i == PRIMARY_INPUT_INDEX) { continue; } // Select the secondary streams' frame that would be composited next. The frame selected is @@ -352,7 +358,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { // 2. Two or more frames, and at least one frame has timestamp greater than the target // timestamp. // The smaller timestamp is taken if two timestamps have the same distance from the primary. - InputSource secondaryInputSource = inputSources.get(inputSources.keyAt(inputId)); + InputSource secondaryInputSource = inputSources.valueAt(i); if (secondaryInputSource.frameInfos.size() == 1 && !secondaryInputSource.isInputEnded) { return ImmutableList.of(); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index f4f86653ba..491572da96 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -33,6 +33,7 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLSurface; import android.util.SparseArray; +import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; @@ -211,9 +212,10 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { } @Override - public void registerInput(int sequenceIndex) throws VideoFrameProcessingException { - checkStateNotNull(videoCompositor); - videoCompositor.registerInputSource(sequenceIndex); + public void registerInput(@IntRange(from = 0) int inputIndex) + throws VideoFrameProcessingException { + checkState(!contains(preProcessors, inputIndex)); + checkNotNull(videoCompositor).registerInputSource(inputIndex); // Creating a new VideoFrameProcessor for the input. VideoFrameProcessor preProcessor = videoFrameProcessorFactory @@ -222,7 +224,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { // Texture output to compositor. (textureProducer, texture, presentationTimeUs, syncObject) -> queuePreProcessingOutputToCompositor( - sequenceIndex, textureProducer, texture, presentationTimeUs), + inputIndex, textureProducer, texture, presentationTimeUs), PRE_COMPOSITOR_TEXTURE_OUTPUT_CAPACITY) .build() .create( @@ -253,16 +255,16 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { @Override public void onEnded() { - onPreProcessingVideoFrameProcessorEnded(sequenceIndex); + onPreProcessingVideoFrameProcessorEnded(inputIndex); } }); - preProcessors.put(sequenceIndex, preProcessor); + preProcessors.put(inputIndex, preProcessor); } @Override - public VideoFrameProcessor getProcessor(int sequenceIndex) { - checkState(contains(preProcessors, sequenceIndex)); - return preProcessors.get(sequenceIndex); + public VideoFrameProcessor getProcessor(int inputIndex) { + checkState(contains(preProcessors, inputIndex)); + return preProcessors.get(inputIndex); } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java index d6edae887e..8998831524 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java @@ -109,6 +109,6 @@ public final class PreviewingSingleInputVideoGraph extends SingleInputVideoGraph @Override public void renderOutputFrame(long renderTimeNs) { - getProcessor(SINGLE_INPUT_INDEX).renderOutputFrame(renderTimeNs); + getProcessor(getInputIndex()).renderOutputFrame(renderTimeNs); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 31520c47ae..9328eab27f 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -16,11 +16,13 @@ package androidx.media3.effect; +import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import androidx.annotation.Nullable; +import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; @@ -38,9 +40,6 @@ import java.util.concurrent.Executor; @UnstableApi public abstract class SingleInputVideoGraph implements VideoGraph { - /** The index of the only {@linkplain #registerInput(int) registered} input. */ - public static final int SINGLE_INPUT_INDEX = 0; - private final Context context; private final VideoFrameProcessor.Factory videoFrameProcessorFactory; private final ColorInfo outputColorInfo; @@ -56,6 +55,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph { private boolean isEnded; private boolean released; private volatile boolean hasProducedFrameWithTimestampZero; + private int inputIndex; /** * Creates an instance. @@ -86,6 +86,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph { this.renderFramesAutomatically = renderFramesAutomatically; this.presentation = presentation; this.initialTimestampOffsetUs = initialTimestampOffsetUs; + this.inputIndex = C.INDEX_UNSET; } /** @@ -99,9 +100,11 @@ public abstract class SingleInputVideoGraph implements VideoGraph { } @Override - public void registerInput(int sequenceIndex) throws VideoFrameProcessingException { + public void registerInput(int inputIndex) throws VideoFrameProcessingException { checkStateNotNull(videoFrameProcessor == null && !released); + checkState(this.inputIndex == C.INDEX_UNSET); + this.inputIndex = inputIndex; videoFrameProcessor = videoFrameProcessorFactory.create( context, @@ -162,7 +165,8 @@ public abstract class SingleInputVideoGraph implements VideoGraph { } @Override - public VideoFrameProcessor getProcessor(int sequenceIndex) { + public VideoFrameProcessor getProcessor(int inputIndex) { + checkArgument(this.inputIndex != C.INDEX_UNSET && this.inputIndex == inputIndex); return checkStateNotNull(videoFrameProcessor); } @@ -192,6 +196,10 @@ public abstract class SingleInputVideoGraph implements VideoGraph { released = true; } + protected int getInputIndex() { + return inputIndex; + } + protected long getInitialTimestampOffsetUs() { return initialTimestampOffsetUs; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 365f41446e..e128ccd252 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -15,6 +15,7 @@ */ package androidx.media3.effect; +import androidx.annotation.IntRange; import androidx.media3.common.ColorInfo; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; @@ -48,29 +49,33 @@ public interface VideoCompositor extends GlTextureProducer { /** * Registers a new input source. * - * @param sequenceIndex The sequence index of the input source which is used to determine the - * order of the input sources. The same index should to be used in {@link #queueInputTexture}. + * @param inputIndex The index of the input source which could be used to determine the order of + * the input sources. The same index should to be used in {@link #queueInputTexture}. The + * index must start from 0. All inputs must be registered before + * {@linkplain #queueInputTexture(int, GlTextureProducer, GlTextureInfo, ColorInfo, long) queueing} + * textures. */ - void registerInputSource(int sequenceIndex); + void registerInputSource(@IntRange(from = 0) int inputIndex); /** * Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}. * - * @param inputId The identifier for an input source, returned from {@link #registerInputSource}. + * @param inputIndex The index of the input source. */ - void signalEndOfInputSource(int inputId); + void signalEndOfInputSource(int inputIndex); /** * Queues an input texture to be composited. * - * @param inputId The identifier for an input source, returned from {@link #registerInputSource}. + * @param inputIndex The index of the input source, the same index used when {@linkplain + * #registerInputSource(int) registering the input source}. * @param textureProducer The source from where the {@code inputTexture} is produced. * @param inputTexture The {@link GlTextureInfo} to composite. * @param colorInfo The {@link ColorInfo} of {@code inputTexture}. * @param presentationTimeUs The presentation time of {@code inputTexture}, in microseconds. */ void queueInputTexture( - int inputId, + int inputIndex, GlTextureProducer textureProducer, GlTextureInfo inputTexture, ColorInfo colorInfo, diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index b9a043136a..5e468015d4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -419,7 +419,6 @@ public final class CompositingVideoSinkProvider outputColorInfo = inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build(); } - int videoGraphInputId; try { videoGraph = previewingVideoGraphFactory.create( @@ -435,12 +434,12 @@ public final class CompositingVideoSinkProvider Size size = currentSurfaceAndSize.second; maybeSetOutputSurfaceInfo(surface, size.getWidth(), size.getHeight()); } - videoGraphInputId = videoGraph.registerInput(); + videoGraph.registerInput(/* inputIndex= */ 0); } catch (VideoFrameProcessingException e) { throw new VideoSink.VideoSinkException(e, sourceFormat); } state = STATE_INITIALIZED; - return videoGraph.getProcessor(videoGraphInputId); + return videoGraph.getProcessor(/* inputIndex= */ 0); } private boolean isInitialized() { @@ -550,8 +549,6 @@ public final class CompositingVideoSinkProvider // reduces decoder timeouts, and consider restoring. videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); - videoGraph.registerInput(/* sequenceIndex= */ 0); - videoFrameProcessor = videoGraph.getProcessor(/* sequenceIndex= */ 0); videoEffects = new ArrayList<>(); finalBufferPresentationTimeUs = C.TIME_UNSET; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java index 8c9442235d..76094d0144 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java @@ -744,7 +744,8 @@ public final class DefaultVideoCompositorPixelTest { textureBitmapReader, videoCompositor, sharedExecutorService, - glObjectsProvider) + glObjectsProvider, + /* inputIndex= */ i) .setEffects(effectsToApply.build()) .build(); inputVideoFrameProcessorTestRunners.add(vfpTestRunner); @@ -855,9 +856,9 @@ public final class DefaultVideoCompositorPixelTest { TextureBitmapReader textureBitmapReader, VideoCompositor videoCompositor, @Nullable ExecutorService executorService, - GlObjectsProvider glObjectsProvider) { - int sequenceIndex = 0; - videoCompositor.registerInputSource(sequenceIndex); + GlObjectsProvider glObjectsProvider, + int inputIndex) { + videoCompositor.registerInputSource(inputIndex); DefaultVideoFrameProcessor.Factory.Builder defaultVideoFrameProcessorFactoryBuilder = new DefaultVideoFrameProcessor.Factory.Builder() .setGlObjectsProvider(glObjectsProvider) @@ -871,7 +872,7 @@ public final class DefaultVideoCompositorPixelTest { textureBitmapReader.readBitmapUnpremultipliedAlpha( outputTexture, presentationTimeUs); videoCompositor.queueInputTexture( - sequenceIndex, + inputIndex, outputTextureProducer, outputTexture, ColorInfo.SRGB_BT709_FULL, @@ -885,7 +886,7 @@ public final class DefaultVideoCompositorPixelTest { .setTestId(testId) .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactoryBuilder.build()) .setBitmapReader(textureBitmapReader) - .setOnEndedListener(() -> videoCompositor.signalEndOfInputSource(sequenceIndex)); + .setOnEndedListener(() -> videoCompositor.signalEndOfInputSource(inputIndex)); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java index 951f8165f8..9b053c3f3b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java @@ -97,7 +97,7 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) + public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format, int inputIndex) throws ExportException { if (!returnedFirstInput) { // First input initialized in constructor because output AudioFormat is needed. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java index 30e1f81207..d7d3e65f87 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java @@ -129,7 +129,7 @@ import java.util.concurrent.atomic.AtomicLong; } @Override - public GraphInput getInput(EditedMediaItem item, Format format, int sequenceIndex) { + public GraphInput getInput(EditedMediaItem item, Format format, int inputIndex) { return this; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java index eb5a5fba51..cafc939a5d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java @@ -64,11 +64,11 @@ import java.util.List; * * @param editedMediaItem The initial {@link EditedMediaItem} of the input. * @param format The initial {@link Format} of the input. - * @param sequenceIndex The index of the input sequence. + * @param inputIndex The index of the input. * @throws ExportException If an error occurs getting the input. */ public abstract GraphInput getInput( - EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException; + EditedMediaItem editedMediaItem, Format format, int inputIndex) throws ExportException; /** * Processes the input data and returns whether it may be possible to process more data by calling diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java index c05ee46c7a..f206525bab 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java @@ -79,9 +79,9 @@ import java.util.concurrent.Executor; } @Override - public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { - registerInput(sequenceIndex); + public GraphInput createInput(int inputIndex) throws VideoFrameProcessingException { + registerInput(inputIndex); return new VideoFrameProcessingWrapper( - getProcessor(sequenceIndex), /* presentation= */ null, getInitialTimestampOffsetUs()); + getProcessor(inputIndex), /* presentation= */ null, getInitialTimestampOffsetUs()); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java index 69c05cb0c9..571dd81ae6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -106,12 +106,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { + public GraphInput createInput(int inputIndex) throws VideoFrameProcessingException { checkState(videoFrameProcessingWrapper == null); - registerInput(sequenceIndex); + registerInput(inputIndex); videoFrameProcessingWrapper = new VideoFrameProcessingWrapper( - getProcessor(sequenceIndex), getPresentation(), getInitialTimestampOffsetUs()); + getProcessor(inputIndex), getPresentation(), getInitialTimestampOffsetUs()); return videoFrameProcessingWrapper; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java index 59f5727a09..016da4c01a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java @@ -70,7 +70,7 @@ import java.util.concurrent.Executor; * *

If the method throws any {@link Exception}, the caller must call {@link #release}. * - * @param sequenceIndex The index of the input sequence, which is used to order the inputs. + * @param inputIndex The index of the input, which could be used to order the inputs. */ - GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException; + GraphInput createInput(int inputIndex) throws VideoFrameProcessingException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index c1a5d4d011..7698b92d9e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -33,6 +33,7 @@ import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.util.Pair; import android.view.Surface; +import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; @@ -154,10 +155,10 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) + public GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int inputIndex) throws ExportException { try { - return videoGraph.createInput(sequenceIndex); + return videoGraph.createInput(inputIndex); } catch (VideoFrameProcessingException e) { throw ExportException.createForVideoFrameProcessingException(e); } @@ -543,18 +544,19 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public void registerInput(int sequenceIndex) throws VideoFrameProcessingException { - videoGraph.registerInput(sequenceIndex); + public void registerInput(@IntRange(from = 0) int inputIndex) + throws VideoFrameProcessingException { + videoGraph.registerInput(inputIndex); } @Override - public VideoFrameProcessor getProcessor(int sequenceIndex) { - return videoGraph.getProcessor(sequenceIndex); + public VideoFrameProcessor getProcessor(int inputIndex) { + return videoGraph.getProcessor(inputIndex); } @Override - public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException { - return videoGraph.createInput(sequenceIndex); + public GraphInput createInput(int inputIndex) throws VideoFrameProcessingException { + return videoGraph.createInput(inputIndex); } @Override