loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix indeterminate z-order of EditedMediaItemSequences by passing sequenceIndex when registeringInput

Browse Source
This commit is contained in:
Patrik Aradi 2024-01-31 09:55:40 +08:00 committed by Luyuan Chen
parent bef3d518d2
commit 177f1f33d0
15 changed files with 66 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libraries/common/src/main/java/androidx/media3/common/VideoGraph.java
View File

@ -75,14 +75,16 @@ public interface VideoGraph {
*
* <p>If the method throws, the caller must call {@link #release}.
*
* @param sequenceIndex The sequence index of the input which can aid ordering of the inputs.
*
* @return The id of the registered input, which can be used to get the underlying {@link
* VideoFrameProcessor} via {@link #getProcessor(int)}.
*/
int registerInput() throws VideoFrameProcessingException;
int registerInput(int sequenceIndex) throws VideoFrameProcessingException;
/**
* Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via
* {@link #registerInput()}. If the {@code inputId} is not {@linkplain #registerInput()
* {@link #registerInput(int)}. If the {@code inputId} is not {@linkplain #registerInput(int)
* registered} before, this method will throw an {@link IllegalStateException}.
*/
VideoFrameProcessor getProcessor(int inputId);

26
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java
View File

@ -26,6 +26,8 @@ import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.util.SparseArray;
import androidx.annotation.GuardedBy;
import androidx.annotation.IntRange;
import androidx.annotation.Nullable;
@ -41,16 +43,18 @@ import androidx.media3.common.util.LongArrayQueue;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ExecutorService;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A basic {@link VideoCompositor} implementation that takes in frames from input sources' streams
@ -88,7 +92,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
@GuardedBy("this")
private final List<InputSource> inputSources;
private final SparseArray<InputSource> inputSources;
@GuardedBy("this")
private boolean allInputsEnded; // Whether all inputSources have signaled end of input.
@ -124,7 +128,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
this.settings = settings;
this.compositorGlProgram = new CompositorGlProgram(context);
inputSources = new ArrayList<>();
inputSources = new SparseArray<>();
outputTexturePool =
new TexturePool(/* useHighPrecisionColorComponents= */ false, textureOutputCapacity);
outputTextureTimestamps = new LongArrayQueue(textureOutputCapacity);
@ -142,9 +146,9 @@ public final class DefaultVideoCompositor implements VideoCompositor {
}
@Override
public synchronized int registerInputSource() {
inputSources.add(new InputSource());
return inputSources.size() - 1;
public synchronized int registerInputSource(int sequenceId) {
inputSources.put(sequenceId, new InputSource());
return sequenceId;
}
@Override
@ -152,7 +156,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
inputSources.get(inputId).isInputEnded = true;
boolean allInputsEnded = true;
for (int i = 0; i < inputSources.size(); i++) {
if (!inputSources.get(i).isInputEnded) {
if (!inputSources.get(inputSources.keyAt(i)).isInputEnded) {
allInputsEnded = false;
break;
}
@ -229,7 +233,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
if (i == PRIMARY_INPUT_ID) {
continue;
}
releaseExcessFramesInSecondaryStream(inputSources.get(i));
releaseExcessFramesInSecondaryStream(inputSources.get(inputSources.keyAt(i)));
}
}
@ -334,7 +338,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
return ImmutableList.of();
}
for (int inputId = 0; inputId < inputSources.size(); inputId++) {
if (inputSources.get(inputId).frameInfos.isEmpty()) {
if (inputSources.get(inputSources.keyAt(inputId)).frameInfos.isEmpty()) {
return ImmutableList.of();
}
}
@ -353,7 +357,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
// 2. Two or more frames, and at least one frame has timestamp greater than the target
// timestamp.
// The smaller timestamp is taken if two timestamps have the same distance from the primary.
InputSource secondaryInputSource = inputSources.get(inputId);
InputSource secondaryInputSource = inputSources.get(inputSources.keyAt(inputId));
if (secondaryInputSource.frameInfos.size() == 1 && !secondaryInputSource.isInputEnded) {
return ImmutableList.of();
}

15
libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java
View File

@ -46,6 +46,7 @@ import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.NullableType;
import androidx.media3.common.util.UnstableApi;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.ArrayDeque;
@ -75,7 +76,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
private final Executor listenerExecutor;
private final VideoCompositorSettings videoCompositorSettings;
private final List<Effect> compositionEffects;
private final List<VideoFrameProcessor> preProcessors;
private final List<@NullableType VideoFrameProcessor> preProcessors;
private final ExecutorService sharedExecutorService;
@ -211,10 +212,11 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
}
@Override
public int registerInput() throws VideoFrameProcessingException {
public int registerInput(int forceId) throws VideoFrameProcessingException {
checkStateNotNull(videoCompositor);
int videoCompositorInputId = videoCompositor.registerInputSource();
int videoCompositorInputId;
videoCompositorInputId = videoCompositor.registerInputSource(forceId);
// Creating a new VideoFrameProcessor for the input.
VideoFrameProcessor preProcessor =
videoFrameProcessorFactory
@ -257,7 +259,12 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId);
}
});
preProcessors.add(preProcessor);
while (preProcessors.size() <= videoCompositorInputId) {
//noinspection DataFlowIssue
preProcessors.add(null);
}
preProcessors.set(videoCompositorInputId, preProcessor);
return videoCompositorInputId;
}

4
libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java
View File

@ -38,7 +38,7 @@ import java.util.concurrent.Executor;
@UnstableApi
public abstract class SingleInputVideoGraph implements VideoGraph {
/** The ID {@link #registerInput()} returns. */
/** The ID {@link #registerInput(int)} returns. */
public static final int SINGLE_INPUT_INDEX = 0;
private final Context context;
@ -99,7 +99,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph {
}
@Override
public int registerInput() throws VideoFrameProcessingException {
public int registerInput(int sequenceIndex) throws VideoFrameProcessingException {
checkStateNotNull(videoFrameProcessor == null && !released);
videoFrameProcessor =

6
libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java
View File

@ -48,9 +48,11 @@ public interface VideoCompositor extends GlTextureProducer {
/**
* Registers a new input source, and returns a unique {@code inputId} corresponding to this
* source, to be used in {@link #queueInputTexture}.
*
* @param sequenceId The sequence ID of the input source, which is can be used to determine the
* order of the input sources.
*/
int registerInputSource();
int registerInputSource(int sequenceId);
/**
* Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}.
*

2
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
View File

@ -550,6 +550,8 @@ public final class CompositingVideoSinkProvider
// reduces decoder timeouts, and consider restoring.
videoFrameProcessorMaxPendingFrameCount =
Util.getMaxPendingFramesCountForMediaCodecDecoders(context);
int videoGraphInputId = videoGraph.registerInput(0);
videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId);
videoEffects = new ArrayList<>();
finalBufferPresentationTimeUs = C.TIME_UNSET;

2
libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java
View File

@ -856,7 +856,7 @@ public final class DefaultVideoCompositorPixelTest {
VideoCompositor videoCompositor,
@Nullable ExecutorService executorService,
GlObjectsProvider glObjectsProvider) {
int inputId = videoCompositor.registerInputSource();
int inputId = videoCompositor.registerInputSource(0);
DefaultVideoFrameProcessor.Factory.Builder defaultVideoFrameProcessorFactoryBuilder =
new DefaultVideoFrameProcessor.Factory.Builder()
.setGlObjectsProvider(glObjectsProvider)

2
libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java
View File

@ -97,7 +97,7 @@ import org.checkerframework.dataflow.qual.Pure;
}
@Override
public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format)
public AudioGraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex)
throws ExportException {
if (!returnedFirstInput) {
// First input initialized in constructor because output AudioFormat is needed.

2
libraries/transformer/src/main/java/androidx/media3/transformer/EncodedSampleExporter.java
View File

@ -129,7 +129,7 @@ import java.util.concurrent.atomic.AtomicLong;
}
@Override
public GraphInput getInput(EditedMediaItem item, Format format) {
public GraphInput getInput(EditedMediaItem item, Format format, int sequenceIndex) {
return this;
}

4
libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java
View File

@ -64,10 +64,10 @@ import java.util.List;
*
* @param editedMediaItem The initial {@link EditedMediaItem} of the input.
* @param format The initial {@link Format} of the input.
* @param sequenceIndex The sequence index of the input.
* @throws ExportException If an error occurs getting the input.
*/
public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format)
throws ExportException;
public abstract GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException;
/**
* Processes the input data and returns whether it may be possible to process more data by calling

2
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -637,7 +637,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
GraphInput sampleExporterInput =
sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat);
sampleExporter.getInput(firstEditedMediaItem, assetLoaderOutputFormat, sequenceIndex);
OnMediaItemChangedListener onMediaItemChangedListener =
(editedMediaItem, durationUs, decodedFormat, isLast) -> {
onMediaItemChanged(trackType, durationUs, isLast);

5
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java
View File

@ -22,6 +22,7 @@ import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoGraph;
import androidx.media3.common.util.Log;
import androidx.media3.effect.MultipleInputVideoGraph;
import androidx.media3.effect.VideoCompositorSettings;
import java.util.List;
@ -79,8 +80,8 @@ import java.util.concurrent.Executor;
}
@Override
public GraphInput createInput() throws VideoFrameProcessingException {
int inputId = registerInput();
public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException {
int inputId = registerInput(sequenceIndex);
return new VideoFrameProcessingWrapper(
getProcessor(inputId), /* presentation= */ null, getInitialTimestampOffsetUs());
}

11
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java
View File

@ -106,12 +106,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public GraphInput createInput() throws VideoFrameProcessingException {
public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException {
checkState(videoFrameProcessingWrapper == null);
int inputId = registerInput();
int inputId = registerInput(sequenceIndex);
videoFrameProcessingWrapper =
new VideoFrameProcessingWrapper(
getProcessor(inputId), getPresentation(), getInitialTimestampOffsetUs());
new VideoFrameProcessingWrapper(
getProcessor(inputId),
getInputColorInfo(),
getPresentation(),
getInitialTimestampOffsetUs());
return videoFrameProcessingWrapper;
}
}

4
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java
View File

@ -69,6 +69,8 @@ import java.util.concurrent.Executor;
* <p>This method must called exactly once for every input stream.
*
* <p>If the method throws any {@link Exception}, the caller must call {@link #release}.
*
* @param sequenceIndex The sequence index of the input, which can aid ordering of the inputs.
*/
GraphInput createInput() throws VideoFrameProcessingException;
GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException;
}

16
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java
View File

@ -50,6 +50,7 @@ import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.effect.DebugTraceUtil;
import androidx.media3.effect.VideoCompositorSettings;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.MoreExecutors;
import java.nio.ByteBuffer;
@ -58,6 +59,8 @@ import java.util.Objects;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.dataflow.qual.Pure;
import java.nio.ByteBuffer;
import java.util.List;
/** Processes, encodes and muxes raw video frames. */
/* package */ final class VideoSampleExporter extends SampleExporter {
@ -153,10 +156,9 @@ import org.checkerframework.dataflow.qual.Pure;
}
@Override
public GraphInput getInput(EditedMediaItem editedMediaItem, Format format)
throws ExportException {
public GraphInput getInput(EditedMediaItem editedMediaItem, Format format, int sequenceIndex) throws ExportException {
try {
return videoGraph.createInput();
return videoGraph.createInput(sequenceIndex);
} catch (VideoFrameProcessingException e) {
throw ExportException.createForVideoFrameProcessingException(e);
}
@ -540,8 +542,8 @@ import org.checkerframework.dataflow.qual.Pure;
}
@Override
public int registerInput() throws VideoFrameProcessingException {
return videoGraph.registerInput();
public int registerInput(int sequenceIndex) throws VideoFrameProcessingException {
return videoGraph.registerInput(sequenceIndex);
}
@Override
@ -550,8 +552,8 @@ import org.checkerframework.dataflow.qual.Pure;
}
@Override
public GraphInput createInput() throws VideoFrameProcessingException {
return videoGraph.createInput();
public GraphInput createInput(int sequenceIndex) throws VideoFrameProcessingException {
return videoGraph.createInput(sequenceIndex);
}
@Override