loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move Single/MultiVideoGraph impl to effect

Browse Source 
PiperOrigin-RevId: 569188658
This commit is contained in:
claincly 2023-09-28 08:24:21 -07:00 committed by Copybara-Service
parent d716de02aa
commit cd6f8a42a5
7 changed files with 263 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
libraries/common/src/main/java/androidx/media3/common/VideoGraph.java
View File

@ -24,6 +24,7 @@ import androidx.media3.common.util.UnstableApi;
public interface VideoGraph {
/** Listener for video frame processing events. */
@UnstableApi
interface Listener {
/**
* Called when the output size changes.
@ -58,6 +59,26 @@ public interface VideoGraph {
*/
void initialize() throws VideoFrameProcessingException;
/**
* Registers a new input to the {@code VideoGraph}.
*
* <p>A underlying processing {@link VideoFrameProcessor} is created every time this method is
* called.
*
* <p>If the method throws, the caller must call {@link #release}.
*
* @return The id of the registered input, which can be used to get the underlying {@link
* VideoFrameProcessor} via {@link #getProcessor(int)}.
*/
int registerInput() throws VideoFrameProcessingException;
/**
* Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via
* {@link #registerInput()}. If the {@code inputId} is not {@linkplain #registerInput()
* registered} before, this method will throw an {@link IllegalStateException}.
*/
VideoFrameProcessor getProcessor(int inputId);
/**
* Sets the output surface and supporting information.
*

153
libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java → libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package androidx.media3.transformer;
package androidx.media3.effect;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID;
import static androidx.media3.common.util.Assertions.checkNotNull;
@ -45,12 +45,7 @@ import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
import androidx.media3.common.util.GlUtil;
import androidx.media3.effect.DefaultGlObjectsProvider;
import androidx.media3.effect.DefaultVideoCompositor;
import androidx.media3.effect.DefaultVideoFrameProcessor;
import androidx.media3.effect.GlTextureProducer;
import androidx.media3.effect.VideoCompositor;
import androidx.media3.effect.VideoCompositorSettings;
import androidx.media3.common.util.UnstableApi;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.ArrayDeque;
import java.util.ArrayList;
@ -61,32 +56,8 @@ import java.util.concurrent.ExecutorService;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** A {@link VideoGraph} that handles multiple input streams. */
/* package */ final class MultipleInputVideoGraph implements TransformerVideoGraph {
public static final class Factory implements TransformerVideoGraph.Factory {
@Override
public MultipleInputVideoGraph create(
Context context,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
DebugViewProvider debugViewProvider,
Listener listener,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
return new MultipleInputVideoGraph(
context,
inputColorInfo,
outputColorInfo,
debugViewProvider,
listener,
listenerExecutor,
videoCompositorSettings,
compositionEffects,
initialTimestampOffsetUs);
}
}
@UnstableApi
public abstract class MultipleInputVideoGraph implements VideoGraph {
private static final String SHARED_EXECUTOR_NAME = "Transformer:MultipleInputVideoGraph:Thread";
@ -95,15 +66,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final int COMPOSITOR_TEXTURE_OUTPUT_CAPACITY = 1;
private final Context context;
private final ColorInfo inputColorInfo;
private final ColorInfo outputColorInfo;
private final GlObjectsProvider glObjectsProvider;
private final DebugViewProvider debugViewProvider;
private final Listener listener;
private final VideoGraph.Listener listener;
private final Executor listenerExecutor;
private final VideoCompositorSettings videoCompositorSettings;
private final List<Effect> compositionEffects;
private final List<VideoFrameProcessingWrapper> preProcessingWrappers;
private final List<VideoFrameProcessor> preProcessors;
private final ExecutorService sharedExecutorService;
@ -124,12 +96,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private volatile boolean hasProducedFrameWithTimestampZero;
private MultipleInputVideoGraph(
protected MultipleInputVideoGraph(
Context context,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
DebugViewProvider debugViewProvider,
Listener listener,
VideoGraph.Listener listener,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
@ -144,7 +116,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.compositionEffects = new ArrayList<>(compositionEffects);
this.initialTimestampOffsetUs = initialTimestampOffsetUs;
lastRenderedPresentationTimeUs = C.TIME_UNSET;
preProcessingWrappers = new ArrayList<>();
preProcessors = new ArrayList<>();
sharedExecutorService = newSingleThreadScheduledExecutor(SHARED_EXECUTOR_NAME);
glObjectsProvider = new SingleContextGlObjectsProvider();
// TODO - b/289986435: Support injecting VideoFrameProcessor.Factory.
@ -165,7 +137,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void initialize() throws VideoFrameProcessingException {
checkState(
preProcessingWrappers.isEmpty()
preProcessors.isEmpty()
&& videoCompositor == null
&& compositionVideoFrameProcessor == null
&& !released);
@ -243,56 +215,61 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public GraphInput createInput() throws VideoFrameProcessingException {
public int registerInput() throws VideoFrameProcessingException {
checkStateNotNull(videoCompositor);
int videoCompositorInputId = videoCompositor.registerInputSource();
// Creating a new VideoFrameProcessor for the input.
VideoFrameProcessingWrapper preProcessingVideoFrameProcessorWrapper =
new VideoFrameProcessingWrapper(
context,
videoFrameProcessorFactory
.buildUpon()
.setTextureOutput(
// Texture output to compositor.
(textureProducer, texture, presentationTimeUs, syncObject) ->
queuePreProcessingOutputToCompositor(
videoCompositorInputId, textureProducer, texture, presentationTimeUs),
PRE_COMPOSITOR_TEXTURE_OUTPUT_CAPACITY)
.build(),
inputColorInfo,
outputColorInfo,
DebugViewProvider.NONE,
listenerExecutor,
new VideoFrameProcessor.Listener() {
// All of this listener's methods are called on the sharedExecutorService.
@Override
public void onInputStreamRegistered(
@VideoFrameProcessor.InputType int inputType,
List<Effect> effects,
FrameInfo frameInfo) {}
VideoFrameProcessor preProcessor =
videoFrameProcessorFactory
.buildUpon()
.setTextureOutput(
// Texture output to compositor.
(textureProducer, texture, presentationTimeUs, syncObject) ->
queuePreProcessingOutputToCompositor(
videoCompositorInputId, textureProducer, texture, presentationTimeUs),
PRE_COMPOSITOR_TEXTURE_OUTPUT_CAPACITY)
.build()
.create(
context,
DebugViewProvider.NONE,
inputColorInfo,
outputColorInfo,
// Pre-processors render frames as soon as available, to VideoCompositor.
/* renderFramesAutomatically= */ true,
listenerExecutor,
new VideoFrameProcessor.Listener() {
// All of this listener's methods are called on the sharedExecutorService.
@Override
public void onInputStreamRegistered(
@VideoFrameProcessor.InputType int inputType,
List<Effect> effects,
FrameInfo frameInfo) {}
@Override
public void onOutputSizeChanged(int width, int height) {}
@Override
public void onOutputSizeChanged(int width, int height) {}
@Override
public void onOutputFrameAvailableForRendering(long presentationTimeUs) {}
@Override
public void onOutputFrameAvailableForRendering(long presentationTimeUs) {}
@Override
public void onError(VideoFrameProcessingException exception) {
handleVideoFrameProcessingException(exception);
}
@Override
public void onError(VideoFrameProcessingException exception) {
handleVideoFrameProcessingException(exception);
}
@Override
public void onEnded() {
onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId);
}
},
/* renderFramesAutomatically= */ true,
/* presentation= */ null,
initialTimestampOffsetUs);
preProcessingWrappers.add(preProcessingVideoFrameProcessorWrapper);
return preProcessingVideoFrameProcessorWrapper;
@Override
public void onEnded() {
onPreProcessingVideoFrameProcessorEnded(videoCompositorInputId);
}
});
preProcessors.add(preProcessor);
return videoCompositorInputId;
}
@Override
public VideoFrameProcessor getProcessor(int inputId) {
checkState(inputId < preProcessors.size());
return preProcessors.get(inputId);
}
@Override
@ -312,10 +289,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
// Needs to release the frame processors before their internal executor services are released.
for (int i = 0; i < preProcessingWrappers.size(); i++) {
preProcessingWrappers.get(i).release();
for (int i = 0; i < preProcessors.size(); i++) {
preProcessors.get(i).release();
}
preProcessingWrappers.clear();
preProcessors.clear();
if (videoCompositor != null) {
videoCompositor.release();
@ -338,6 +315,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
released = true;
}
protected ColorInfo getInputColorInfo() {
return inputColorInfo;
}
protected long getInitialTimestampOffsetUs() {
return initialTimestampOffsetUs;
}
// This method is called on the sharedExecutorService.
private void queuePreProcessingOutputToCompositor(
int videoCompositorInputId,

63
libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java → libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package androidx.media3.transformer;
package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
@ -30,13 +30,16 @@ import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.VideoCompositorSettings;
import androidx.media3.common.util.UnstableApi;
import java.util.List;
import java.util.concurrent.Executor;
/** A {@link VideoGraph} that handles one input stream. */
/* package */ abstract class SingleInputVideoGraph implements VideoGraph {
@UnstableApi
public abstract class SingleInputVideoGraph implements VideoGraph {
/** The ID {@link #registerInput()} returns. */
public static final int SINGLE_INPUT_INDEX = 0;
private final Context context;
private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
@ -46,10 +49,11 @@ import java.util.concurrent.Executor;
private final DebugViewProvider debugViewProvider;
private final Executor listenerExecutor;
private final boolean renderFramesAutomatically;
private final long initialTimestampOffsetUs;
@Nullable private final Presentation presentation;
@Nullable private VideoFrameProcessingWrapper videoFrameProcessingWrapper;
@Nullable private VideoFrameProcessor videoFrameProcessor;
private boolean released;
private volatile boolean hasProducedFrameWithTimestampZero;
@ -93,16 +97,21 @@ import java.util.concurrent.Executor;
* <p>This method must be called at most once.
*/
@Override
public void initialize() throws VideoFrameProcessingException {
checkStateNotNull(videoFrameProcessingWrapper == null && !released);
public void initialize() {
// Initialization is deferred to registerInput();
}
videoFrameProcessingWrapper =
new VideoFrameProcessingWrapper(
@Override
public int registerInput() throws VideoFrameProcessingException {
checkStateNotNull(videoFrameProcessor == null && !released);
videoFrameProcessor =
videoFrameProcessorFactory.create(
context,
videoFrameProcessorFactory,
debugViewProvider,
inputColorInfo,
outputColorInfo,
debugViewProvider,
renderFramesAutomatically,
listenerExecutor,
new VideoFrameProcessor.Listener() {
private long lastProcessedFramePresentationTimeUs;
@ -136,15 +145,18 @@ import java.util.concurrent.Executor;
public void onEnded() {
listener.onEnded(lastProcessedFramePresentationTimeUs);
}
},
renderFramesAutomatically,
presentation,
initialTimestampOffsetUs);
});
return SINGLE_INPUT_INDEX;
}
@Override
public VideoFrameProcessor getProcessor(int inputId) {
return checkStateNotNull(videoFrameProcessor);
}
@Override
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
checkNotNull(videoFrameProcessingWrapper).setOutputSurfaceInfo(outputSurfaceInfo);
checkNotNull(videoFrameProcessor).setOutputSurfaceInfo(outputSurfaceInfo);
}
@Override
@ -158,14 +170,23 @@ import java.util.concurrent.Executor;
return;
}
if (videoFrameProcessingWrapper != null) {
videoFrameProcessingWrapper.release();
videoFrameProcessingWrapper = null;
if (videoFrameProcessor != null) {
videoFrameProcessor.release();
videoFrameProcessor = null;
}
released = true;
}
protected VideoFrameProcessingWrapper getVideoFrameProcessingWrapper() {
return checkNotNull(videoFrameProcessingWrapper);
protected ColorInfo getInputColorInfo() {
return inputColorInfo;
}
protected long getInitialTimestampOffsetUs() {
return initialTimestampOffsetUs;
}
@Nullable
protected Presentation getPresentation() {
return presentation;
}
}

94
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import android.content.Context;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoGraph;
import androidx.media3.effect.MultipleInputVideoGraph;
import androidx.media3.effect.VideoCompositorSettings;
import java.util.List;
import java.util.concurrent.Executor;
/**
* A {@link TransformerVideoGraph Transformer}-specific implementation of {@link
* MultipleInputVideoGraph}.
*/
/* package */ final class TransformerMultipleInputVideoGraph extends MultipleInputVideoGraph
implements TransformerVideoGraph {
/** A factory for creating {@link TransformerMultipleInputVideoGraph} instances. */
public static final class Factory implements TransformerVideoGraph.Factory {
@Override
public TransformerMultipleInputVideoGraph create(
Context context,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
DebugViewProvider debugViewProvider,
VideoGraph.Listener listener,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
return new TransformerMultipleInputVideoGraph(
context,
inputColorInfo,
outputColorInfo,
debugViewProvider,
listener,
listenerExecutor,
videoCompositorSettings,
compositionEffects,
initialTimestampOffsetUs);
}
}
private TransformerMultipleInputVideoGraph(
Context context,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
DebugViewProvider debugViewProvider,
Listener listener,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
super(
context,
inputColorInfo,
outputColorInfo,
debugViewProvider,
listener,
listenerExecutor,
videoCompositorSettings,
compositionEffects,
initialTimestampOffsetUs);
}
@Override
public GraphInput createInput() throws VideoFrameProcessingException {
int inputId = registerInput();
return new VideoFrameProcessingWrapper(
getProcessor(inputId),
getInputColorInfo(),
/* presentation= */ null,
getInitialTimestampOffsetUs());
}
}

27
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java
View File

@ -16,21 +16,30 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import androidx.annotation.Nullable;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.SingleInputVideoGraph;
import androidx.media3.effect.VideoCompositorSettings;
import java.util.List;
import java.util.concurrent.Executor;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A {@link TransformerVideoGraph Transformer}-specific implementation of {@link
* SingleInputVideoGraph}.
*/
/* package */ final class TransformerSingleInputVideoGraph extends SingleInputVideoGraph
implements TransformerVideoGraph {
/** A factory for creating a {@link SingleInputVideoGraph}. */
/** A factory for creating {@link TransformerSingleInputVideoGraph} instances. */
public static final class Factory implements TransformerVideoGraph.Factory {
private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
@ -40,7 +49,7 @@ import java.util.concurrent.Executor;
}
@Override
public TransformerVideoGraph create(
public TransformerSingleInputVideoGraph create(
Context context,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
@ -72,6 +81,8 @@ import java.util.concurrent.Executor;
}
}
private @MonotonicNonNull VideoFrameProcessingWrapper videoFrameProcessingWrapper;
private TransformerSingleInputVideoGraph(
Context context,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
@ -99,7 +110,15 @@ import java.util.concurrent.Executor;
}
@Override
public GraphInput createInput() {
return getVideoFrameProcessingWrapper();
public GraphInput createInput() throws VideoFrameProcessingException {
checkState(videoFrameProcessingWrapper == null);
int inputId = registerInput();
videoFrameProcessingWrapper =
new VideoFrameProcessingWrapper(
getProcessor(inputId),
getInputColorInfo(),
getPresentation(),
getInitialTimestampOffsetUs());
return videoFrameProcessingWrapper;
}
}

26
libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java
View File

@ -21,26 +21,22 @@ import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID;
import static androidx.media3.common.util.Assertions.checkNotNull;
import android.content.Context;
import android.graphics.Bitmap;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.OnInputFrameProcessedListener;
import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.TimestampIterator;
import androidx.media3.effect.Presentation;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicLong;
/** A wrapper for {@link VideoFrameProcessor} that handles {@link GraphInput} events. */
@ -52,31 +48,15 @@ import java.util.concurrent.atomic.AtomicLong;
@Nullable final Presentation presentation;
public VideoFrameProcessingWrapper(
Context context,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
VideoFrameProcessor videoFrameProcessor,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
DebugViewProvider debugViewProvider,
Executor listenerExecutor,
VideoFrameProcessor.Listener listener,
boolean renderFramesAutomatically,
@Nullable Presentation presentation,
long initialTimestampOffsetUs)
throws VideoFrameProcessingException {
long initialTimestampOffsetUs) {
this.videoFrameProcessor = videoFrameProcessor;
this.mediaItemOffsetUs = new AtomicLong();
this.inputColorInfo = inputColorInfo;
this.initialTimestampOffsetUs = initialTimestampOffsetUs;
this.presentation = presentation;
videoFrameProcessor =
videoFrameProcessorFactory.create(
context,
debugViewProvider,
inputColorInfo,
outputColorInfo,
renderFramesAutomatically,
listenerExecutor,
listener);
}
@Override

12
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java
View File

@ -147,7 +147,7 @@ import org.checkerframework.dataflow.qual.Pure;
new VideoGraphWrapper(
context,
hasMultipleInputs
? new MultipleInputVideoGraph.Factory()
? new TransformerMultipleInputVideoGraph.Factory()
: new TransformerSingleInputVideoGraph.Factory(videoFrameProcessorFactory),
videoGraphInputColor,
videoGraphOutputColor,
@ -533,6 +533,16 @@ import org.checkerframework.dataflow.qual.Pure;
videoGraph.initialize();
}
@Override
public int registerInput() throws VideoFrameProcessingException {
return videoGraph.registerInput();
}
@Override
public VideoFrameProcessor getProcessor(int inputId) {
return videoGraph.getProcessor(inputId);
}
@Override
public GraphInput createInput() throws VideoFrameProcessingException {
return videoGraph.createInput();