From d9563b133e1a29f51ea81ee0d06da8a5a4ec3982 Mon Sep 17 00:00:00 2001 From: claincly Date: Fri, 22 Sep 2023 05:58:49 -0700 Subject: [PATCH] Split VideoGraph interface and move VideoGraph to common PiperOrigin-RevId: 567599249 --- .../androidx/media3/common}/VideoGraph.java | 55 +--------- .../transformer/MultipleInputVideoGraph.java | 6 +- .../transformer/SingleInputVideoGraph.java | 62 ++--------- .../TransformerSingleInputVideoGraph.java | 100 ++++++++++++++++++ .../transformer/TransformerVideoGraph.java | 71 +++++++++++++ .../transformer/VideoSampleExporter.java | 11 +- 6 files changed, 192 insertions(+), 113 deletions(-) rename libraries/{transformer/src/main/java/androidx/media3/transformer => common/src/main/java/androidx/media3/common}/VideoGraph.java (59%) create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java similarity index 59% rename from libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java rename to libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 115e767af2..58dbc029f6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -14,49 +14,14 @@ * limitations under the License. */ -package androidx.media3.transformer; +package androidx.media3.common; -import android.content.Context; import androidx.annotation.Nullable; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.SurfaceInfo; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import java.util.List; -import java.util.concurrent.Executor; +import androidx.media3.common.util.UnstableApi; /** Represents a graph for processing decoded video frames. */ -/* package */ interface VideoGraph { - - /** A factory for creating a {@link VideoGraph}. */ - interface Factory { - /** - * Creates a new {@link VideoGraph} instance. - * - * @param context A {@link Context}. - * @param inputColorInfo The {@link ColorInfo} for the input frames. - * @param outputColorInfo The {@link ColorInfo} for the output frames. - * @param debugViewProvider A {@link DebugViewProvider}. - * @param listener A {@link Listener}. - * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked. - * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition. - * @return A new instance. - * @throws VideoFrameProcessingException If a problem occurs while creating the {@link - * VideoFrameProcessor}. - */ - VideoGraph create( - Context context, - ColorInfo inputColorInfo, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - List compositionEffects, - long initialTimestampOffsetUs) - throws VideoFrameProcessingException; - } +@UnstableApi +public interface VideoGraph { /** Listener for video frame processing events. */ interface Listener { @@ -89,18 +54,6 @@ import java.util.concurrent.Executor; */ void initialize() throws VideoFrameProcessingException; - /** - * Returns a {@link GraphInput} object to which the {@code VideoGraph} inputs are queued. - * - *

This method must be called after successfully {@linkplain #initialize() initializing} the - * {@code VideoGraph}. - * - *

This method must called exactly once for every input stream. - * - *

If the method throws any {@link Exception}, the caller must call {@link #release}. - */ - GraphInput createInput() throws VideoFrameProcessingException; - /** * Sets the output surface and supporting information. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java index 9cc93894a0..cecfa85f66 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java @@ -43,6 +43,7 @@ import androidx.media3.common.GlTextureInfo; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoGraph; import androidx.media3.common.util.GlUtil; import androidx.media3.effect.DefaultGlObjectsProvider; import androidx.media3.effect.DefaultVideoCompositor; @@ -60,10 +61,9 @@ import java.util.concurrent.ExecutorService; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** A {@link VideoGraph} that handles multiple input streams. */ -/* package */ final class MultipleInputVideoGraph implements VideoGraph { - - public static final class Factory implements VideoGraph.Factory { +/* package */ final class MultipleInputVideoGraph implements TransformerVideoGraph { + public static final class Factory implements TransformerVideoGraph.Factory { @Override public MultipleInputVideoGraph create( Context context, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java index 1471e32e4f..b8dce37abb 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java @@ -28,52 +28,13 @@ import androidx.media3.common.FrameInfo; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoGraph; import androidx.media3.effect.Presentation; import java.util.List; import java.util.concurrent.Executor; /** A {@link VideoGraph} that handles one input stream. */ -/* package */ final class SingleInputVideoGraph implements VideoGraph { - - /** A factory for creating a {@link SingleInputVideoGraph}. */ - public static final class Factory implements VideoGraph.Factory { - - private final VideoFrameProcessor.Factory videoFrameProcessorFactory; - - public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) { - this.videoFrameProcessorFactory = videoFrameProcessorFactory; - } - - @Override - public VideoGraph create( - Context context, - ColorInfo inputColorInfo, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - List compositionEffects, - long initialTimestampOffsetUs) { - @Nullable Presentation presentation = null; - for (int i = 0; i < compositionEffects.size(); i++) { - Effect effect = compositionEffects.get(i); - if (effect instanceof Presentation) { - presentation = (Presentation) effect; - } - } - return new SingleInputVideoGraph( - context, - videoFrameProcessorFactory, - inputColorInfo, - outputColorInfo, - listener, - debugViewProvider, - listenerExecutor, - /* renderFramesAutomatically= */ true, - presentation, - initialTimestampOffsetUs); - } - } +/* package */ abstract class SingleInputVideoGraph implements VideoGraph { private final Context context; private final VideoFrameProcessor.Factory videoFrameProcessorFactory; @@ -91,7 +52,7 @@ import java.util.concurrent.Executor; private boolean released; private volatile boolean hasProducedFrameWithTimestampZero; - private SingleInputVideoGraph( + public SingleInputVideoGraph( Context context, VideoFrameProcessor.Factory videoFrameProcessorFactory, ColorInfo inputColorInfo, @@ -120,17 +81,7 @@ import java.util.concurrent.Executor; *

This method must be called at most once. */ @Override - public void initialize() { - // Initialization is deferred to createInput(). - } - - /** - * {@inheritDoc} - * - *

This method must only be called once. - */ - @Override - public GraphInput createInput() throws VideoFrameProcessingException { + public void initialize() throws VideoFrameProcessingException { checkStateNotNull(videoFrameProcessingWrapper == null && !released); videoFrameProcessingWrapper = @@ -177,7 +128,6 @@ import java.util.concurrent.Executor; renderFramesAutomatically, presentation, initialTimestampOffsetUs); - return videoFrameProcessingWrapper; } @Override @@ -202,4 +152,8 @@ import java.util.concurrent.Executor; } released = true; } + + protected VideoFrameProcessingWrapper getVideoFrameProcessingWrapper() { + return checkNotNull(videoFrameProcessingWrapper); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java new file mode 100644 index 0000000000..cc5c7619b0 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -0,0 +1,100 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import android.content.Context; +import androidx.annotation.Nullable; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.effect.Presentation; +import java.util.List; +import java.util.concurrent.Executor; + +/* package */ final class TransformerSingleInputVideoGraph extends SingleInputVideoGraph + implements TransformerVideoGraph { + + /** A factory for creating a {@link SingleInputVideoGraph}. */ + public static final class Factory implements TransformerVideoGraph.Factory { + + private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + + public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) { + this.videoFrameProcessorFactory = videoFrameProcessorFactory; + } + + @Override + public TransformerVideoGraph create( + Context context, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + Listener listener, + Executor listenerExecutor, + List compositionEffects, + long initialTimestampOffsetUs) { + @Nullable Presentation presentation = null; + for (int i = 0; i < compositionEffects.size(); i++) { + Effect effect = compositionEffects.get(i); + if (effect instanceof Presentation) { + presentation = (Presentation) effect; + } + } + return new TransformerSingleInputVideoGraph( + context, + videoFrameProcessorFactory, + inputColorInfo, + outputColorInfo, + listener, + debugViewProvider, + listenerExecutor, + /* renderFramesAutomatically= */ true, + presentation, + initialTimestampOffsetUs); + } + } + + private TransformerSingleInputVideoGraph( + Context context, + VideoFrameProcessor.Factory videoFrameProcessorFactory, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + Listener listener, + DebugViewProvider debugViewProvider, + Executor listenerExecutor, + boolean renderFramesAutomatically, + @Nullable Presentation presentation, + long initialTimestampOffsetUs) { + super( + context, + videoFrameProcessorFactory, + inputColorInfo, + outputColorInfo, + listener, + debugViewProvider, + listenerExecutor, + renderFramesAutomatically, + presentation, + initialTimestampOffsetUs); + } + + @Override + public GraphInput createInput() { + return getVideoFrameProcessingWrapper(); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java new file mode 100644 index 0000000000..f85598b474 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java @@ -0,0 +1,71 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import android.content.Context; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoGraph; +import java.util.List; +import java.util.concurrent.Executor; + +/** The {@link VideoGraph} to support {@link Transformer} specific use cases. */ +/* package */ interface TransformerVideoGraph extends VideoGraph { + + /** A factory for creating a {@link TransformerVideoGraph}. */ + interface Factory { + /** + * Creates a new {@link TransformerVideoGraph} instance. + * + * @param context A {@link Context}. + * @param inputColorInfo The {@link ColorInfo} for the input frames. + * @param outputColorInfo The {@link ColorInfo} for the output frames. + * @param debugViewProvider A {@link DebugViewProvider}. + * @param listener A {@link Listener}. + * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked. + * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition. + * @return A new instance. + * @throws VideoFrameProcessingException If a problem occurs while creating the {@link + * VideoFrameProcessor}. + */ + TransformerVideoGraph create( + Context context, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + Listener listener, + Executor listenerExecutor, + List compositionEffects, + long initialTimestampOffsetUs) + throws VideoFrameProcessingException; + } + + /** + * Returns a {@link GraphInput} object to which the {@code VideoGraph} inputs are queued. + * + *

This method must be called after successfully {@linkplain #initialize() initializing} the + * {@code VideoGraph}. + * + *

This method must called exactly once for every input stream. + * + *

If the method throws any {@link Exception}, the caller must call {@link #release}. + */ + GraphInput createInput() throws VideoFrameProcessingException; +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index f0af001f5d..75e8f34723 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -42,6 +42,7 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoGraph; import androidx.media3.common.util.Consumer; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; @@ -60,7 +61,7 @@ import org.checkerframework.dataflow.qual.Pure; /* package */ final class VideoSampleExporter extends SampleExporter { private static final String TAG = "VideoSampleExporter"; - private final VideoGraph videoGraph; + private final TransformerVideoGraph videoGraph; private final EncoderWrapper encoderWrapper; private final DecoderInputBuffer encoderOutputBuffer; private final long initialTimestampOffsetUs; @@ -145,7 +146,7 @@ import org.checkerframework.dataflow.qual.Pure; context, hasMultipleInputs ? new MultipleInputVideoGraph.Factory() - : new SingleInputVideoGraph.Factory(videoFrameProcessorFactory), + : new TransformerSingleInputVideoGraph.Factory(videoFrameProcessorFactory), videoGraphInputColor, videoGraphOutputColor, errorConsumer, @@ -464,14 +465,14 @@ import org.checkerframework.dataflow.qual.Pure; } } - private final class VideoGraphWrapper implements VideoGraph, VideoGraph.Listener { + private final class VideoGraphWrapper implements TransformerVideoGraph, VideoGraph.Listener { - private final VideoGraph videoGraph; + private final TransformerVideoGraph videoGraph; private final Consumer errorConsumer; public VideoGraphWrapper( Context context, - VideoGraph.Factory videoGraphFactory, + TransformerVideoGraph.Factory videoGraphFactory, ColorInfo videoFrameProcessorInputColor, ColorInfo videoFrameProcessorOutputColor, Consumer errorConsumer,