From 44b3a43652c0fb462cd1799d253d663e5207d268 Mon Sep 17 00:00:00 2001 From: claincly Date: Mon, 10 Mar 2025 11:28:28 -0700 Subject: [PATCH] Use VideoGraph as top level component ...and reduce the number of classes/interfaces PiperOrigin-RevId: 735451687 --- .../media3/common/PreviewingVideoGraph.java | 80 ------ .../media3/common/VideoFrameProcessor.java | 2 +- .../androidx/media3/common/VideoGraph.java | 161 ++++++++++- .../effect/MultipleInputVideoGraph.java | 140 +++++++-- .../PreviewingMultipleInputVideoGraph.java | 104 ------- .../PreviewingSingleInputVideoGraph.java | 110 ------- .../media3/effect/SingleInputVideoGraph.java | 160 +++++++++-- libraries/exoplayer/proguard-rules.txt | 4 +- .../media3/exoplayer/ExoPlayerImpl.java | 2 +- .../video/PlaybackVideoGraphWrapper.java | 120 ++++---- .../video/PlaybackVideoGraphWrapperTest.java | 125 +++----- ...mpositionMultipleSequencePlaybackTest.java | 4 +- .../CompositionPlayerSeekTest.java | 24 +- .../transformer/CompositionPlayerTest.java | 190 +++---------- .../CompositionPlayerPixelTest.java | 8 +- .../media3/transformer/CompositionPlayer.java | 29 +- .../TransformerMultipleInputVideoGraph.java | 113 -------- .../TransformerSingleInputVideoGraph.java | 116 -------- .../transformer/TransformerVideoGraph.java | 96 ------- .../VideoFrameProcessingWrapper.java | 168 ----------- .../transformer/VideoSampleExporter.java | 268 +++++++++++++----- 21 files changed, 779 insertions(+), 1245 deletions(-) delete mode 100644 libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java delete mode 100644 libraries/effect/src/main/java/androidx/media3/effect/PreviewingMultipleInputVideoGraph.java delete mode 100644 libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java delete mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java delete mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java delete mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java delete mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java diff --git a/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java deleted file mode 100644 index dafbcdd0bb..0000000000 --- a/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.common; - -import android.content.Context; -import androidx.media3.common.util.UnstableApi; -import java.util.List; -import java.util.concurrent.Executor; - -/** A {@link VideoGraph} specific to previewing. */ -@UnstableApi -public interface PreviewingVideoGraph extends VideoGraph { - - /** A factory for creating a {@link PreviewingVideoGraph}. */ - interface Factory { - /** - * Creates a new {@link PreviewingVideoGraph} instance. - * - * @param context A {@link Context}. - * @param outputColorInfo The {@link ColorInfo} for the output frames. - * @param debugViewProvider A {@link DebugViewProvider}. - * @param listener A {@link Listener}. - * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked. - * @param videoCompositorSettings The {@link VideoCompositorSettings}. - * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition. - * @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds. - * @return A new instance. - * @throws VideoFrameProcessingException If a problem occurs while creating the {@link - * VideoFrameProcessor}. - */ - PreviewingVideoGraph create( - Context context, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs) - throws VideoFrameProcessingException; - - /** - * Returns whether the {@link VideoGraph} implementation supports {@linkplain #registerInput - * registering} multiple inputs. - */ - boolean supportsMultipleInputs(); - } - - /** - * Renders the oldest unrendered output frame that has become {@linkplain - * Listener#onOutputFrameAvailableForRendering(long) available for rendering} at the given {@code - * renderTimeNs}. - * - *

This will either render the output frame to the {@linkplain #setOutputSurfaceInfo output - * surface}, or drop the frame, per {@code renderTimeNs}. - * - *

The {@code renderTimeNs} may be passed to {@link - * android.opengl.EGLExt#eglPresentationTimeANDROID} depending on the implementation. - * - * @param renderTimeNs The render time to use for the frame, in nanoseconds. The render time can - * be before or after the current system time. Use {@link - * VideoFrameProcessor#DROP_OUTPUT_FRAME} to drop the frame, or {@link - * VideoFrameProcessor#RENDER_OUTPUT_FRAME_IMMEDIATELY} to render the frame immediately. - */ - void renderOutputFrame(long renderTimeNs); -} diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 880c7480e2..013bae75d7 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -288,7 +288,7 @@ public interface VideoFrameProcessor { * *

After registering the first input stream, this method must only be called after the last * frame of the already-registered input stream has been {@linkplain #registerInputFrame - * registered}, last bitmap {@link #queueInputBitmap queued} or last texture id {@linkplain + * registered}, last bitmap {@linkplain #queueInputBitmap queued} or last texture id {@linkplain * #queueInputTexture queued}. * *

This method blocks the calling thread until the previous calls to this method finish, that diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 460bb0f4bf..1e3502305c 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -16,16 +16,61 @@ package androidx.media3.common; +import android.content.Context; +import android.graphics.Bitmap; +import android.view.Surface; import androidx.annotation.IntRange; import androidx.annotation.Nullable; +import androidx.media3.common.VideoFrameProcessor.InputType; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; +import java.util.List; +import java.util.concurrent.Executor; /** Represents a graph for processing raw video frames. */ @UnstableApi public interface VideoGraph { + /** A factory for {@link VideoGraph} instances. */ + interface Factory { + /** + * Creates a new {@link VideoGraph} instance. + * + * @param context A {@link Context}. + * @param outputColorInfo The {@link ColorInfo} for the output frames. + * @param debugViewProvider A {@link DebugViewProvider}. + * @param listener A {@link Listener}. + * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked. + * @param videoCompositorSettings The {@link VideoCompositorSettings} to apply to the + * composition. + * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition. + * @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds. + * @param renderFramesAutomatically If {@code true}, the instance will render output frames to + * the {@linkplain VideoGraph#setOutputSurfaceInfo(SurfaceInfo) output surface} + * automatically as the instance is done processing them. If {@code false}, the instance + * will block until {@code VideoGraph#renderOutputFrameWithMediaPresentationTime()} is + * called, to render the frame. + * @return A new instance. + */ + VideoGraph create( + Context context, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + Listener listener, + Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, + List compositionEffects, + long initialTimestampOffsetUs, + boolean renderFramesAutomatically); + + /** + * Returns whether the {@linkplain #create created} {@link VideoGraph} supports multiple video + * {@linkplain VideoGraph#registerInputStream inputs}. + */ + boolean supportsMultipleInputs(); + } + /** Listener for video frame processing events. */ - @UnstableApi interface Listener { /** * Called when the output size changes. @@ -79,11 +124,8 @@ public interface VideoGraph { /** * Registers a new input to the {@code VideoGraph}. * - *

A underlying processing {@link VideoFrameProcessor} is created every time this method is - * called. - * - *

All inputs must be registered before rendering frames to the underlying {@link - * #getProcessor(int) VideoFrameProcessor}. + *

All inputs must be registered before rendering frames by calling {@link + * #registerInputFrame}, {@link #queueInputBitmap} or {@link #queueInputTexture}. * *

If the method throws, the caller must call {@link #release}. * @@ -92,13 +134,6 @@ public interface VideoGraph { */ void registerInput(@IntRange(from = 0) int inputIndex) throws VideoFrameProcessingException; - /** - * Returns the {@link VideoFrameProcessor} that handles the processing for an input registered via - * {@link #registerInput(int)}. If the {@code inputIndex} is not {@linkplain #registerInput(int) - * registered} before, this method will throw an {@link IllegalStateException}. - */ - VideoFrameProcessor getProcessor(int inputIndex); - /** * Sets the output surface and supporting information. * @@ -116,11 +151,111 @@ public interface VideoGraph { */ void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo); + /** + * Sets a listener that's called when the {@linkplain #getInputSurface input surface} is ready to + * use at {@code inputIndex}. + */ + void setOnInputSurfaceReadyListener(int inputIndex, Runnable listener); + + /** Returns the input {@link Surface} at {@code inputIndex}. */ + Surface getInputSurface(int inputIndex); + + /** Sets the {@link OnInputFrameProcessedListener} at {@code inputIndex}. */ + void setOnInputFrameProcessedListener(int inputIndex, OnInputFrameProcessedListener listener); + + /** + * Informs the graph that a new input stream will be queued to the graph input corresponding to + * {@code inputIndex}. + * + *

After registering the first input stream, this method must only be called for the same index + * after the last frame of the already-registered input stream has been {@linkplain + * #registerInputFrame registered}, last bitmap {@linkplain #queueInputBitmap queued} or last + * texture id {@linkplain #queueInputTexture queued}. + * + *

This method blocks the calling thread until the previous input stream corresponding to the + * same {@code inputIndex} has been fully registered internally. + * + * @param inputIndex The index of the input for which a new input stream should be registered. + * This index must start from 0. + * @param inputType The {@link InputType} of the new input stream. + * @param format The {@link Format} of the new input stream. The {@link Format#colorInfo}, the + * {@link Format#width}, the {@link Format#height} and the {@link + * Format#pixelWidthHeightRatio} must be set. + * @param effects The list of {@link Effect effects} to apply to the new input stream. + * @param offsetToAddUs The offset that must be added to the frame presentation timestamps, in + * microseconds. This offset is not part of the input timestamps. It is added to the frame + * timestamps before processing, and is retained in the output timestamps. + */ + void registerInputStream( + int inputIndex, + @InputType int inputType, + Format format, + List effects, + long offsetToAddUs); + + /** + * Returns the number of pending input frames at {@code inputIndex} that has not been processed + * yet. + */ + int getPendingInputFrameCount(int inputIndex); + + /** + * Registers a new input frame at {@code inputIndex}. + * + * @see VideoFrameProcessor#registerInputFrame() + */ + boolean registerInputFrame(int inputIndex); + + /** + * Queues the input {@link Bitmap} at {@code inputIndex}. + * + * @see VideoFrameProcessor#queueInputBitmap(Bitmap, TimestampIterator) + */ + boolean queueInputBitmap(int inputIndex, Bitmap inputBitmap, TimestampIterator timestampIterator); + + /** + * Queues the input texture at {@code inputIndex}. + * + * @see VideoFrameProcessor#queueInputTexture(int, long) + */ + boolean queueInputTexture(int inputIndex, int textureId, long presentationTimeUs); + + /** + * Renders the output frame from the {@code VideoGraph}. + * + *

This method must be called only for frames that have become {@linkplain + * Listener#onOutputFrameAvailableForRendering(long) available}, calling the method renders the + * frame that becomes available the earliest but not yet rendered. + * + * @see VideoFrameProcessor#renderOutputFrame(long) + */ + void renderOutputFrame(long renderTimeNs); + + /** + * Updates an {@linkplain Listener#onOutputFrameAvailableForRendering available frame} with the + * modified effects. + */ + void redraw(); + /** * Returns whether the {@code VideoGraph} has produced a frame with zero presentation timestamp. */ boolean hasProducedFrameWithTimestampZero(); + /** + * Flushes the {@linkplain #registerInput inputs} of the {@code VideoGraph}. + * + * @see VideoFrameProcessor#flush() + */ + void flush(); + + /** + * Informs that no further inputs should be accepted at {@code inputIndex}. + * + * @see VideoFrameProcessor#signalEndOfInput() + */ + void signalEndOfInput(int inputIndex); + /** * Releases the associated resources. * diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index 3ce329e250..4cc6156cb1 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -31,10 +31,12 @@ import static androidx.media3.effect.DebugTraceUtil.EVENT_OUTPUT_TEXTURE_RENDERE import static java.util.concurrent.TimeUnit.MILLISECONDS; import android.content.Context; +import android.graphics.Bitmap; import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLSurface; import android.util.SparseArray; +import android.view.Surface; import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.media3.common.C; @@ -44,6 +46,7 @@ import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoCompositorSettings; import androidx.media3.common.VideoFrameProcessingException; @@ -51,6 +54,7 @@ import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.GlUtil.GlException; import androidx.media3.common.util.Log; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import com.google.common.util.concurrent.MoreExecutors; import java.util.ArrayDeque; @@ -64,7 +68,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** A {@link VideoGraph} that handles multiple input streams. */ @UnstableApi -public abstract class MultipleInputVideoGraph implements VideoGraph { +public final class MultipleInputVideoGraph implements VideoGraph { private static final String TAG = "MultiInputVG"; private static final String SHARED_EXECUTOR_NAME = "Effect:MultipleInputVideoGraph:Thread"; @@ -90,7 +94,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { private final Queue compositorOutputTextures; private final SparseArray compositorOutputTextureReleases; - private final long initialTimestampOffsetUs; private final boolean renderFramesAutomatically; @Nullable private VideoFrameProcessor compositionVideoFrameProcessor; @@ -104,7 +107,52 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { private volatile boolean hasProducedFrameWithTimestampZero; - protected MultipleInputVideoGraph( + /** A {@link VideoGraph.Factory} for {@link MultipleInputVideoGraph}. */ + public static final class Factory implements VideoGraph.Factory { + private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + + /** + * A {@code Factory} for {@link MultipleInputVideoGraph} that uses a {@link + * DefaultVideoFrameProcessor.Factory}. + */ + public Factory() { + this(new DefaultVideoFrameProcessor.Factory.Builder().build()); + } + + public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) { + this.videoFrameProcessorFactory = videoFrameProcessorFactory; + } + + @Override + public MultipleInputVideoGraph create( + Context context, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + Listener listener, + Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, + List compositionEffects, + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { + return new MultipleInputVideoGraph( + context, + videoFrameProcessorFactory, + outputColorInfo, + debugViewProvider, + listener, + listenerExecutor, + videoCompositorSettings, + compositionEffects, + renderFramesAutomatically); + } + + @Override + public boolean supportsMultipleInputs() { + return true; + } + } + + private MultipleInputVideoGraph( Context context, VideoFrameProcessor.Factory videoFrameProcessorFactory, ColorInfo outputColorInfo, @@ -113,7 +161,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs, boolean renderFramesAutomatically) { checkArgument(videoFrameProcessorFactory instanceof DefaultVideoFrameProcessor.Factory); this.context = context; @@ -123,7 +170,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { this.listenerExecutor = listenerExecutor; this.videoCompositorSettings = videoCompositorSettings; this.compositionEffects = new ArrayList<>(compositionEffects); - this.initialTimestampOffsetUs = initialTimestampOffsetUs; this.renderFramesAutomatically = renderFramesAutomatically; lastRenderedPresentationTimeUs = C.TIME_UNSET; preProcessors = new SparseArray<>(); @@ -268,12 +314,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { preProcessors.put(inputIndex, preProcessor); } - @Override - public VideoFrameProcessor getProcessor(int inputIndex) { - checkState(contains(preProcessors, inputIndex)); - return preProcessors.get(inputIndex); - } - @Override public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { checkNotNull(compositionVideoFrameProcessor).setOutputSurfaceInfo(outputSurfaceInfo); @@ -284,6 +324,75 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { return hasProducedFrameWithTimestampZero; } + @Override + public boolean queueInputBitmap( + int inputIndex, Bitmap inputBitmap, TimestampIterator timestampIterator) { + return getProcessor(inputIndex).queueInputBitmap(inputBitmap, timestampIterator); + } + + @Override + public boolean queueInputTexture(int inputIndex, int textureId, long presentationTimeUs) { + return getProcessor(inputIndex).queueInputTexture(textureId, presentationTimeUs); + } + + @Override + public void setOnInputFrameProcessedListener( + int inputIndex, OnInputFrameProcessedListener listener) { + getProcessor(inputIndex).setOnInputFrameProcessedListener(listener); + } + + @Override + public void setOnInputSurfaceReadyListener(int inputIndex, Runnable listener) { + getProcessor(inputIndex).setOnInputSurfaceReadyListener(listener); + } + + @Override + public Surface getInputSurface(int inputIndex) { + return getProcessor(inputIndex).getInputSurface(); + } + + @Override + public void registerInputStream( + int inputIndex, + @VideoFrameProcessor.InputType int inputType, + Format format, + List effects, + long offsetToAddUs) { + getProcessor(inputIndex).registerInputStream(inputType, format, effects, offsetToAddUs); + } + + @Override + public boolean registerInputFrame(int inputIndex) { + return getProcessor(inputIndex).registerInputFrame(); + } + + @Override + public int getPendingInputFrameCount(int inputIndex) { + return getProcessor(inputIndex).getPendingInputFrameCount(); + } + + @Override + public void renderOutputFrame(long renderTimeNs) { + checkNotNull(compositionVideoFrameProcessor).renderOutputFrame(renderTimeNs); + } + + @Override + public void redraw() { + throw new UnsupportedOperationException(); + } + + @Override + public void flush() { + for (int i = 0; i < preProcessors.size(); i++) { + preProcessors.get(preProcessors.keyAt(i)).flush(); + } + } + + @Override + public void signalEndOfInput(int inputIndex) { + getProcessor(inputIndex).signalEndOfInput(); + } + @Override public void release() { if (released) { @@ -327,12 +436,9 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { released = true; } - protected VideoFrameProcessor getCompositionVideoFrameProcessor() { - return checkStateNotNull(compositionVideoFrameProcessor); - } - - protected long getInitialTimestampOffsetUs() { - return initialTimestampOffsetUs; + private VideoFrameProcessor getProcessor(int inputIndex) { + checkState(contains(preProcessors, inputIndex)); + return preProcessors.get(inputIndex); } // This method is called on the sharedExecutorService. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingMultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingMultipleInputVideoGraph.java deleted file mode 100644 index 0d1e8787f1..0000000000 --- a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingMultipleInputVideoGraph.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2024 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.effect; - -import android.content.Context; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.PreviewingVideoGraph; -import androidx.media3.common.VideoCompositorSettings; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.util.UnstableApi; -import java.util.List; -import java.util.concurrent.Executor; - -/** - * A {@linkplain PreviewingVideoGraph previewing} specific implementation of {@link - * MultipleInputVideoGraph}. - */ -@UnstableApi -public final class PreviewingMultipleInputVideoGraph extends MultipleInputVideoGraph - implements PreviewingVideoGraph { - - /** A factory for creating a {@link PreviewingMultipleInputVideoGraph}. */ - public static final class Factory implements PreviewingVideoGraph.Factory { - private final VideoFrameProcessor.Factory videoFrameProcessorFactory; - - /** - * Creates a new factory that uses the {@link DefaultVideoFrameProcessor.Factory} with its - * default values. - */ - public Factory() { - videoFrameProcessorFactory = new DefaultVideoFrameProcessor.Factory.Builder().build(); - } - - @Override - public PreviewingVideoGraph create( - Context context, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs) { - return new PreviewingMultipleInputVideoGraph( - context, - videoFrameProcessorFactory, - outputColorInfo, - debugViewProvider, - listener, - listenerExecutor, - videoCompositorSettings, - compositionEffects, - initialTimestampOffsetUs); - } - - @Override - public boolean supportsMultipleInputs() { - return true; - } - } - - private PreviewingMultipleInputVideoGraph( - Context context, - VideoFrameProcessor.Factory videoFrameProcessorFactory, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs) { - super( - context, - videoFrameProcessorFactory, - outputColorInfo, - debugViewProvider, - listener, - listenerExecutor, - videoCompositorSettings, - compositionEffects, - initialTimestampOffsetUs, - /* renderFramesAutomatically= */ false); - } - - @Override - public void renderOutputFrame(long renderTimeNs) { - getCompositionVideoFrameProcessor().renderOutputFrame(renderTimeNs); - } -} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java deleted file mode 100644 index e953886ea5..0000000000 --- a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.effect; - -import android.content.Context; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.PreviewingVideoGraph; -import androidx.media3.common.VideoCompositorSettings; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.util.UnstableApi; -import java.util.List; -import java.util.concurrent.Executor; - -/** - * A {@link PreviewingVideoGraph Previewing} specific implementation of {@link - * SingleInputVideoGraph}. - */ -@UnstableApi -public final class PreviewingSingleInputVideoGraph extends SingleInputVideoGraph - implements PreviewingVideoGraph { - - /** A factory for creating a {@link PreviewingSingleInputVideoGraph}. */ - public static final class Factory implements PreviewingVideoGraph.Factory { - - private final VideoFrameProcessor.Factory videoFrameProcessorFactory; - - /** - * Creates a new factory that uses the {@link DefaultVideoFrameProcessor.Factory} with its - * default values. - */ - public Factory() { - this(new DefaultVideoFrameProcessor.Factory.Builder().build()); - } - - /** - * Creates an instance that uses the supplied {@code videoFrameProcessorFactory} to create - * {@link VideoFrameProcessor} instances. - */ - public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) { - this.videoFrameProcessorFactory = videoFrameProcessorFactory; - } - - @Override - public PreviewingVideoGraph create( - Context context, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs) { - return new PreviewingSingleInputVideoGraph( - context, - videoFrameProcessorFactory, - outputColorInfo, - debugViewProvider, - listener, - listenerExecutor, - initialTimestampOffsetUs); - } - - @Override - public boolean supportsMultipleInputs() { - return false; - } - } - - private PreviewingSingleInputVideoGraph( - Context context, - VideoFrameProcessor.Factory videoFrameProcessorFactory, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - long initialTimestampOffsetUs) { - super( - context, - videoFrameProcessorFactory, - outputColorInfo, - listener, - debugViewProvider, - listenerExecutor, - VideoCompositorSettings.DEFAULT, - // Previewing needs frame render timing. - /* renderFramesAutomatically= */ false, - initialTimestampOffsetUs); - } - - @Override - public void renderOutputFrame(long renderTimeNs) { - getProcessor(getInputIndex()).renderOutputFrame(renderTimeNs); - } -} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 290c411966..639a0056d7 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -16,27 +16,34 @@ package androidx.media3.effect; -import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; +import android.graphics.Bitmap; +import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; +import androidx.media3.common.Format; +import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoCompositorSettings; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; +import java.util.List; import java.util.concurrent.Executor; /** A {@link VideoGraph} that handles one input stream. */ @UnstableApi -public abstract class SingleInputVideoGraph implements VideoGraph { +public class SingleInputVideoGraph implements VideoGraph { private final Context context; private final VideoFrameProcessor.Factory videoFrameProcessorFactory; @@ -44,8 +51,8 @@ public abstract class SingleInputVideoGraph implements VideoGraph { private final Listener listener; private final DebugViewProvider debugViewProvider; private final Executor listenerExecutor; + private final List compositionEffects; private final boolean renderFramesAutomatically; - private final long initialTimestampOffsetUs; @Nullable private VideoFrameProcessor videoFrameProcessor; @Nullable private SurfaceInfo outputSurfaceInfo; @@ -53,6 +60,51 @@ public abstract class SingleInputVideoGraph implements VideoGraph { private volatile boolean hasProducedFrameWithTimestampZero; private int inputIndex; + /** A {@link VideoGraph.Factory} for {@link SingleInputVideoGraph}. */ + public static final class Factory implements VideoGraph.Factory { + private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + + /** + * A {@code Factory} for {@link SingleInputVideoGraph} that uses a {@link + * DefaultVideoFrameProcessor.Factory}. + */ + public Factory() { + this(new DefaultVideoFrameProcessor.Factory.Builder().build()); + } + + public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) { + this.videoFrameProcessorFactory = videoFrameProcessorFactory; + } + + @Override + public SingleInputVideoGraph create( + Context context, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + Listener listener, + Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, + List compositionEffects, + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { + return new SingleInputVideoGraph( + context, + videoFrameProcessorFactory, + outputColorInfo, + listener, + compositionEffects, + debugViewProvider, + listenerExecutor, + videoCompositorSettings, + renderFramesAutomatically); + } + + @Override + public boolean supportsMultipleInputs() { + return false; + } + } + /** * Creates an instance. * @@ -63,11 +115,11 @@ public abstract class SingleInputVideoGraph implements VideoGraph { VideoFrameProcessor.Factory videoFrameProcessorFactory, ColorInfo outputColorInfo, Listener listener, + List compositionEffects, DebugViewProvider debugViewProvider, Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, - boolean renderFramesAutomatically, - long initialTimestampOffsetUs) { + boolean renderFramesAutomatically) { checkState( VideoCompositorSettings.DEFAULT.equals(videoCompositorSettings), "SingleInputVideoGraph does not use VideoCompositor, and therefore cannot apply" @@ -78,8 +130,8 @@ public abstract class SingleInputVideoGraph implements VideoGraph { this.listener = listener; this.debugViewProvider = debugViewProvider; this.listenerExecutor = listenerExecutor; + this.compositionEffects = compositionEffects; this.renderFramesAutomatically = renderFramesAutomatically; - this.initialTimestampOffsetUs = initialTimestampOffsetUs; this.inputIndex = C.INDEX_UNSET; } @@ -146,12 +198,6 @@ public abstract class SingleInputVideoGraph implements VideoGraph { } } - @Override - public VideoFrameProcessor getProcessor(int inputIndex) { - checkArgument(this.inputIndex != C.INDEX_UNSET && this.inputIndex == inputIndex); - return checkStateNotNull(videoFrameProcessor); - } - @Override public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { this.outputSurfaceInfo = outputSurfaceInfo; @@ -165,6 +211,88 @@ public abstract class SingleInputVideoGraph implements VideoGraph { return hasProducedFrameWithTimestampZero; } + @Override + public boolean queueInputBitmap( + int inputIndex, Bitmap inputBitmap, TimestampIterator timestampIterator) { + checkStateNotNull(videoFrameProcessor); + return videoFrameProcessor.queueInputBitmap(inputBitmap, timestampIterator); + } + + @Override + public boolean queueInputTexture(int inputIndex, int textureId, long presentationTimeUs) { + checkStateNotNull(videoFrameProcessor); + return videoFrameProcessor.queueInputTexture(textureId, presentationTimeUs); + } + + @Override + public void setOnInputFrameProcessedListener( + int inputIndex, OnInputFrameProcessedListener listener) { + checkStateNotNull(videoFrameProcessor); + videoFrameProcessor.setOnInputFrameProcessedListener(listener); + } + + @Override + public void setOnInputSurfaceReadyListener(int inputIndex, Runnable listener) { + checkStateNotNull(videoFrameProcessor); + videoFrameProcessor.setOnInputSurfaceReadyListener(listener); + } + + @Override + public Surface getInputSurface(int inputIndex) { + checkStateNotNull(videoFrameProcessor); + return videoFrameProcessor.getInputSurface(); + } + + @Override + public void registerInputStream( + int inputIndex, + @VideoFrameProcessor.InputType int inputType, + Format format, + List effects, + long offsetToAddUs) { + checkStateNotNull(videoFrameProcessor); + videoFrameProcessor.registerInputStream( + inputType, + format, + new ImmutableList.Builder().addAll(effects).addAll(compositionEffects).build(), + offsetToAddUs); + } + + @Override + public boolean registerInputFrame(int inputIndex) { + checkStateNotNull(videoFrameProcessor); + return videoFrameProcessor.registerInputFrame(); + } + + @Override + public int getPendingInputFrameCount(int inputIndex) { + checkStateNotNull(videoFrameProcessor); + return videoFrameProcessor.getPendingInputFrameCount(); + } + + @Override + public void renderOutputFrame(long renderTimeNs) { + checkStateNotNull(videoFrameProcessor); + videoFrameProcessor.renderOutputFrame(renderTimeNs); + } + + @Override + public void redraw() { + throw new UnsupportedOperationException(); + } + + @Override + public void flush() { + checkStateNotNull(videoFrameProcessor); + videoFrameProcessor.flush(); + } + + @Override + public void signalEndOfInput(int inputIndex) { + checkStateNotNull(videoFrameProcessor); + videoFrameProcessor.signalEndOfInput(); + } + @Override public void release() { if (released) { @@ -177,12 +305,4 @@ public abstract class SingleInputVideoGraph implements VideoGraph { } released = true; } - - protected int getInputIndex() { - return inputIndex; - } - - protected long getInitialTimestampOffsetUs() { - return initialTimestampOffsetUs; - } } diff --git a/libraries/exoplayer/proguard-rules.txt b/libraries/exoplayer/proguard-rules.txt index e439ce04e3..91685e1af6 100644 --- a/libraries/exoplayer/proguard-rules.txt +++ b/libraries/exoplayer/proguard-rules.txt @@ -71,8 +71,8 @@ } # Constructors and methods accessed via reflection in PlaybackVideoGraphWrapper --dontnote androidx.media3.effect.PreviewingSingleInputVideoGraph$Factory --keepclasseswithmembers class androidx.media3.effect.PreviewingSingleInputVideoGraph$Factory { +-dontnote androidx.media3.effect.SingleInputVideoGraph$Factory +-keepclasseswithmembers class androidx.media3.effect.SingleInputVideoGraph$Factory { (androidx.media3.common.VideoFrameProcessor$Factory); } -dontnote androidx.media3.effect.DefaultVideoFrameProcessor$Factory$Builder diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java index cfde132fa9..eb26340a73 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java @@ -1278,7 +1278,7 @@ import java.util.concurrent.CopyOnWriteArraySet; verifyApplicationThread(); try { // LINT.IfChange(set_video_effects) - Class.forName("androidx.media3.effect.PreviewingSingleInputVideoGraph$Factory") + Class.forName("androidx.media3.effect.SingleInputVideoGraph$Factory") .getConstructor(VideoFrameProcessor.Factory.class); // LINT.ThenChange(video/PlaybackVideoGraphWrapper.java) } catch (ClassNotFoundException | NoSuchMethodException e) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index 93579798fa..5c8469ac54 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -19,7 +19,6 @@ import static androidx.media3.common.VideoFrameProcessor.DROP_OUTPUT_FRAME; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; -import static androidx.media3.common.util.Util.castNonNull; import static androidx.media3.common.util.Util.contains; import static androidx.media3.common.util.Util.getMaxPendingFramesCountForMediaCodecDecoders; import static androidx.media3.exoplayer.video.VideoSink.INPUT_TYPE_SURFACE; @@ -41,7 +40,6 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.Format; -import androidx.media3.common.PreviewingVideoGraph; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoCompositorSettings; import androidx.media3.common.VideoFrameProcessingException; @@ -67,7 +65,6 @@ import java.lang.annotation.Target; import java.util.List; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.Executor; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** @@ -125,7 +122,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private final VideoFrameReleaseControl videoFrameReleaseControl; private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory; - private PreviewingVideoGraph.@MonotonicNonNull Factory previewingVideoGraphFactory; + private VideoGraph.@MonotonicNonNull Factory videoGraphFactory; private List compositionEffects; private VideoCompositorSettings compositorSettings; private Clock clock; @@ -159,18 +156,17 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video } /** - * Sets the {@link PreviewingVideoGraph.Factory} that will be used for creating {@link - * PreviewingVideoGraph} instances. + * Sets the {@link VideoGraph.Factory} that will be used for creating {@link VideoGraph} + * instances. * - *

By default, the {@code PreviewingSingleInputVideoGraph.Factory} will be used. + *

By default, the {@code SingleInputVideoGraph.Factory} will be used. * - * @param previewingVideoGraphFactory The {@link PreviewingVideoGraph.Factory}. + * @param videoGraphFactory The {@link VideoGraph.Factory}. * @return This builder, for convenience. */ @CanIgnoreReturnValue - public Builder setPreviewingVideoGraphFactory( - PreviewingVideoGraph.Factory previewingVideoGraphFactory) { - this.previewingVideoGraphFactory = previewingVideoGraphFactory; + public Builder setVideoGraphFactory(VideoGraph.Factory videoGraphFactory) { + this.videoGraphFactory = videoGraphFactory; return this; } @@ -235,12 +231,11 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video public PlaybackVideoGraphWrapper build() { checkState(!built); - if (previewingVideoGraphFactory == null) { + if (videoGraphFactory == null) { if (videoFrameProcessorFactory == null) { videoFrameProcessorFactory = new ReflectiveDefaultVideoFrameProcessorFactory(); } - previewingVideoGraphFactory = - new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory); + videoGraphFactory = new ReflectiveSingleInputVideoGraphFactory(videoFrameProcessorFactory); } PlaybackVideoGraphWrapper playbackVideoGraphWrapper = new PlaybackVideoGraphWrapper(this); built = true; @@ -270,7 +265,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video */ private final TimedValueQueue streamStartPositionsUs; - private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; + private final VideoGraph.Factory videoGraphFactory; private final SparseArray inputVideoSinks; private final List compositionEffects; private final VideoCompositorSettings compositorSettings; @@ -282,7 +277,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private Format videoGraphOutputFormat; private @MonotonicNonNull HandlerWrapper handler; - private @MonotonicNonNull PreviewingVideoGraph videoGraph; + private @MonotonicNonNull VideoGraph videoGraph; private long outputStreamStartPositionUs; @Nullable private Pair currentSurfaceAndSize; private int pendingFlushCount; @@ -313,7 +308,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private PlaybackVideoGraphWrapper(Builder builder) { context = builder.context; streamStartPositionsUs = new TimedValueQueue<>(); - previewingVideoGraphFactory = checkStateNotNull(builder.previewingVideoGraphFactory); + videoGraphFactory = checkStateNotNull(builder.videoGraphFactory); inputVideoSinks = new SparseArray<>(); compositionEffects = builder.compositionEffects; compositorSettings = builder.compositorSettings; @@ -469,8 +464,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video // Internal methods - @Nullable - private VideoFrameProcessor registerInput(Format sourceFormat, int inputIndex) + private boolean registerInput(Format sourceFormat, int inputIndex) throws VideoSink.VideoSinkException { if (inputIndex == PRIMARY_SEQUENCE_INDEX) { checkState(state == STATE_CREATED); @@ -493,7 +487,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null); try { videoGraph = - previewingVideoGraphFactory.create( + videoGraphFactory.create( context, outputColorInfo, DebugViewProvider.NONE, @@ -501,12 +495,12 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video /* listenerExecutor= */ handler::post, compositorSettings, compositionEffects, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* renderFramesAutomatically= */ false); videoGraph.initialize(); } catch (VideoFrameProcessingException e) { throw new VideoSink.VideoSinkException(e, sourceFormat); } - if (currentSurfaceAndSize != null) { Surface surface = currentSurfaceAndSize.first; Size size = currentSurfaceAndSize.second; @@ -517,7 +511,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video } else { if (!isInitialized()) { // Make sure the primary sequence is initialized first. - return null; + return false; } } @@ -529,7 +523,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video registeredVideoInputCount++; defaultVideoSink.setListener( new DefaultVideoSinkListener(), /* executor= */ checkNotNull(handler)::post); - return videoGraph.getProcessor(inputIndex); + return true; } private boolean isInitialized() { @@ -632,7 +626,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private final int inputIndex; private ImmutableList videoEffects; - @Nullable private VideoFrameProcessor videoFrameProcessor; @Nullable private Format inputFormat; private @InputType int inputType; private long inputBufferTimestampAdjustmentUs; @@ -645,6 +638,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private VideoSink.Listener listener; private Executor listenerExecutor; private boolean signaledEndOfStream; + private boolean isInitialized; /** Creates a new instance. */ public InputVideoSink(Context context, int inputIndex) { @@ -684,26 +678,25 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video @Override public boolean initialize(Format sourceFormat) throws VideoSinkException { checkState(!isInitialized()); - videoFrameProcessor = PlaybackVideoGraphWrapper.this.registerInput(sourceFormat, inputIndex); - return videoFrameProcessor != null; + isInitialized = PlaybackVideoGraphWrapper.this.registerInput(sourceFormat, inputIndex); + return isInitialized; } @Override - @EnsuresNonNullIf(result = true, expression = "videoFrameProcessor") public boolean isInitialized() { - return videoFrameProcessor != null; + return isInitialized; } @Override public void redraw() { checkState(isInitialized()); - castNonNull(videoFrameProcessor).redraw(); + checkNotNull(videoGraph).redraw(); } @Override public void flush(boolean resetPosition) { if (isInitialized()) { - videoFrameProcessor.flush(); + checkNotNull(videoGraph).flush(); } lastBufferPresentationTimeUs = C.TIME_UNSET; PlaybackVideoGraphWrapper.this.flush(resetPosition); @@ -735,7 +728,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video return; } if (isInitialized()) { - videoFrameProcessor.signalEndOfInput(); + checkNotNull(videoGraph).signalEndOfInput(inputIndex); signaledEndOfStream = true; } } @@ -767,7 +760,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video @Override public Surface getInputSurface() { checkState(isInitialized()); - return checkStateNotNull(videoFrameProcessor).getInputSurface(); + return checkNotNull(videoGraph).getInputSurface(inputIndex); } @Override @@ -841,11 +834,11 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video if (!shouldRenderToInputVideoSink()) { return false; } - if (checkStateNotNull(videoFrameProcessor).getPendingInputFrameCount() + if (checkNotNull(videoGraph).getPendingInputFrameCount(inputIndex) >= videoFrameProcessorMaxPendingFrameCount) { return false; } - if (!checkStateNotNull(videoFrameProcessor).registerInputFrame()) { + if (!checkNotNull(videoGraph).registerInputFrame(inputIndex)) { return false; } @@ -871,7 +864,8 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video public boolean handleInputBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator) { checkState(isInitialized()); if (!shouldRenderToInputVideoSink() - || !checkNotNull(videoFrameProcessor).queueInputBitmap(inputBitmap, timestampIterator)) { + || !checkNotNull(videoGraph) + .queueInputBitmap(inputIndex, inputBitmap, timestampIterator)) { return false; } @@ -947,7 +941,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video *

Effects are pending until a new input stream is registered. */ private void setPendingVideoEffects(List newVideoEffects) { - if (previewingVideoGraphFactory.supportsMultipleInputs()) { + if (videoGraphFactory.supportsMultipleInputs()) { this.videoEffects = ImmutableList.copyOf(newVideoEffects); } else { this.videoEffects = @@ -964,9 +958,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video .buildUpon() .setColorInfo(getAdjustedInputColorInfo(inputFormat.colorInfo)) .build(); - checkStateNotNull(videoFrameProcessor) + checkNotNull(videoGraph) .registerInputStream( - inputType, adjustedInputFormat, videoEffects, /* offsetToAddUs= */ 0); + inputIndex, inputType, adjustedInputFormat, videoEffects, /* offsetToAddUs= */ 0); } } @@ -1002,22 +996,18 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video } } - /** - * Delays reflection for loading a {@linkplain PreviewingVideoGraph.Factory - * PreviewingSingleInputVideoGraph} instance. - */ - private static final class ReflectivePreviewingSingleInputVideoGraphFactory - implements PreviewingVideoGraph.Factory { + /** Delays reflection for loading a {@link VideoGraph.Factory SingleInputVideoGraph} instance. */ + private static final class ReflectiveSingleInputVideoGraphFactory implements VideoGraph.Factory { private final VideoFrameProcessor.Factory videoFrameProcessorFactory; - public ReflectivePreviewingSingleInputVideoGraphFactory( + public ReflectiveSingleInputVideoGraphFactory( VideoFrameProcessor.Factory videoFrameProcessorFactory) { this.videoFrameProcessorFactory = videoFrameProcessorFactory; } @Override - public PreviewingVideoGraph create( + public VideoGraph create( Context context, ColorInfo outputColorInfo, DebugViewProvider debugViewProvider, @@ -1025,32 +1015,34 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs) - throws VideoFrameProcessingException { + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { + VideoGraph.Factory factory; try { // LINT.IfChange - Class previewingSingleInputVideoGraphFactoryClass = - Class.forName("androidx.media3.effect.PreviewingSingleInputVideoGraph$Factory"); - PreviewingVideoGraph.Factory factory = - (PreviewingVideoGraph.Factory) - previewingSingleInputVideoGraphFactoryClass + Class singleInputVideoGraphFactoryClass = + Class.forName("androidx.media3.effect.SingleInputVideoGraph$Factory"); + factory = + (VideoGraph.Factory) + singleInputVideoGraphFactoryClass .getConstructor(VideoFrameProcessor.Factory.class) .newInstance(videoFrameProcessorFactory); // LINT.ThenChange( // ../../../../../../../proguard-rules.txt, // ../ExoPlayerImpl.java:set_video_effects) - return factory.create( - context, - outputColorInfo, - debugViewProvider, - listener, - listenerExecutor, - videoCompositorSettings, - compositionEffects, - initialTimestampOffsetUs); } catch (Exception e) { - throw VideoFrameProcessingException.from(e); + throw new IllegalStateException(e); } + return factory.create( + context, + outputColorInfo, + debugViewProvider, + listener, + listenerExecutor, + videoCompositorSettings, + compositionEffects, + initialTimestampOffsetUs, + renderFramesAutomatically); } @Override diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapperTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapperTest.java index 182ea092c4..54ee6a0258 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapperTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapperTest.java @@ -17,24 +17,21 @@ package androidx.media3.exoplayer.video; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import android.content.Context; -import android.graphics.Bitmap; -import android.view.Surface; -import androidx.annotation.Nullable; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.Format; -import androidx.media3.common.OnInputFrameProcessedListener; -import androidx.media3.common.PreviewingVideoGraph; -import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoCompositorSettings; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; -import androidx.media3.common.util.TimestampIterator; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -42,6 +39,7 @@ import java.util.List; import java.util.concurrent.Executor; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; /** Unit test for {@link PlaybackVideoGraphWrapper}. */ @@ -61,7 +59,7 @@ public final class PlaybackVideoGraphWrapperTest { @Test public void initializeSink_calledTwice_throws() throws VideoSink.VideoSinkException { PlaybackVideoGraphWrapper playbackVideoGraphWrapper = - createPlaybackVideoGraphWrapper(new FakeVideoFrameProcessor()); + createPlaybackVideoGraphWrapper(new TestVideoGraphFactory()); VideoSink sink = playbackVideoGraphWrapper.getSink(/* inputIndex= */ 0); sink.initialize(new Format.Builder().build()); @@ -73,27 +71,27 @@ public final class PlaybackVideoGraphWrapperTest { ImmutableList firstEffects = ImmutableList.of(Mockito.mock(Effect.class)); ImmutableList secondEffects = ImmutableList.of(Mockito.mock(Effect.class), Mockito.mock(Effect.class)); - FakeVideoFrameProcessor videoFrameProcessor = new FakeVideoFrameProcessor(); + TestVideoGraphFactory testVideoGraphFactory = new TestVideoGraphFactory(); PlaybackVideoGraphWrapper playbackVideoGraphWrapper = - createPlaybackVideoGraphWrapper(videoFrameProcessor); + createPlaybackVideoGraphWrapper(testVideoGraphFactory); Format format = new Format.Builder().build(); VideoSink sink = playbackVideoGraphWrapper.getSink(/* inputIndex= */ 0); sink.initialize(format); sink.onInputStreamChanged(VideoSink.INPUT_TYPE_SURFACE, format, firstEffects); - assertThat(videoFrameProcessor.registeredEffects).isEqualTo(firstEffects); sink.onInputStreamChanged(VideoSink.INPUT_TYPE_SURFACE, format, secondEffects); - assertThat(videoFrameProcessor.registeredEffects).isEqualTo(secondEffects); sink.onInputStreamChanged(VideoSink.INPUT_TYPE_SURFACE, format, ImmutableList.of()); - assertThat(videoFrameProcessor.registeredEffects).isEmpty(); + testVideoGraphFactory.verifyRegisteredEffectsMatches(/* invocationTimes= */ 3); + assertThat(testVideoGraphFactory.getCapturedEffects()) + .isEqualTo(ImmutableList.of(firstEffects, secondEffects, ImmutableList.of())); } private static PlaybackVideoGraphWrapper createPlaybackVideoGraphWrapper( - VideoFrameProcessor videoFrameProcessor) { + VideoGraph.Factory videoGraphFactory) { Context context = ApplicationProvider.getApplicationContext(); return new PlaybackVideoGraphWrapper.Builder(context, createVideoFrameReleaseControl()) - .setPreviewingVideoGraphFactory(new TestPreviewingVideoGraphFactory(videoFrameProcessor)) + .setVideoGraphFactory(videoGraphFactory) .build(); } @@ -126,79 +124,16 @@ public final class PlaybackVideoGraphWrapperTest { context, frameTimingEvaluator, /* allowedJoiningTimeMs= */ 0); } - private static class FakeVideoFrameProcessor implements VideoFrameProcessor { - - List registeredEffects = ImmutableList.of(); - - @Override - public boolean queueInputBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator) { - return false; - } - - @Override - public boolean queueInputTexture(int textureId, long presentationTimeUs) { - return false; - } - - @Override - public void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener) {} - - @Override - public void setOnInputSurfaceReadyListener(Runnable listener) {} - - @Override - public Surface getInputSurface() { - return null; - } - - @Override - public void redraw() {} - - @Override - public void registerInputStream( - @InputType int inputType, Format format, List effects, long offsetToAddUs) { - registeredEffects = effects; - } - - @Override - public boolean registerInputFrame() { - return true; - } - - @Override - public int getPendingInputFrameCount() { - return 0; - } - - @Override - public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {} - - @Override - public void renderOutputFrame(long renderTimeNs) {} - - @Override - public void signalEndOfInput() {} - - @Override - public void flush() {} - - @Override - public void release() {} - } - - private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory { + private static class TestVideoGraphFactory implements VideoGraph.Factory { // Using a mock but we don't assert mock interactions. If needed to assert interactions, we // should a fake instead. - private final PreviewingVideoGraph previewingVideoGraph = - Mockito.mock(PreviewingVideoGraph.class); - private final VideoFrameProcessor videoFrameProcessor; + private final VideoGraph videoGraph = Mockito.mock(VideoGraph.class); - public TestPreviewingVideoGraphFactory(VideoFrameProcessor videoFrameProcessor) { - this.videoFrameProcessor = videoFrameProcessor; - } + @SuppressWarnings("unchecked") + private final ArgumentCaptor> effectsCaptor = ArgumentCaptor.forClass(List.class); @Override - public PreviewingVideoGraph create( + public VideoGraph create( Context context, ColorInfo outputColorInfo, DebugViewProvider debugViewProvider, @@ -206,14 +141,30 @@ public final class PlaybackVideoGraphWrapperTest { Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs) { - when(previewingVideoGraph.getProcessor(anyInt())).thenReturn(videoFrameProcessor); - return previewingVideoGraph; + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { + + when(videoGraph.registerInputFrame(anyInt())).thenReturn(true); + return videoGraph; } @Override public boolean supportsMultipleInputs() { return false; } + + public void verifyRegisteredEffectsMatches(int invocationTimes) { + verify(videoGraph, times(invocationTimes)) + .registerInputStream( + /* inputIndex= */ anyInt(), + /* inputType= */ eq(VideoSink.INPUT_TYPE_SURFACE), + /* format= */ any(), + effectsCaptor.capture(), + /* offsetToAddUs= */ anyLong()); + } + + public List> getCapturedEffects() { + return effectsCaptor.getAllValues(); + } } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionMultipleSequencePlaybackTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionMultipleSequencePlaybackTest.java index 36a304751d..853057ce41 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionMultipleSequencePlaybackTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionMultipleSequencePlaybackTest.java @@ -26,7 +26,6 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.PlaybackException; import androidx.media3.effect.GlEffect; import androidx.media3.effect.MultipleInputVideoGraph; -import androidx.media3.effect.PreviewingMultipleInputVideoGraph; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -205,8 +204,7 @@ public class CompositionMultipleSequencePlaybackTest { () -> { player = new CompositionPlayer.Builder(context) - .setPreviewingVideoGraphFactory( - new PreviewingMultipleInputVideoGraph.Factory()) + .setVideoGraphFactory(new MultipleInputVideoGraph.Factory()) .build(); player.addListener(playerTestListener); player.setComposition(composition); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java index a822d46cdd..8ac77db0db 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java @@ -39,14 +39,13 @@ import androidx.media3.common.GlTextureInfo; import androidx.media3.common.MediaItem; import androidx.media3.common.PlaybackException; import androidx.media3.common.Player; -import androidx.media3.common.PreviewingVideoGraph; import androidx.media3.common.VideoCompositorSettings; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.NullableType; import androidx.media3.common.util.Util; import androidx.media3.effect.GlEffect; -import androidx.media3.effect.PreviewingSingleInputVideoGraph; +import androidx.media3.effect.SingleInputVideoGraph; import androidx.test.ext.junit.rules.ActivityScenarioRule; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -868,8 +867,7 @@ public class CompositionPlayerSeekTest { () -> { compositionPlayer = new CompositionPlayer.Builder(applicationContext) - .setPreviewingVideoGraphFactory( - new ListenerCapturingVideoGraphFactory(videoGraphEnded)) + .setVideoGraphFactory(new ListenerCapturingVideoGraphFactory(videoGraphEnded)) .setVideoPrewarmingEnabled(videoPrewarmingEnabled) .build(); // Set a surface on the player even though there is no UI on this test. We need a @@ -945,8 +943,7 @@ public class CompositionPlayerSeekTest { () -> { compositionPlayer = new CompositionPlayer.Builder(applicationContext) - .setPreviewingVideoGraphFactory( - new ListenerCapturingVideoGraphFactory(videoGraphEnded)) + .setVideoGraphFactory(new ListenerCapturingVideoGraphFactory(videoGraphEnded)) .build(); // Set a surface on the player even though there is no UI on this test. We need a // surface otherwise the player will skip/drop video frames. @@ -1037,19 +1034,18 @@ public class CompositionPlayerSeekTest { .build(); } - private static final class ListenerCapturingVideoGraphFactory - implements PreviewingVideoGraph.Factory { + private static final class ListenerCapturingVideoGraphFactory implements VideoGraph.Factory { - private final PreviewingSingleInputVideoGraph.Factory singleInputVideoGraphFactory; + private final VideoGraph.Factory singleInputVideoGraphFactory; private final CountDownLatch videoGraphEnded; public ListenerCapturingVideoGraphFactory(CountDownLatch videoGraphEnded) { - singleInputVideoGraphFactory = new PreviewingSingleInputVideoGraph.Factory(); + singleInputVideoGraphFactory = new SingleInputVideoGraph.Factory(); this.videoGraphEnded = videoGraphEnded; } @Override - public PreviewingVideoGraph create( + public VideoGraph create( Context context, ColorInfo outputColorInfo, DebugViewProvider debugViewProvider, @@ -1057,7 +1053,8 @@ public class CompositionPlayerSeekTest { Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs) { + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { return singleInputVideoGraphFactory.create( context, outputColorInfo, @@ -1093,7 +1090,8 @@ public class CompositionPlayerSeekTest { listenerExecutor, videoCompositorSettings, compositionEffects, - initialTimestampOffsetUs); + initialTimestampOffsetUs, + renderFramesAutomatically); } @Override diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java index f80f5dd5b6..5143e0cea2 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java @@ -15,13 +15,11 @@ */ package androidx.media3.transformer; -import static androidx.media3.common.PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED; import static androidx.media3.common.util.Util.isRunningOnEmulator; import static androidx.media3.transformer.AndroidTestUtil.JPG_SINGLE_PIXEL_ASSET; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET; import static com.google.common.truth.Truth.assertThat; import static com.google.common.util.concurrent.Futures.immediateFuture; -import static org.junit.Assert.assertThrows; import android.app.Instrumentation; import android.content.Context; @@ -30,7 +28,6 @@ import android.util.Pair; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.TextureView; -import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; @@ -38,12 +35,7 @@ import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; -import androidx.media3.common.PlaybackException; -import androidx.media3.common.PreviewingVideoGraph; -import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoCompositorSettings; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.util.SystemClock; @@ -51,7 +43,8 @@ import androidx.media3.common.util.Util; import androidx.media3.datasource.AssetDataSource; import androidx.media3.datasource.DataSourceUtil; import androidx.media3.datasource.DataSpec; -import androidx.media3.effect.PreviewingSingleInputVideoGraph; +import androidx.media3.effect.DefaultVideoFrameProcessor; +import androidx.media3.effect.SingleInputVideoGraph; import androidx.media3.exoplayer.RendererCapabilities; import androidx.media3.exoplayer.image.BitmapFactoryImageDecoder; import androidx.media3.exoplayer.image.ImageDecoder; @@ -456,53 +449,6 @@ public class CompositionPlayerTest { listener.waitUntilPlayerEnded(); } - @Test - public void playback_videoGraphWrapperFails_playerRaisesError() { - PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); - EditedMediaItem video = - new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET.uri)) - .setDurationUs(MP4_ASSET.videoDurationUs) - .build(); - - instrumentation.runOnMainSync( - () -> { - compositionPlayer = - new CompositionPlayer.Builder(applicationContext) - .setPreviewingVideoGraphFactory( - new PreviewingVideoGraph.Factory() { - @Override - public PreviewingVideoGraph create( - Context context, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - VideoGraph.Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs) - throws VideoFrameProcessingException { - throw new VideoFrameProcessingException( - "Test video graph failed to initialize"); - } - - @Override - public boolean supportsMultipleInputs() { - return false; - } - }) - .build(); - compositionPlayer.addListener(listener); - compositionPlayer.setComposition( - new Composition.Builder(new EditedMediaItemSequence.Builder(video).build()).build()); - compositionPlayer.prepare(); - compositionPlayer.play(); - }); - - PlaybackException thrownException = - assertThrows(PlaybackException.class, listener::waitUntilPlayerEnded); - assertThat(thrownException.errorCode).isEqualTo(ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED); - } - @Test public void release_videoGraphWrapperFailsDuringRelease_playerDoesNotRaiseError() throws Exception { @@ -515,7 +461,34 @@ public class CompositionPlayerTest { () -> { compositionPlayer = new CompositionPlayer.Builder(applicationContext) - .setPreviewingVideoGraphFactory(new FailingReleaseVideoGraph.Factory()) + .setVideoGraphFactory( + new VideoGraph.Factory() { + @Override + public VideoGraph create( + Context context, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + VideoGraph.Listener listener, + Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, + List compositionEffects, + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { + return new FailingReleaseVideoGraph( + context, + outputColorInfo, + debugViewProvider, + listener, + listenerExecutor, + videoCompositorSettings, + renderFramesAutomatically); + } + + @Override + public boolean supportsMultipleInputs() { + return false; + } + }) .build(); compositionPlayer.addListener(playerTestListener); compositionPlayer.setComposition( @@ -559,106 +532,31 @@ public class CompositionPlayerTest { } } - private static final class FailingReleaseVideoGraph extends ForwardingVideoGraph { - public static final class Factory implements PreviewingVideoGraph.Factory { - - @Override - public PreviewingVideoGraph create( - Context context, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs) - throws VideoFrameProcessingException { - return new FailingReleaseVideoGraph( - context, - outputColorInfo, - debugViewProvider, - listener, - listenerExecutor, - videoCompositorSettings, - compositionEffects, - initialTimestampOffsetUs); - } - - @Override - public boolean supportsMultipleInputs() { - return false; - } - } - - private FailingReleaseVideoGraph( + private static final class FailingReleaseVideoGraph extends SingleInputVideoGraph { + public FailingReleaseVideoGraph( Context context, ColorInfo outputColorInfo, DebugViewProvider debugViewProvider, - VideoGraph.Listener listener, + Listener listener, Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs) { + boolean renderFramesAutomatically) { super( - new PreviewingSingleInputVideoGraph.Factory() - .create( - context, - outputColorInfo, - debugViewProvider, - listener, - listenerExecutor, - videoCompositorSettings, - compositionEffects, - initialTimestampOffsetUs)); + context, + new DefaultVideoFrameProcessor.Factory.Builder().build(), + outputColorInfo, + listener, + /* compositionEffects= */ ImmutableList.of(), + debugViewProvider, + listenerExecutor, + videoCompositorSettings, + renderFramesAutomatically); } @Override public void release() { + super.release(); throw new RuntimeException("VideoGraph release error"); } } - - private static class ForwardingVideoGraph implements PreviewingVideoGraph { - - private final PreviewingVideoGraph videoGraph; - - public ForwardingVideoGraph(PreviewingVideoGraph videoGraph) { - this.videoGraph = videoGraph; - } - - @Override - public void initialize() throws VideoFrameProcessingException { - videoGraph.initialize(); - } - - @Override - public void registerInput(int inputIndex) throws VideoFrameProcessingException { - videoGraph.registerInput(inputIndex); - } - - @Override - public VideoFrameProcessor getProcessor(int inputId) { - return videoGraph.getProcessor(inputId); - } - - @Override - public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { - videoGraph.setOutputSurfaceInfo(outputSurfaceInfo); - } - - @Override - public boolean hasProducedFrameWithTimestampZero() { - return videoGraph.hasProducedFrameWithTimestampZero(); - } - - @Override - public void release() { - videoGraph.release(); - } - - @Override - public void renderOutputFrame(long renderTimeNs) { - videoGraph.renderOutputFrame(renderTimeNs); - } - } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerPixelTest.java index 1be08e2c5f..01702ee1d6 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerPixelTest.java @@ -43,7 +43,7 @@ import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.Size; import androidx.media3.common.util.Util; import androidx.media3.effect.MatrixTransformation; -import androidx.media3.effect.PreviewingMultipleInputVideoGraph; +import androidx.media3.effect.MultipleInputVideoGraph; import androidx.media3.effect.StaticOverlaySettings; import androidx.media3.transformer.Composition; import androidx.media3.transformer.CompositionPlayer; @@ -216,8 +216,7 @@ public class CompositionPlayerPixelTest { () -> { player = new CompositionPlayer.Builder(context) - .setPreviewingVideoGraphFactory( - new PreviewingMultipleInputVideoGraph.Factory()) + .setVideoGraphFactory(new MultipleInputVideoGraph.Factory()) .build(); outputImageReader.setOnImageAvailableListener( imageReader -> { @@ -316,8 +315,7 @@ public class CompositionPlayerPixelTest { () -> { player = new CompositionPlayer.Builder(context) - .setPreviewingVideoGraphFactory( - new PreviewingMultipleInputVideoGraph.Factory()) + .setVideoGraphFactory(new MultipleInputVideoGraph.Factory()) .build(); outputImageReader.setOnImageAvailableListener( imageReader -> { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java index 0c60c5a354..79735067e5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java @@ -41,10 +41,10 @@ import androidx.media3.common.Effect; import androidx.media3.common.MediaItem; import androidx.media3.common.PlaybackException; import androidx.media3.common.Player; -import androidx.media3.common.PreviewingVideoGraph; import androidx.media3.common.SimpleBasePlayer; import androidx.media3.common.Timeline; import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoGraph; import androidx.media3.common.VideoSize; import androidx.media3.common.audio.SpeedProvider; import androidx.media3.common.util.Clock; @@ -53,7 +53,7 @@ import androidx.media3.common.util.Log; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; -import androidx.media3.effect.PreviewingSingleInputVideoGraph; +import androidx.media3.effect.SingleInputVideoGraph; import androidx.media3.effect.TimestampAdjustment; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.ExoPlayer; @@ -123,7 +123,7 @@ public final class CompositionPlayer extends SimpleBasePlayer private ImageDecoder.Factory imageDecoderFactory; private boolean videoPrewarmingEnabled; private Clock clock; - private PreviewingVideoGraph.@MonotonicNonNull Factory previewingVideoGraphFactory; + private VideoGraph.@MonotonicNonNull Factory videoGraphFactory; private boolean built; /** @@ -232,18 +232,17 @@ public final class CompositionPlayer extends SimpleBasePlayer } /** - * Sets the {@link PreviewingVideoGraph.Factory} that will be used by the player. + * Sets the {@link VideoGraph.Factory} that will be used by the player. * - *

By default, a {@link PreviewingSingleInputVideoGraph.Factory} is used. + *

By default, a {@link SingleInputVideoGraph.Factory} is used. * - * @param previewingVideoGraphFactory The {@link PreviewingVideoGraph.Factory}. + * @param videoGraphFactory The {@link VideoGraph.Factory}. * @return This builder, for convenience. */ @VisibleForTesting @CanIgnoreReturnValue - public Builder setPreviewingVideoGraphFactory( - PreviewingVideoGraph.Factory previewingVideoGraphFactory) { - this.previewingVideoGraphFactory = previewingVideoGraphFactory; + public Builder setVideoGraphFactory(VideoGraph.Factory videoGraphFactory) { + this.videoGraphFactory = videoGraphFactory; return this; } @@ -262,8 +261,8 @@ public final class CompositionPlayer extends SimpleBasePlayer if (audioSink == null) { audioSink = new DefaultAudioSink.Builder(context).build(); } - if (previewingVideoGraphFactory == null) { - previewingVideoGraphFactory = new PreviewingSingleInputVideoGraph.Factory(); + if (videoGraphFactory == null) { + videoGraphFactory = new SingleInputVideoGraph.Factory(); } CompositionPlayer compositionPlayer = new CompositionPlayer(this); built = true; @@ -309,7 +308,7 @@ public final class CompositionPlayer extends SimpleBasePlayer private final AudioSink finalAudioSink; private final MediaSource.Factory mediaSourceFactory; private final ImageDecoder.Factory imageDecoderFactory; - private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; + private final VideoGraph.Factory videoGraphFactory; private final boolean videoPrewarmingEnabled; private final HandlerWrapper compositionInternalListenerHandler; @@ -350,7 +349,7 @@ public final class CompositionPlayer extends SimpleBasePlayer finalAudioSink = checkNotNull(builder.audioSink); mediaSourceFactory = builder.mediaSourceFactory; imageDecoderFactory = builder.imageDecoderFactory; - previewingVideoGraphFactory = checkNotNull(builder.previewingVideoGraphFactory); + videoGraphFactory = checkNotNull(builder.videoGraphFactory); videoPrewarmingEnabled = builder.videoPrewarmingEnabled; compositionInternalListenerHandler = clock.createHandler(builder.looper, /* callback= */ null); videoTracksSelected = new SparseBooleanArray(); @@ -378,7 +377,7 @@ public final class CompositionPlayer extends SimpleBasePlayer checkState(this.composition == null); composition = deactivateSpeedAdjustingVideoEffects(composition); - if (composition.sequences.size() > 1 && !previewingVideoGraphFactory.supportsMultipleInputs()) { + if (composition.sequences.size() > 1 && !videoGraphFactory.supportsMultipleInputs()) { Log.w(TAG, "Setting multi-sequence Composition with single input video graph."); } @@ -727,7 +726,7 @@ public final class CompositionPlayer extends SimpleBasePlayer context, new CompositionFrameTimingEvaluator(), /* allowedJoiningTimeMs= */ 0); playbackVideoGraphWrapper = new PlaybackVideoGraphWrapper.Builder(context, videoFrameReleaseControl) - .setPreviewingVideoGraphFactory(checkNotNull(previewingVideoGraphFactory)) + .setVideoGraphFactory(checkNotNull(videoGraphFactory)) .setCompositorSettings(composition.videoCompositorSettings) .setCompositionEffects(composition.effects.videoEffects) .setClock(clock) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java deleted file mode 100644 index e6c33682ed..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.transformer; - -import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME; - -import android.content.Context; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.VideoCompositorSettings; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.VideoGraph; -import androidx.media3.effect.MultipleInputVideoGraph; -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.concurrent.Executor; - -/** - * A {@link TransformerVideoGraph Transformer}-specific implementation of {@link - * MultipleInputVideoGraph}. - */ -/* package */ final class TransformerMultipleInputVideoGraph extends MultipleInputVideoGraph - implements TransformerVideoGraph { - - /** A factory for creating {@link TransformerMultipleInputVideoGraph} instances. */ - public static final class Factory implements TransformerVideoGraph.Factory { - - private final VideoFrameProcessor.Factory videoFrameProcessorFactory; - - public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) { - this.videoFrameProcessorFactory = videoFrameProcessorFactory; - } - - @Override - public TransformerMultipleInputVideoGraph create( - Context context, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - VideoGraph.Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs, - boolean renderFramesAutomatically) { - return new TransformerMultipleInputVideoGraph( - context, - videoFrameProcessorFactory, - outputColorInfo, - debugViewProvider, - listener, - listenerExecutor, - videoCompositorSettings, - compositionEffects, - initialTimestampOffsetUs, - renderFramesAutomatically); - } - } - - private TransformerMultipleInputVideoGraph( - Context context, - VideoFrameProcessor.Factory videoFrameProcessorFactory, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs, - boolean renderFramesAutomatically) { - super( - context, - videoFrameProcessorFactory, - outputColorInfo, - debugViewProvider, - listener, - listenerExecutor, - videoCompositorSettings, - compositionEffects, - initialTimestampOffsetUs, - renderFramesAutomatically); - } - - @Override - public GraphInput createInput(int inputIndex) throws VideoFrameProcessingException { - registerInput(inputIndex); - return new VideoFrameProcessingWrapper( - getProcessor(inputIndex), - /* postProcessingEffects= */ ImmutableList.of(), - getInitialTimestampOffsetUs()); - } - - @Override - public void renderOutputFrameWithMediaPresentationTime() { - getCompositionVideoFrameProcessor() - .renderOutputFrame(RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME); - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java deleted file mode 100644 index 92b445abc4..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.transformer; - -import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME; -import static androidx.media3.common.util.Assertions.checkState; - -import android.content.Context; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.VideoCompositorSettings; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.effect.SingleInputVideoGraph; -import java.util.List; -import java.util.concurrent.Executor; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -/** - * A {@link TransformerVideoGraph Transformer}-specific implementation of {@link - * SingleInputVideoGraph}. - */ -/* package */ final class TransformerSingleInputVideoGraph extends SingleInputVideoGraph - implements TransformerVideoGraph { - - /** A factory for creating {@link TransformerSingleInputVideoGraph} instances. */ - public static final class Factory implements TransformerVideoGraph.Factory { - - private final VideoFrameProcessor.Factory videoFrameProcessorFactory; - - public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) { - this.videoFrameProcessorFactory = videoFrameProcessorFactory; - } - - @Override - public TransformerSingleInputVideoGraph create( - Context context, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs, - boolean renderFramesAutomatically) { - return new TransformerSingleInputVideoGraph( - context, - videoFrameProcessorFactory, - outputColorInfo, - listener, - debugViewProvider, - listenerExecutor, - videoCompositorSettings, - renderFramesAutomatically, - compositionEffects, - initialTimestampOffsetUs); - } - } - - private final List compositionEffects; - private @MonotonicNonNull VideoFrameProcessingWrapper videoFrameProcessingWrapper; - - private TransformerSingleInputVideoGraph( - Context context, - VideoFrameProcessor.Factory videoFrameProcessorFactory, - ColorInfo outputColorInfo, - Listener listener, - DebugViewProvider debugViewProvider, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - boolean renderFramesAutomatically, - List compositionEffects, - long initialTimestampOffsetUs) { - super( - context, - videoFrameProcessorFactory, - outputColorInfo, - listener, - debugViewProvider, - listenerExecutor, - videoCompositorSettings, - renderFramesAutomatically, - initialTimestampOffsetUs); - this.compositionEffects = compositionEffects; - } - - @Override - public GraphInput createInput(int inputIndex) throws VideoFrameProcessingException { - checkState(videoFrameProcessingWrapper == null); - registerInput(inputIndex); - videoFrameProcessingWrapper = - new VideoFrameProcessingWrapper( - getProcessor(inputIndex), compositionEffects, getInitialTimestampOffsetUs()); - return videoFrameProcessingWrapper; - } - - @Override - public void renderOutputFrameWithMediaPresentationTime() { - getProcessor(getInputIndex()).renderOutputFrame(RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME); - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java deleted file mode 100644 index 75ccac73b4..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.transformer; - -import android.content.Context; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.SurfaceInfo; -import androidx.media3.common.VideoCompositorSettings; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.VideoGraph; -import java.util.List; -import java.util.concurrent.Executor; - -/** The {@link VideoGraph} to support {@link Transformer} specific use cases. */ -/* package */ interface TransformerVideoGraph extends VideoGraph { - - /** A factory for creating a {@link TransformerVideoGraph}. */ - interface Factory { - /** - * Creates a new {@link TransformerVideoGraph} instance. - * - * @param context A {@link Context}. - * @param outputColorInfo The {@link ColorInfo} for the output frames. - * @param debugViewProvider A {@link DebugViewProvider}. - * @param listener A {@link Listener}. - * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked. - * @param videoCompositorSettings The {@link VideoCompositorSettings} to apply to the - * composition. - * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition. - * @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds. - * @param renderFramesAutomatically If {@code true}, the instance will render output frames to - * the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as the - * instance is done processing them. If {@code false}, the instance will block until {@link - * #renderOutputFrameWithMediaPresentationTime()} is called, to render the frame. - * @return A new instance. - * @throws VideoFrameProcessingException If a problem occurs while creating the {@link - * VideoFrameProcessor}. - */ - TransformerVideoGraph create( - Context context, - ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - VideoCompositorSettings videoCompositorSettings, - List compositionEffects, - long initialTimestampOffsetUs, - boolean renderFramesAutomatically) - throws VideoFrameProcessingException; - } - - /** - * Returns a {@link GraphInput} object to which the {@code VideoGraph} inputs are queued. - * - *

This method must be called after successfully {@linkplain #initialize() initializing} the - * {@code VideoGraph}. - * - *

This method must called exactly once for every input stream. - * - *

If the method throws any {@link Exception}, the caller must call {@link #release}. - * - * @param inputIndex The index of the input, which could be used to order the inputs. - */ - GraphInput createInput(int inputIndex) throws VideoFrameProcessingException; - - /** - * Renders the oldest unrendered output frame that has become {@linkplain - * Listener#onOutputFrameAvailableForRendering(long) available for rendering} to the output - * surface. - * - *

This method must only be called if {@code renderFramesAutomatically} was set to {@code - * false} using the {@link Factory} and should be called exactly once for each frame that becomes - * {@linkplain Listener#onOutputFrameAvailableForRendering(long) available for rendering}. - * - *

This will render the output frame to the {@linkplain #setOutputSurfaceInfo output surface} - * with the presentation seen in {@link Listener#onOutputFrameAvailableForRendering(long)}. - */ - void renderOutputFrameWithMediaPresentationTime(); -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java deleted file mode 100644 index 0a2647b10a..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.transformer; - -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; -import static androidx.media3.common.util.Assertions.checkNotNull; - -import android.graphics.Bitmap; -import android.view.Surface; -import androidx.annotation.Nullable; -import androidx.media3.common.Effect; -import androidx.media3.common.Format; -import androidx.media3.common.MediaItem; -import androidx.media3.common.MimeTypes; -import androidx.media3.common.OnInputFrameProcessedListener; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.util.TimestampIterator; -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -/** A wrapper for {@link VideoFrameProcessor} that handles {@link GraphInput} events. */ -/* package */ final class VideoFrameProcessingWrapper implements GraphInput { - private final VideoFrameProcessor videoFrameProcessor; - private final List postProcessingEffects; - private final long initialTimestampOffsetUs; - private final AtomicLong mediaItemOffsetUs; - - public VideoFrameProcessingWrapper( - VideoFrameProcessor videoFrameProcessor, - List postProcessingEffects, - long initialTimestampOffsetUs) { - this.videoFrameProcessor = videoFrameProcessor; - this.postProcessingEffects = postProcessingEffects; - this.initialTimestampOffsetUs = initialTimestampOffsetUs; - mediaItemOffsetUs = new AtomicLong(); - } - - @Override - public void onMediaItemChanged( - EditedMediaItem editedMediaItem, - long durationUs, - @Nullable Format decodedFormat, - boolean isLast) { - boolean isSurfaceAssetLoaderMediaItem = isMediaItemForSurfaceAssetLoader(editedMediaItem); - durationUs = editedMediaItem.getDurationAfterEffectsApplied(durationUs); - if (decodedFormat != null) { - decodedFormat = applyDecoderRotation(decodedFormat); - ImmutableList combinedEffects = - new ImmutableList.Builder() - .addAll(editedMediaItem.effects.videoEffects) - .addAll(postProcessingEffects) - .build(); - videoFrameProcessor.registerInputStream( - isSurfaceAssetLoaderMediaItem - ? VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION - : getInputTypeForMimeType(checkNotNull(decodedFormat.sampleMimeType)), - decodedFormat, - combinedEffects, - /* offsetToAddUs= */ initialTimestampOffsetUs + mediaItemOffsetUs.get()); - } - mediaItemOffsetUs.addAndGet(durationUs); - } - - @Override - public @InputResult int queueInputBitmap( - Bitmap inputBitmap, TimestampIterator timestampIterator) { - return videoFrameProcessor.queueInputBitmap(inputBitmap, timestampIterator) - ? INPUT_RESULT_SUCCESS - : INPUT_RESULT_TRY_AGAIN_LATER; - } - - @Override - public void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener) { - videoFrameProcessor.setOnInputFrameProcessedListener(listener); - } - - @Override - public void setOnInputSurfaceReadyListener(Runnable runnable) { - videoFrameProcessor.setOnInputSurfaceReadyListener(runnable); - } - - @Override - public @InputResult int queueInputTexture(int texId, long presentationTimeUs) { - return videoFrameProcessor.queueInputTexture(texId, presentationTimeUs) - ? INPUT_RESULT_SUCCESS - : INPUT_RESULT_TRY_AGAIN_LATER; - } - - @Override - public Surface getInputSurface() { - return videoFrameProcessor.getInputSurface(); - } - - @Override - public int getPendingVideoFrameCount() { - return videoFrameProcessor.getPendingInputFrameCount(); - } - - @Override - public boolean registerVideoFrame(long presentationTimeUs) { - return videoFrameProcessor.registerInputFrame(); - } - - @Override - public void signalEndOfVideoInput() { - videoFrameProcessor.signalEndOfInput(); - } - - public void release() { - videoFrameProcessor.release(); - } - - private static Format applyDecoderRotation(Format format) { - // The decoder rotates encoded frames for display by format.rotationDegrees. - if (format.rotationDegrees % 180 == 0) { - return format; - } - return format - .buildUpon() - .setWidth(format.height) - .setHeight(format.width) - .setRotationDegrees(0) - .build(); - } - - private static @VideoFrameProcessor.InputType int getInputTypeForMimeType(String sampleMimeType) { - if (MimeTypes.isImage(sampleMimeType)) { - return INPUT_TYPE_BITMAP; - } - if (sampleMimeType.equals(MimeTypes.VIDEO_RAW)) { - return INPUT_TYPE_TEXTURE_ID; - } - if (MimeTypes.isVideo(sampleMimeType)) { - return INPUT_TYPE_SURFACE; - } - throw new IllegalArgumentException("MIME type not supported " + sampleMimeType); - } - - private static boolean isMediaItemForSurfaceAssetLoader(EditedMediaItem editedMediaItem) { - @Nullable - MediaItem.LocalConfiguration localConfiguration = editedMediaItem.mediaItem.localConfiguration; - if (localConfiguration == null) { - return false; - } - @Nullable String scheme = localConfiguration.uri.getScheme(); - if (scheme == null) { - return false; - } - return scheme.equals(SurfaceAssetLoader.MEDIA_ITEM_URI_SCHEME); - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 634da92c39..63fa88dfdd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -22,18 +22,23 @@ import static androidx.media3.common.C.COLOR_TRANSFER_HLG; import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED; import static androidx.media3.common.ColorInfo.SRGB_BT709_FULL; import static androidx.media3.common.ColorInfo.isTransferHdr; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; +import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR; import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL; import static androidx.media3.transformer.TransformerUtil.getOutputMimeTypeAndHdrModeAfterFallback; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; +import android.graphics.Bitmap; import android.media.MediaCodec; import android.util.Pair; import android.view.Surface; -import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; @@ -41,18 +46,23 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; +import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoCompositorSettings; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.Consumer; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.decoder.DecoderInputBuffer; -import com.google.common.util.concurrent.MoreExecutors; +import androidx.media3.effect.MultipleInputVideoGraph; +import androidx.media3.effect.SingleInputVideoGraph; import java.nio.ByteBuffer; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.lock.qual.GuardedBy; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -64,7 +74,6 @@ import org.checkerframework.dataflow.qual.Pure; private final VideoGraphWrapper videoGraph; private final EncoderWrapper encoderWrapper; private final DecoderInputBuffer encoderOutputBuffer; - private final long initialTimestampOffsetUs; /** * The timestamp of the last buffer processed before {@linkplain @@ -95,7 +104,9 @@ import org.checkerframework.dataflow.qual.Pure; // TODO: b/278259383 - Consider delaying configuration of VideoSampleExporter to use the decoder // output format instead of the extractor output format, to match AudioSampleExporter behavior. super(firstInputFormat, muxerWrapper); - this.initialTimestampOffsetUs = initialTimestampOffsetUs; + // Automatically render frames if the sample exporter does not limit the number of frames in + // the encoder. + boolean renderFramesAutomatically = maxFramesInEncoder < 1; finalFramePresentationTimeUs = C.TIME_UNSET; lastMuxerInputBufferTimestampUs = C.TIME_UNSET; @@ -132,7 +143,7 @@ import org.checkerframework.dataflow.qual.Pure; boolean isGlToneMapping = encoderWrapper.getHdrModeAfterFallback() == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL - && ColorInfo.isTransferHdr(videoGraphInputColor); + && isTransferHdr(videoGraphInputColor); if (isGlToneMapping) { videoGraphOutputColor = SDR_BT709_LIMITED; } @@ -142,14 +153,16 @@ import org.checkerframework.dataflow.qual.Pure; new VideoGraphWrapper( context, hasMultipleInputs - ? new TransformerMultipleInputVideoGraph.Factory(videoFrameProcessorFactory) - : new TransformerSingleInputVideoGraph.Factory(videoFrameProcessorFactory), + ? new MultipleInputVideoGraph.Factory(videoFrameProcessorFactory) + : new SingleInputVideoGraph.Factory(videoFrameProcessorFactory), videoGraphOutputColor, - errorConsumer, debugViewProvider, videoCompositorSettings, compositionEffects, - maxFramesInEncoder); + errorConsumer, + initialTimestampOffsetUs, + maxFramesInEncoder, + renderFramesAutomatically); videoGraph.initialize(); } catch (VideoFrameProcessingException e) { throw ExportException.createForVideoFrameProcessingException(e); @@ -367,10 +380,10 @@ import org.checkerframework.dataflow.qual.Pure; if (isInputToneMapped) { // When tone-mapping HDR to SDR is enabled, assume we get BT.709 to avoid having the encoder // populate default color info, which depends on the resolution. - return ColorInfo.SDR_BT709_LIMITED; + return SDR_BT709_LIMITED; } if (SRGB_BT709_FULL.equals(inputFormat.colorInfo)) { - return ColorInfo.SDR_BT709_LIMITED; + return SDR_BT709_LIMITED; } return checkNotNull(inputFormat.colorInfo); } @@ -462,51 +475,82 @@ import org.checkerframework.dataflow.qual.Pure; } } - private final class VideoGraphWrapper implements TransformerVideoGraph, VideoGraph.Listener { + private final class VideoGraphWrapper implements VideoGraph.Listener { - private final TransformerVideoGraph videoGraph; - private final Consumer errorConsumer; - private final int maxFramesInEncoder; - private final boolean renderFramesAutomatically; + private final VideoGraph videoGraph; private final Object lock; - + private final Consumer errorConsumer; + private final boolean renderFramesAutomatically; + private final long initialTimestampOffsetUs; + private final int maxFramesInEncoder; private @GuardedBy("lock") int framesInEncoder; private @GuardedBy("lock") int framesAvailableToRender; public VideoGraphWrapper( Context context, - TransformerVideoGraph.Factory videoGraphFactory, + VideoGraph.Factory videoGraphFactory, ColorInfo videoFrameProcessorOutputColor, - Consumer errorConsumer, DebugViewProvider debugViewProvider, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - int maxFramesInEncoder) + Consumer errorConsumer, + long initialTimestampOffsetUs, + int maxFramesInEncoder, + boolean renderFramesAutomatically) throws VideoFrameProcessingException { this.errorConsumer = errorConsumer; - // To satisfy the nullness checker by declaring an initialized this reference used in the - // videoGraphFactory.create method + this.lock = new Object(); + this.renderFramesAutomatically = renderFramesAutomatically; + this.initialTimestampOffsetUs = initialTimestampOffsetUs; + this.maxFramesInEncoder = maxFramesInEncoder; + @SuppressWarnings("nullness:assignment") @Initialized VideoGraphWrapper thisRef = this; - this.maxFramesInEncoder = maxFramesInEncoder; - // Automatically render frames if the sample exporter does not limit the number of frames in - // the encoder. - renderFramesAutomatically = maxFramesInEncoder < 1; - lock = new Object(); videoGraph = videoGraphFactory.create( context, videoFrameProcessorOutputColor, debugViewProvider, /* listener= */ thisRef, - /* listenerExecutor= */ MoreExecutors.directExecutor(), + /* listenerExecutor= */ directExecutor(), videoCompositorSettings, compositionEffects, initialTimestampOffsetUs, renderFramesAutomatically); } + public void initialize() throws VideoFrameProcessingException { + videoGraph.initialize(); + } + + public boolean hasProducedFrameWithTimestampZero() { + return videoGraph.hasProducedFrameWithTimestampZero(); + } + + /** + * Returns a {@link GraphInput} object to which the {@code VideoGraph} inputs are queued. + * + *

This method must be called after successfully {@linkplain #initialize() initializing} the + * {@code VideoGraph}. + * + *

This method must called exactly once for every input stream. + * + *

If the method throws any {@link Exception}, the caller must call {@link #release}. + * + * @param inputIndex The index of the input, which could be used to order the inputs. + */ + public GraphInput createInput(int inputIndex) throws VideoFrameProcessingException { + videoGraph.registerInput(inputIndex); + // Applies the composition effects here if there's only one input. In multiple-input case, the + // effects are applied as a part of the video graph. + return new VideoGraphInput(videoGraph, inputIndex, initialTimestampOffsetUs); + } + + public void release() { + videoGraph.release(); + } + @Override public void onOutputSizeChanged(int width, int height) { @Nullable SurfaceInfo surfaceInfo = null; @@ -515,7 +559,7 @@ import org.checkerframework.dataflow.qual.Pure; } catch (ExportException e) { errorConsumer.accept(e); } - setOutputSurfaceInfo(surfaceInfo); + videoGraph.setOutputSurfaceInfo(surfaceInfo); } @Override @@ -543,47 +587,6 @@ import org.checkerframework.dataflow.qual.Pure; errorConsumer.accept(ExportException.createForVideoFrameProcessingException(e)); } - @Override - public void initialize() throws VideoFrameProcessingException { - videoGraph.initialize(); - } - - @Override - public void registerInput(@IntRange(from = 0) int inputIndex) - throws VideoFrameProcessingException { - videoGraph.registerInput(inputIndex); - } - - @Override - public VideoFrameProcessor getProcessor(int inputIndex) { - return videoGraph.getProcessor(inputIndex); - } - - @Override - public GraphInput createInput(int inputIndex) throws VideoFrameProcessingException { - return videoGraph.createInput(inputIndex); - } - - @Override - public void renderOutputFrameWithMediaPresentationTime() { - videoGraph.renderOutputFrameWithMediaPresentationTime(); - } - - @Override - public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { - videoGraph.setOutputSurfaceInfo(outputSurfaceInfo); - } - - @Override - public boolean hasProducedFrameWithTimestampZero() { - return videoGraph.hasProducedFrameWithTimestampZero(); - } - - @Override - public void release() { - videoGraph.release(); - } - public boolean hasEncoderReleasedAllBuffersAfterEndOfStream() { if (renderFramesAutomatically) { // Video graph wrapper does not track encoder buffers. @@ -616,8 +619,131 @@ import org.checkerframework.dataflow.qual.Pure; } } if (shouldRender) { - renderOutputFrameWithMediaPresentationTime(); + videoGraph.renderOutputFrame(RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME); } } } + + /** A wrapper for {@link VideoGraph} input that handles {@link GraphInput} events. */ + private static final class VideoGraphInput implements GraphInput { + private final VideoGraph videoGraph; + private final int inputIndex; + private final long initialTimestampOffsetUs; + private final AtomicLong mediaItemOffsetUs; + + public VideoGraphInput(VideoGraph videoGraph, int inputIndex, long initialTimestampOffsetUs) { + this.videoGraph = videoGraph; + this.inputIndex = inputIndex; + this.initialTimestampOffsetUs = initialTimestampOffsetUs; + mediaItemOffsetUs = new AtomicLong(); + } + + @Override + public void onMediaItemChanged( + EditedMediaItem editedMediaItem, + long durationUs, + @Nullable Format decodedFormat, + boolean isLast) { + boolean isSurfaceAssetLoaderMediaItem = isMediaItemForSurfaceAssetLoader(editedMediaItem); + durationUs = editedMediaItem.getDurationAfterEffectsApplied(durationUs); + if (decodedFormat != null) { + decodedFormat = applyDecoderRotation(decodedFormat); + videoGraph.registerInputStream( + inputIndex, + isSurfaceAssetLoaderMediaItem + ? VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION + : getInputTypeForMimeType(checkNotNull(decodedFormat.sampleMimeType)), + decodedFormat, + editedMediaItem.effects.videoEffects, + /* offsetToAddUs= */ initialTimestampOffsetUs + mediaItemOffsetUs.get()); + } + mediaItemOffsetUs.addAndGet(durationUs); + } + + @Override + public @InputResult int queueInputBitmap( + Bitmap inputBitmap, TimestampIterator timestampIterator) { + return videoGraph.queueInputBitmap(inputIndex, inputBitmap, timestampIterator) + ? INPUT_RESULT_SUCCESS + : INPUT_RESULT_TRY_AGAIN_LATER; + } + + @Override + public void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener) { + videoGraph.setOnInputFrameProcessedListener(inputIndex, listener); + } + + @Override + public void setOnInputSurfaceReadyListener(Runnable runnable) { + videoGraph.setOnInputSurfaceReadyListener(inputIndex, runnable); + } + + @Override + public @InputResult int queueInputTexture(int texId, long presentationTimeUs) { + return videoGraph.queueInputTexture(inputIndex, texId, presentationTimeUs) + ? INPUT_RESULT_SUCCESS + : INPUT_RESULT_TRY_AGAIN_LATER; + } + + @Override + public Surface getInputSurface() { + return videoGraph.getInputSurface(inputIndex); + } + + @Override + public int getPendingVideoFrameCount() { + return videoGraph.getPendingInputFrameCount(inputIndex); + } + + @Override + public boolean registerVideoFrame(long presentationTimeUs) { + return videoGraph.registerInputFrame(inputIndex); + } + + @Override + public void signalEndOfVideoInput() { + videoGraph.signalEndOfInput(inputIndex); + } + + private static Format applyDecoderRotation(Format format) { + // The decoder rotates encoded frames for display by format.rotationDegrees. + if (format.rotationDegrees % 180 == 0) { + return format; + } + return format + .buildUpon() + .setWidth(format.height) + .setHeight(format.width) + .setRotationDegrees(0) + .build(); + } + + private static @VideoFrameProcessor.InputType int getInputTypeForMimeType( + String sampleMimeType) { + if (MimeTypes.isImage(sampleMimeType)) { + return INPUT_TYPE_BITMAP; + } + if (sampleMimeType.equals(MimeTypes.VIDEO_RAW)) { + return INPUT_TYPE_TEXTURE_ID; + } + if (MimeTypes.isVideo(sampleMimeType)) { + return INPUT_TYPE_SURFACE; + } + throw new IllegalArgumentException("MIME type not supported " + sampleMimeType); + } + + private static boolean isMediaItemForSurfaceAssetLoader(EditedMediaItem editedMediaItem) { + @Nullable + MediaItem.LocalConfiguration localConfiguration = + editedMediaItem.mediaItem.localConfiguration; + if (localConfiguration == null) { + return false; + } + @Nullable String scheme = localConfiguration.uri.getScheme(); + if (scheme == null) { + return false; + } + return scheme.equals(SurfaceAssetLoader.MEDIA_ITEM_URI_SCHEME); + } + } }