From f3893c146d5e4521b84af45551ea5fda4d9c66bd Mon Sep 17 00:00:00 2001 From: hschlueter Date: Thu, 23 Jun 2022 19:00:01 +0100 Subject: [PATCH] Extract FrameProcessor interface from GlEffectsFrameProcessor. PiperOrigin-RevId: 456814150 --- .../GlEffectsFrameProcessorPixelTest.java | 2 +- .../ChainingGlTextureProcessorListener.java | 7 +- ...lMatrixTransformationProcessorWrapper.java | 8 +- .../FrameProcessingTaskExecutor.java | 8 +- .../media3/transformer/FrameProcessor.java | 77 +++++++++++++++++++ .../transformer/GlEffectsFrameProcessor.java | 75 ++++-------------- .../VideoTranscodingSamplePipeline.java | 4 +- ...hainingGlTextureProcessorListenerTest.java | 4 +- 8 files changed, 107 insertions(+), 78 deletions(-) create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java index 4cc38115bb..e77ae2dfa2 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java @@ -355,7 +355,7 @@ public final class GlEffectsFrameProcessorPixelTest { checkNotNull( GlEffectsFrameProcessor.create( context, - new GlEffectsFrameProcessor.Listener() { + new FrameProcessor.Listener() { @Override public void onFrameProcessingError(FrameProcessingException exception) { frameProcessingException.set(exception); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ChainingGlTextureProcessorListener.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ChainingGlTextureProcessorListener.java index b4d3b43bdf..db4120ea36 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ChainingGlTextureProcessorListener.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ChainingGlTextureProcessorListener.java @@ -31,7 +31,7 @@ import java.util.Queue; @Nullable private final GlTextureProcessor previousGlTextureProcessor; @Nullable private final GlTextureProcessor nextGlTextureProcessor; private final FrameProcessingTaskExecutor frameProcessingTaskExecutor; - private final GlEffectsFrameProcessor.Listener frameProcessorListener; + private final FrameProcessor.Listener frameProcessorListener; private final Queue> pendingFrames; /** @@ -45,14 +45,13 @@ import java.util.Queue; * OpenGL calls. All calls to the previous/next {@link GlTextureProcessor} will be executed by * the {@link FrameProcessingTaskExecutor}. The caller is responsible for releasing the {@link * FrameProcessingTaskExecutor}. - * @param frameProcessorListener The {@link GlEffectsFrameProcessor.Listener} to forward - * exceptions to. + * @param frameProcessorListener The {@link FrameProcessor.Listener} to forward exceptions to. */ public ChainingGlTextureProcessorListener( @Nullable GlTextureProcessor previousGlTextureProcessor, @Nullable GlTextureProcessor nextGlTextureProcessor, FrameProcessingTaskExecutor frameProcessingTaskExecutor, - GlEffectsFrameProcessor.Listener frameProcessorListener) { + FrameProcessor.Listener frameProcessorListener) { this.previousGlTextureProcessor = previousGlTextureProcessor; this.nextGlTextureProcessor = nextGlTextureProcessor; this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java index e936495398..996fce157b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java @@ -47,7 +47,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * dimensions specified by the provided {@link SurfaceInfo}. * *

This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link - * GlTextureProcessor} instances used by {@link GlEffectsFrameProcessor}. + * GlTextureProcessor} instances used by {@link FrameProcessor}. */ /* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor { @@ -60,7 +60,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final SurfaceInfo.Provider outputSurfaceProvider; private final long streamOffsetUs; private final Transformer.DebugViewProvider debugViewProvider; - private final GlEffectsFrameProcessor.Listener frameProcessorListener; + private final FrameProcessor.Listener frameProcessorListener; private final boolean enableExperimentalHdrEditing; private int inputWidth; @@ -78,7 +78,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ImmutableList matrixTransformations, SurfaceInfo.Provider outputSurfaceProvider, long streamOffsetUs, - GlEffectsFrameProcessor.Listener frameProcessorListener, + FrameProcessor.Listener frameProcessorListener, Transformer.DebugViewProvider debugViewProvider, boolean enableExperimentalHdrEditing) { this.context = context; @@ -97,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * *

The {@code FinalMatrixTransformationProcessorWrapper} will only call {@link * Listener#onInputFrameProcessed(TextureInfo)}. Other events are handled via the {@link - * GlEffectsFrameProcessor.Listener} passed to the constructor. + * FrameProcessor.Listener} passed to the constructor. */ @Override public void setListener(Listener listener) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingTaskExecutor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingTaskExecutor.java index 7f3399532c..86e03bd132 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingTaskExecutor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingTaskExecutor.java @@ -30,19 +30,19 @@ import java.util.concurrent.atomic.AtomicBoolean; * instances. * *

The wrapper handles calling {@link - * GlEffectsFrameProcessor.Listener#onFrameProcessingError(FrameProcessingException)} for errors - * that occur during these tasks. + * FrameProcessor.Listener#onFrameProcessingError(FrameProcessingException)} for errors that occur + * during these tasks. */ /* package */ final class FrameProcessingTaskExecutor { private final ExecutorService singleThreadExecutorService; - private final GlEffectsFrameProcessor.Listener listener; + private final FrameProcessor.Listener listener; private final ConcurrentLinkedQueue> futures; private final AtomicBoolean shouldCancelTasks; /** Creates a new instance. */ public FrameProcessingTaskExecutor( - ExecutorService singleThreadExecutorService, GlEffectsFrameProcessor.Listener listener) { + ExecutorService singleThreadExecutorService, FrameProcessor.Listener listener) { this.singleThreadExecutorService = singleThreadExecutorService; this.listener = listener; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java new file mode 100644 index 0000000000..a4338cd6a2 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java @@ -0,0 +1,77 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import android.view.Surface; + +/** Interface for a frame processor that applies changes to individual video frames. */ +/* package */ interface FrameProcessor { + /** + * Listener for asynchronous frame processing events. + * + *

All listener methods must be called from the same thread. + */ + interface Listener { + + /** + * Called when an exception occurs during asynchronous frame processing. + * + *

If an error occurred, consuming and producing further frames will not work as expected and + * the {@link FrameProcessor} should be released. + */ + void onFrameProcessingError(FrameProcessingException exception); + + /** Called after the {@link FrameProcessor} has produced its final output frame. */ + void onFrameProcessingEnded(); + } + + /** Returns the input {@link Surface}. */ + Surface getInputSurface(); + + /** + * Informs the {@code FrameProcessor} that a frame will be queued to its input surface. + * + *

Must be called before rendering a frame to the frame processor's input surface. + * + * @throws IllegalStateException If called after {@link #signalEndOfInputStream()}. + */ + void registerInputFrame(); + + /** + * Returns the number of input frames that have been {@linkplain #registerInputFrame() registered} + * but not processed off the {@linkplain #getInputSurface() input surface} yet. + */ + int getPendingInputFrameCount(); + + /** + * Informs the {@code FrameProcessor} that no further input frames should be accepted. + * + * @throws IllegalStateException If called more than once. + */ + void signalEndOfInputStream(); + + /** + * Releases all resources. + * + *

If the frame processor is released before it has {@linkplain + * Listener#onFrameProcessingEnded() ended}, it will attempt to cancel processing any input frames + * that have already become available. Input frames that become available after release are + * ignored. + * + *

This method blocks until all resources are released or releasing times out. + */ + void release(); +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index 38917b99b5..fc73daeb36 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -37,37 +37,12 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; /** - * {@code GlEffectsFrameProcessor} applies changes to individual video frames. - * - *

Input becomes available on its {@linkplain #getInputSurface() input surface} asynchronously - * and is processed on a background thread as it becomes available. All input frames should be - * {@linkplain #registerInputFrame() registered} before they are rendered to the input surface. - * {@link #getPendingInputFrameCount()} can be used to check whether there are frames that have not - * been fully processed yet. Output is written to the provided {@linkplain #create(Context, - * Listener, float, int, int, long, List, SurfaceInfo.Provider, Transformer.DebugViewProvider, - * boolean) output surface}. + * A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a + * background thread. */ -// TODO(b/227625423): Factor out FrameProcessor interface -/* package */ final class GlEffectsFrameProcessor { - - /** - * Listener for asynchronous frame processing events. - * - *

This listener is only called from the {@link GlEffectsFrameProcessor} instance's background - * thread. - */ - public interface Listener { - /** - * Called when an exception occurs during asynchronous frame processing. - * - *

If an error occurred, consuming and producing further frames will not work as expected and - * the {@link GlEffectsFrameProcessor} should be released. - */ - void onFrameProcessingError(FrameProcessingException exception); - - /** Called after the frame processor has produced its final output frame. */ - void onFrameProcessingEnded(); - } +/* package */ final class GlEffectsFrameProcessor implements FrameProcessor { + // TODO(b/227625423): Replace factory method with setters once output surface and effects can be + // replaced. /** * Creates a new instance. @@ -89,7 +64,7 @@ import java.util.concurrent.atomic.AtomicInteger; */ public static GlEffectsFrameProcessor create( Context context, - GlEffectsFrameProcessor.Listener listener, + FrameProcessor.Listener listener, float pixelWidthHeightRatio, int inputWidth, int inputHeight, @@ -141,7 +116,7 @@ import java.util.concurrent.atomic.AtomicInteger; @WorkerThread private static GlEffectsFrameProcessor createOpenGlObjectsAndFrameProcessor( Context context, - GlEffectsFrameProcessor.Listener listener, + FrameProcessor.Listener listener, float pixelWidthHeightRatio, int inputWidth, int inputHeight, @@ -245,7 +220,7 @@ import java.util.concurrent.atomic.AtomicInteger; ImmutableList.Builder matrixTransformationListBuilder, SurfaceInfo.Provider outputSurfaceProvider, long streamOffsetUs, - GlEffectsFrameProcessor.Listener listener, + FrameProcessor.Listener listener, Transformer.DebugViewProvider debugViewProvider, boolean enableExperimentalHdrEditing) throws FrameProcessingException { @@ -290,7 +265,7 @@ import java.util.concurrent.atomic.AtomicInteger; ExternalTextureProcessor externalTextureProcessor, ImmutableList textureProcessors, FrameProcessingTaskExecutor frameProcessingTaskExecutor, - GlEffectsFrameProcessor.Listener listener) { + FrameProcessor.Listener listener) { externalTextureProcessor.setListener( new ChainingGlTextureProcessorListener( /* previousGlTextureProcessor= */ null, @@ -366,7 +341,7 @@ import java.util.concurrent.atomic.AtomicInteger; inputSurfaceTextureTransformMatrix = new float[16]; } - /** Returns the input {@link Surface}. */ + @Override public Surface getInputSurface() { // TODO(b/227625423): Allow input surface to be recreated for input size change. inputSurfaceTexture.setOnFrameAvailableListener( @@ -374,47 +349,25 @@ import java.util.concurrent.atomic.AtomicInteger; return inputSurface; } - /** - * Informs the {@code GlEffectsFrameProcessor} that a frame will be queued to its input surface. - * - *

Must be called before rendering a frame to the frame processor's input surface. - * - * @throws IllegalStateException If called after {@link #signalEndOfInputStream()}. - */ + @Override public void registerInputFrame() { checkState(!inputStreamEnded); pendingInputFrameCount.incrementAndGet(); } - /** - * Returns the number of input frames that have been {@linkplain #registerInputFrame() registered} - * but not processed off the {@linkplain #getInputSurface() input surface} yet. - */ + @Override public int getPendingInputFrameCount() { return pendingInputFrameCount.get(); } - /** - * Informs the {@code GlEffectsFrameProcessor} that no further input frames should be accepted. - * - * @throws IllegalStateException If called more than once. - */ + @Override public void signalEndOfInputStream() { checkState(!inputStreamEnded); inputStreamEnded = true; frameProcessingTaskExecutor.submit(this::processEndOfInputStream); } - /** - * Releases all resources. - * - *

If the frame processor is released before it has {@linkplain - * Listener#onFrameProcessingEnded() ended}, it will attempt to cancel processing any input frames - * that have already become available. Input frames that become available after release are - * ignored. - * - *

This method blocks until all resources are released or releasing times out. - */ + @Override public void release() { try { frameProcessingTaskExecutor.release( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 99703bc8c1..aab980b4a5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -44,7 +44,7 @@ import org.checkerframework.dataflow.qual.Pure; private final Codec decoder; private final ArrayList decodeOnlyPresentationTimestamps; - private final GlEffectsFrameProcessor frameProcessor; + private final FrameProcessor frameProcessor; private final EncoderWrapper encoderWrapper; private final DecoderInputBuffer encoderOutputBuffer; @@ -102,7 +102,7 @@ import org.checkerframework.dataflow.qual.Pure; frameProcessor = GlEffectsFrameProcessor.create( context, - new GlEffectsFrameProcessor.Listener() { + new FrameProcessor.Listener() { @Override public void onFrameProcessingError(FrameProcessingException exception) { asyncErrorListener.onTransformationException( diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ChainingGlTextureProcessorListenerTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ChainingGlTextureProcessorListenerTest.java index 39bbf1b3d4..c48ca303bf 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ChainingGlTextureProcessorListenerTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ChainingGlTextureProcessorListenerTest.java @@ -31,8 +31,8 @@ import org.junit.runner.RunWith; public final class ChainingGlTextureProcessorListenerTest { private static final long EXECUTOR_WAIT_TIME_MS = 100; - private final GlEffectsFrameProcessor.Listener mockframeProcessorListener = - mock(GlEffectsFrameProcessor.Listener.class); + private final FrameProcessor.Listener mockframeProcessorListener = + mock(FrameProcessor.Listener.class); private final FrameProcessingTaskExecutor frameProcessingTaskExecutor = new FrameProcessingTaskExecutor( Util.newSingleThreadExecutor("Test"), mockframeProcessorListener);