From ecbe9a502cf62c92005a2ff045ab17988ea2838f Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 26 Jul 2022 21:52:30 +0000 Subject: [PATCH] Allow FrameProcessor.Factory to be set on Transformer.Builder. Extract a FrameProcessor.Factory interface from GlEffectsFrameProcessor and allow it to be customized using a setter on Transformer.Builder. PiperOrigin-RevId: 463433438 --- .../GlEffectsFrameProcessorPixelTest.java | 49 ++++++------ .../transformer/FrameProcessor.java | 42 ++++++++++- .../transformer/GlEffectsFrameProcessor.java | 74 +++++++++---------- .../exoplayer2/transformer/Transformer.java | 32 ++++++++ .../transformer/TransformerVideoRenderer.java | 4 + .../VideoTranscodingSamplePipeline.java | 3 +- 6 files changed, 138 insertions(+), 66 deletions(-) diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java index bb9d77f00d..314cd1eac8 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java @@ -356,32 +356,33 @@ public final class GlEffectsFrameProcessorPixelTest { int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); glEffectsFrameProcessor = checkNotNull( - GlEffectsFrameProcessor.create( - context, - new FrameProcessor.Listener() { - @Override - public void onOutputSizeChanged(int width, int height) { - outputImageReader = - ImageReader.newInstance( - width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); - checkNotNull(glEffectsFrameProcessor) - .setOutputSurfaceInfo( - new SurfaceInfo(outputImageReader.getSurface(), width, height)); - } + new GlEffectsFrameProcessor.Factory() + .create( + context, + new FrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + outputImageReader = + ImageReader.newInstance( + width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); + checkNotNull(glEffectsFrameProcessor) + .setOutputSurfaceInfo( + new SurfaceInfo(outputImageReader.getSurface(), width, height)); + } - @Override - public void onFrameProcessingError(FrameProcessingException exception) { - frameProcessingException.set(exception); - } + @Override + public void onFrameProcessingError(FrameProcessingException exception) { + frameProcessingException.set(exception); + } - @Override - public void onFrameProcessingEnded() { - frameProcessingEnded = true; - } - }, - effects, - DebugViewProvider.NONE, - /* useHdr= */ false)); + @Override + public void onFrameProcessingEnded() { + frameProcessingEnded = true; + } + }, + effects, + DebugViewProvider.NONE, + /* useHdr= */ false)); glEffectsFrameProcessor.setInputFrameInfo( new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0)); glEffectsFrameProcessor.registerInputFrame(); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java index 85179d3465..4371996e0e 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java @@ -15,11 +15,49 @@ */ package com.google.android.exoplayer2.transformer; +import android.content.Context; import android.view.Surface; import androidx.annotation.Nullable; +import java.util.List; + +/** + * Interface for a frame processor that applies changes to individual video frames. + * + *

The changes are specified by {@link GlEffect} instances passed to the {@link Factory}. + * + *

The frame processor manages its input {@link Surface} which can be accessed via {@link + * #getInputSurface()}. The output {@link Surface} must be set by the caller using {@link + * #setOutputSurfaceInfo(SurfaceInfo)}. + * + *

The caller must {@linkplain #registerInputFrame() register} input frames before rendering them + * to the input {@link Surface}. + */ +public interface FrameProcessor { + // TODO(b/227625423): Allow effects to be replaced. + + /** A factory for {@link FrameProcessor} instances. */ + interface Factory { + /** + * Creates a new {@link FrameProcessor} instance. + * + * @param context A {@link Context}. + * @param listener A {@link Listener}. + * @param effects The {@link GlEffect} instances to apply to each frame. + * @param debugViewProvider A {@link DebugViewProvider}. + * @param useHdr Whether to process the input as an HDR signal. + * @return A new instance. + * @throws FrameProcessingException If a problem occurs while creating the {@link + * FrameProcessor}. + */ + FrameProcessor create( + Context context, + FrameProcessor.Listener listener, + List effects, + DebugViewProvider debugViewProvider, + boolean useHdr) + throws FrameProcessingException; + } -/** Interface for a frame processor that applies changes to individual video frames. */ -/* package */ interface FrameProcessor { /** * Listener for asynchronous frame processing events. * diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java index 3cdef5f62c..5c5c4ad607 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java @@ -42,51 +42,47 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a * background thread. */ -/* package */ final class GlEffectsFrameProcessor implements FrameProcessor { +public final class GlEffectsFrameProcessor implements FrameProcessor { // TODO(b/227625423): Replace factory method with setters once output surface and effects can be // replaced. - /** - * Creates a new instance. - * - * @param context A {@link Context}. - * @param listener A {@link Listener}. - * @param effects The {@link GlEffect GlEffects} to apply to each frame. - * @param debugViewProvider A {@link DebugViewProvider}. - * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code - * EXT_YUV_target} OpenGL extension. - * @return A new instance. - * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while - * creating and configuring the OpenGL components. - */ - public static GlEffectsFrameProcessor create( - Context context, - FrameProcessor.Listener listener, - List effects, - DebugViewProvider debugViewProvider, - boolean useHdr) - throws FrameProcessingException { + /** A factory for {@link GlEffectsFrameProcessor} instances. */ + public static class Factory implements FrameProcessor.Factory { + /** + * {@inheritDoc} + * + *

Using HDR requires the {@code EXT_YUV_target} OpenGL extension. + */ + @Override + public GlEffectsFrameProcessor create( + Context context, + FrameProcessor.Listener listener, + List effects, + DebugViewProvider debugViewProvider, + boolean useHdr) + throws FrameProcessingException { - ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); + ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); - Future glFrameProcessorFuture = - singleThreadExecutorService.submit( - () -> - createOpenGlObjectsAndFrameProcessor( - context, - listener, - effects, - debugViewProvider, - useHdr, - singleThreadExecutorService)); + Future glFrameProcessorFuture = + singleThreadExecutorService.submit( + () -> + createOpenGlObjectsAndFrameProcessor( + context, + listener, + effects, + debugViewProvider, + useHdr, + singleThreadExecutorService)); - try { - return glFrameProcessorFuture.get(); - } catch (ExecutionException e) { - throw new FrameProcessingException(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new FrameProcessingException(e); + try { + return glFrameProcessorFuture.get(); + } catch (ExecutionException e) { + throw new FrameProcessingException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new FrameProcessingException(e); + } } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 022dde67b6..693b20dafe 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -102,6 +102,7 @@ public final class Transformer { private String containerMimeType; private TransformationRequest transformationRequest; private ImmutableList videoEffects; + private FrameProcessor.Factory frameProcessorFactory; private ListenerSet listeners; private DebugViewProvider debugViewProvider; private Looper looper; @@ -126,6 +127,7 @@ public final class Transformer { containerMimeType = MimeTypes.VIDEO_MP4; transformationRequest = new TransformationRequest.Builder().build(); videoEffects = ImmutableList.of(); + frameProcessorFactory = new GlEffectsFrameProcessor.Factory(); } /** Creates a builder with the values of the provided {@link Transformer}. */ @@ -138,6 +140,7 @@ public final class Transformer { this.containerMimeType = transformer.containerMimeType; this.transformationRequest = transformer.transformationRequest; this.videoEffects = transformer.videoEffects; + this.frameProcessorFactory = transformer.frameProcessorFactory; this.listeners = transformer.listeners; this.looper = transformer.looper; this.encoderFactory = transformer.encoderFactory; @@ -181,6 +184,26 @@ public final class Transformer { return this; } + /** + * Sets the {@link FrameProcessor.Factory} for the {@link FrameProcessor} to use when applying + * {@linkplain GlEffect effects} to the video frames. + * + *

This factory will be used to create the {@link FrameProcessor} used for applying the + * {@link GlEffect} instances passed to {@link #setVideoEffects(List)} and any + * additional {@link GlMatrixTransformation} instances derived from the {@link + * TransformationRequest} set using {@link #setTransformationRequest(TransformationRequest)}. + * + *

The default is {@link GlEffectsFrameProcessor.Factory}. + * + * @param frameProcessorFactory The {@link FrameProcessor.Factory} to use. + * @return This builder. + */ + @CanIgnoreReturnValue + public Builder setFrameProcessorFactory(FrameProcessor.Factory frameProcessorFactory) { + this.frameProcessorFactory = frameProcessorFactory; + return this; + } + /** * Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform. * @@ -438,6 +461,7 @@ public final class Transformer { containerMimeType, transformationRequest, videoEffects, + frameProcessorFactory, listeners, looper, clock, @@ -546,6 +570,7 @@ public final class Transformer { private final String containerMimeType; private final TransformationRequest transformationRequest; private final ImmutableList videoEffects; + private final FrameProcessor.Factory frameProcessorFactory; private final Looper looper; private final Clock clock; private final DebugViewProvider debugViewProvider; @@ -567,6 +592,7 @@ public final class Transformer { String containerMimeType, TransformationRequest transformationRequest, ImmutableList videoEffects, + FrameProcessor.Factory frameProcessorFactory, ListenerSet listeners, Looper looper, Clock clock, @@ -582,6 +608,7 @@ public final class Transformer { this.containerMimeType = containerMimeType; this.transformationRequest = transformationRequest; this.videoEffects = videoEffects; + this.frameProcessorFactory = frameProcessorFactory; this.listeners = listeners; this.looper = looper; this.clock = clock; @@ -731,6 +758,7 @@ public final class Transformer { transformationRequest, mediaItem.clippingConfiguration.startsAtKeyFrame, videoEffects, + frameProcessorFactory, encoderFactory, decoderFactory, new FallbackListener(mediaItem, listeners, transformationRequest), @@ -845,6 +873,7 @@ public final class Transformer { private final TransformationRequest transformationRequest; private final boolean clippingStartsAtKeyFrame; private final ImmutableList videoEffects; + private final FrameProcessor.Factory frameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; private final FallbackListener fallbackListener; @@ -859,6 +888,7 @@ public final class Transformer { TransformationRequest transformationRequest, boolean clippingStartsAtKeyFrame, ImmutableList videoEffects, + FrameProcessor.Factory frameProcessorFactory, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, FallbackListener fallbackListener, @@ -871,6 +901,7 @@ public final class Transformer { this.transformationRequest = transformationRequest; this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; this.videoEffects = videoEffects; + this.frameProcessorFactory = frameProcessorFactory; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; this.fallbackListener = fallbackListener; @@ -910,6 +941,7 @@ public final class Transformer { transformationRequest, clippingStartsAtKeyFrame, videoEffects, + frameProcessorFactory, encoderFactory, decoderFactory, asyncErrorListener, diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java index 58dd5cef6b..032adda9fb 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java @@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final Context context; private final boolean clippingStartsAtKeyFrame; private final ImmutableList effects; + private final FrameProcessor.Factory frameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; private final DebugViewProvider debugViewProvider; @@ -53,6 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationRequest transformationRequest, boolean clippingStartsAtKeyFrame, ImmutableList effects, + FrameProcessor.Factory frameProcessorFactory, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, Transformer.AsyncErrorListener asyncErrorListener, @@ -68,6 +70,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; this.context = context; this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; this.effects = effects; + this.frameProcessorFactory = frameProcessorFactory; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; this.debugViewProvider = debugViewProvider; @@ -113,6 +116,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; streamOffsetUs, transformationRequest, effects, + frameProcessorFactory, decoderFactory, encoderFactory, muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index d4b5ab2341..4887dc7873 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -58,6 +58,7 @@ import org.checkerframework.dataflow.qual.Pure; long streamOffsetUs, TransformationRequest transformationRequest, ImmutableList effects, + FrameProcessor.Factory frameProcessorFactory, Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, List allowedOutputMimeTypes, @@ -102,7 +103,7 @@ import org.checkerframework.dataflow.qual.Pure; try { frameProcessor = - GlEffectsFrameProcessor.create( + frameProcessorFactory.create( context, new FrameProcessor.Listener() { @Override