diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java index 4dd20603c8..9d83cd5300 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java @@ -356,32 +356,33 @@ public final class GlEffectsFrameProcessorPixelTest { int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); glEffectsFrameProcessor = checkNotNull( - GlEffectsFrameProcessor.create( - context, - new FrameProcessor.Listener() { - @Override - public void onOutputSizeChanged(int width, int height) { - outputImageReader = - ImageReader.newInstance( - width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); - checkNotNull(glEffectsFrameProcessor) - .setOutputSurfaceInfo( - new SurfaceInfo(outputImageReader.getSurface(), width, height)); - } + new GlEffectsFrameProcessor.Factory() + .create( + context, + new FrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + outputImageReader = + ImageReader.newInstance( + width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); + checkNotNull(glEffectsFrameProcessor) + .setOutputSurfaceInfo( + new SurfaceInfo(outputImageReader.getSurface(), width, height)); + } - @Override - public void onFrameProcessingError(FrameProcessingException exception) { - frameProcessingException.set(exception); - } + @Override + public void onFrameProcessingError(FrameProcessingException exception) { + frameProcessingException.set(exception); + } - @Override - public void onFrameProcessingEnded() { - frameProcessingEnded = true; - } - }, - effects, - DebugViewProvider.NONE, - /* useHdr= */ false)); + @Override + public void onFrameProcessingEnded() { + frameProcessingEnded = true; + } + }, + effects, + DebugViewProvider.NONE, + /* useHdr= */ false)); glEffectsFrameProcessor.setInputFrameInfo( new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0)); glEffectsFrameProcessor.registerInputFrame(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java index 45c1835def..e2eb0250c6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java @@ -15,11 +15,51 @@ */ package androidx.media3.transformer; +import android.content.Context; import android.view.Surface; import androidx.annotation.Nullable; +import androidx.media3.common.util.UnstableApi; +import java.util.List; + +/** + * Interface for a frame processor that applies changes to individual video frames. + * + *

The changes are specified by {@link GlEffect} instances passed to the {@link Factory}. + * + *

The frame processor manages its input {@link Surface} which can be accessed via {@link + * #getInputSurface()}. The output {@link Surface} must be set by the caller using {@link + * #setOutputSurfaceInfo(SurfaceInfo)}. + * + *

The caller must {@linkplain #registerInputFrame() register} input frames before rendering them + * to the input {@link Surface}. + */ +@UnstableApi +public interface FrameProcessor { + // TODO(b/227625423): Allow effects to be replaced. + + /** A factory for {@link FrameProcessor} instances. */ + interface Factory { + /** + * Creates a new {@link FrameProcessor} instance. + * + * @param context A {@link Context}. + * @param listener A {@link Listener}. + * @param effects The {@link GlEffect} instances to apply to each frame. + * @param debugViewProvider A {@link DebugViewProvider}. + * @param useHdr Whether to process the input as an HDR signal. + * @return A new instance. + * @throws FrameProcessingException If a problem occurs while creating the {@link + * FrameProcessor}. + */ + FrameProcessor create( + Context context, + FrameProcessor.Listener listener, + List effects, + DebugViewProvider debugViewProvider, + boolean useHdr) + throws FrameProcessingException; + } -/** Interface for a frame processor that applies changes to individual video frames. */ -/* package */ interface FrameProcessor { /** * Listener for asynchronous frame processing events. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index c3f582291c..aa1d6c5902 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -29,6 +29,7 @@ import androidx.annotation.Nullable; import androidx.annotation.WorkerThread; import androidx.media3.common.C; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import java.util.List; @@ -42,51 +43,48 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a * background thread. */ -/* package */ final class GlEffectsFrameProcessor implements FrameProcessor { +@UnstableApi +public final class GlEffectsFrameProcessor implements FrameProcessor { // TODO(b/227625423): Replace factory method with setters once output surface and effects can be // replaced. - /** - * Creates a new instance. - * - * @param context A {@link Context}. - * @param listener A {@link Listener}. - * @param effects The {@link GlEffect GlEffects} to apply to each frame. - * @param debugViewProvider A {@link DebugViewProvider}. - * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code - * EXT_YUV_target} OpenGL extension. - * @return A new instance. - * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while - * creating and configuring the OpenGL components. - */ - public static GlEffectsFrameProcessor create( - Context context, - FrameProcessor.Listener listener, - List effects, - DebugViewProvider debugViewProvider, - boolean useHdr) - throws FrameProcessingException { + /** A factory for {@link GlEffectsFrameProcessor} instances. */ + public static class Factory implements FrameProcessor.Factory { + /** + * {@inheritDoc} + * + *

Using HDR requires the {@code EXT_YUV_target} OpenGL extension. + */ + @Override + public GlEffectsFrameProcessor create( + Context context, + FrameProcessor.Listener listener, + List effects, + DebugViewProvider debugViewProvider, + boolean useHdr) + throws FrameProcessingException { - ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); + ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); - Future glFrameProcessorFuture = - singleThreadExecutorService.submit( - () -> - createOpenGlObjectsAndFrameProcessor( - context, - listener, - effects, - debugViewProvider, - useHdr, - singleThreadExecutorService)); + Future glFrameProcessorFuture = + singleThreadExecutorService.submit( + () -> + createOpenGlObjectsAndFrameProcessor( + context, + listener, + effects, + debugViewProvider, + useHdr, + singleThreadExecutorService)); - try { - return glFrameProcessorFuture.get(); - } catch (ExecutionException e) { - throw new FrameProcessingException(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new FrameProcessingException(e); + try { + return glFrameProcessorFuture.get(); + } catch (ExecutionException e) { + throw new FrameProcessingException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new FrameProcessingException(e); + } } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 0016bf3591..d35feefce1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -104,6 +104,7 @@ public final class Transformer { private String containerMimeType; private TransformationRequest transformationRequest; private ImmutableList videoEffects; + private FrameProcessor.Factory frameProcessorFactory; private ListenerSet listeners; private DebugViewProvider debugViewProvider; private Looper looper; @@ -128,6 +129,7 @@ public final class Transformer { containerMimeType = MimeTypes.VIDEO_MP4; transformationRequest = new TransformationRequest.Builder().build(); videoEffects = ImmutableList.of(); + frameProcessorFactory = new GlEffectsFrameProcessor.Factory(); } /** Creates a builder with the values of the provided {@link Transformer}. */ @@ -140,6 +142,7 @@ public final class Transformer { this.containerMimeType = transformer.containerMimeType; this.transformationRequest = transformer.transformationRequest; this.videoEffects = transformer.videoEffects; + this.frameProcessorFactory = transformer.frameProcessorFactory; this.listeners = transformer.listeners; this.looper = transformer.looper; this.encoderFactory = transformer.encoderFactory; @@ -183,6 +186,26 @@ public final class Transformer { return this; } + /** + * Sets the {@link FrameProcessor.Factory} for the {@link FrameProcessor} to use when applying + * {@linkplain GlEffect effects} to the video frames. + * + *

This factory will be used to create the {@link FrameProcessor} used for applying the + * {@link GlEffect} instances passed to {@link #setVideoEffects(List)} and any + * additional {@link GlMatrixTransformation} instances derived from the {@link + * TransformationRequest} set using {@link #setTransformationRequest(TransformationRequest)}. + * + *

The default is {@link GlEffectsFrameProcessor.Factory}. + * + * @param frameProcessorFactory The {@link FrameProcessor.Factory} to use. + * @return This builder. + */ + @CanIgnoreReturnValue + public Builder setFrameProcessorFactory(FrameProcessor.Factory frameProcessorFactory) { + this.frameProcessorFactory = frameProcessorFactory; + return this; + } + /** * Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform. * @@ -440,6 +463,7 @@ public final class Transformer { containerMimeType, transformationRequest, videoEffects, + frameProcessorFactory, listeners, looper, clock, @@ -548,6 +572,7 @@ public final class Transformer { private final String containerMimeType; private final TransformationRequest transformationRequest; private final ImmutableList videoEffects; + private final FrameProcessor.Factory frameProcessorFactory; private final Looper looper; private final Clock clock; private final DebugViewProvider debugViewProvider; @@ -569,6 +594,7 @@ public final class Transformer { String containerMimeType, TransformationRequest transformationRequest, ImmutableList videoEffects, + FrameProcessor.Factory frameProcessorFactory, ListenerSet listeners, Looper looper, Clock clock, @@ -584,6 +610,7 @@ public final class Transformer { this.containerMimeType = containerMimeType; this.transformationRequest = transformationRequest; this.videoEffects = videoEffects; + this.frameProcessorFactory = frameProcessorFactory; this.listeners = listeners; this.looper = looper; this.clock = clock; @@ -733,6 +760,7 @@ public final class Transformer { transformationRequest, mediaItem.clippingConfiguration.startsAtKeyFrame, videoEffects, + frameProcessorFactory, encoderFactory, decoderFactory, new FallbackListener(mediaItem, listeners, transformationRequest), @@ -847,6 +875,7 @@ public final class Transformer { private final TransformationRequest transformationRequest; private final boolean clippingStartsAtKeyFrame; private final ImmutableList videoEffects; + private final FrameProcessor.Factory frameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; private final FallbackListener fallbackListener; @@ -861,6 +890,7 @@ public final class Transformer { TransformationRequest transformationRequest, boolean clippingStartsAtKeyFrame, ImmutableList videoEffects, + FrameProcessor.Factory frameProcessorFactory, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, FallbackListener fallbackListener, @@ -873,6 +903,7 @@ public final class Transformer { this.transformationRequest = transformationRequest; this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; this.videoEffects = videoEffects; + this.frameProcessorFactory = frameProcessorFactory; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; this.fallbackListener = fallbackListener; @@ -912,6 +943,7 @@ public final class Transformer { transformationRequest, clippingStartsAtKeyFrame, videoEffects, + frameProcessorFactory, encoderFactory, decoderFactory, asyncErrorListener, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index b52ea8e054..f1587042d5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final Context context; private final boolean clippingStartsAtKeyFrame; private final ImmutableList effects; + private final FrameProcessor.Factory frameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; private final DebugViewProvider debugViewProvider; @@ -53,6 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationRequest transformationRequest, boolean clippingStartsAtKeyFrame, ImmutableList effects, + FrameProcessor.Factory frameProcessorFactory, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, Transformer.AsyncErrorListener asyncErrorListener, @@ -68,6 +70,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; this.context = context; this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; this.effects = effects; + this.frameProcessorFactory = frameProcessorFactory; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; this.debugViewProvider = debugViewProvider; @@ -113,6 +116,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; streamOffsetUs, transformationRequest, effects, + frameProcessorFactory, decoderFactory, encoderFactory, muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 4f2344c1af..c3496e0b3c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -58,6 +58,7 @@ import org.checkerframework.dataflow.qual.Pure; long streamOffsetUs, TransformationRequest transformationRequest, ImmutableList effects, + FrameProcessor.Factory frameProcessorFactory, Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, List allowedOutputMimeTypes, @@ -102,7 +103,7 @@ import org.checkerframework.dataflow.qual.Pure; try { frameProcessor = - GlEffectsFrameProcessor.create( + frameProcessorFactory.create( context, new FrameProcessor.Listener() { @Override