From 6e390771a00606edf9852b77526a5ad47b6874c5 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 27 Jul 2022 22:40:20 +0000 Subject: [PATCH] Add Effect marker interface. This allows non-GL effects to be passed to custom FrameProcessor implementations. PiperOrigin-RevId: 463696384 (cherry picked from commit 580e44fc47ec5b62da5a133d12048a20d716e94f) --- .../demo/transformer/TransformerActivity.java | 57 ++++++++++--------- .../GlEffectsFrameProcessorPixelTest.java | 4 +- .../androidx/media3/transformer/Effect.java | 23 ++++++++ .../media3/transformer/FrameProcessor.java | 6 +- .../androidx/media3/transformer/GlEffect.java | 2 +- .../transformer/GlEffectsFrameProcessor.java | 23 +++++--- .../media3/transformer/Transformer.java | 30 +++++----- .../transformer/TransformerVideoRenderer.java | 4 +- .../VideoTranscodingSamplePipeline.java | 8 +-- 9 files changed, 98 insertions(+), 59 deletions(-) create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/Effect.java diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index bd622f7dd0..bc0e3fdfb9 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -44,6 +44,7 @@ import androidx.media3.exoplayer.util.DebugTextViewHelper; import androidx.media3.transformer.Contrast; import androidx.media3.transformer.DebugViewProvider; import androidx.media3.transformer.DefaultEncoderFactory; +import androidx.media3.transformer.Effect; import androidx.media3.transformer.GlEffect; import androidx.media3.transformer.GlTextureProcessor; import androidx.media3.transformer.ProgressHolder; @@ -265,7 +266,7 @@ public final class TransformerActivity extends AppCompatActivity { .setEnableFallback(bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK)) .build()); - ImmutableList.Builder effects = new ImmutableList.Builder<>(); + ImmutableList.Builder effects = new ImmutableList.Builder<>(); @Nullable boolean[] selectedEffects = bundle.getBooleanArray(ConfigurationActivity.DEMO_EFFECTS_SELECTIONS); @@ -280,43 +281,45 @@ public final class TransformerActivity extends AppCompatActivity { clazz.getConstructor( Context.class, Boolean.class, String.class, String.class, String.class); effects.add( - (Context context, boolean useHdr) -> { - try { - return (GlTextureProcessor) - constructor.newInstance( - context, - useHdr, - /* graphName= */ "edge_detector_mediapipe_graph.binarypb", - /* inputStreamName= */ "input_video", - /* outputStreamName= */ "output_video"); - } catch (Exception e) { - runOnUiThread(() -> showToast(R.string.no_media_pipe_error)); - throw new RuntimeException("Failed to load MediaPipe processor", e); - } - }); + (GlEffect) + (Context context, boolean useHdr) -> { + try { + return (GlTextureProcessor) + constructor.newInstance( + context, + useHdr, + /* graphName= */ "edge_detector_mediapipe_graph.binarypb", + /* inputStreamName= */ "input_video", + /* outputStreamName= */ "output_video"); + } catch (Exception e) { + runOnUiThread(() -> showToast(R.string.no_media_pipe_error)); + throw new RuntimeException("Failed to load MediaPipe processor", e); + } + }); } catch (Exception e) { showToast(R.string.no_media_pipe_error); } } if (selectedEffects[2]) { effects.add( - (Context context, boolean useHdr) -> - new PeriodicVignetteProcessor( - context, - useHdr, - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), - /* minInnerRadius= */ bundle.getFloat( - ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS), - /* maxInnerRadius= */ bundle.getFloat( - ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS), - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); + (GlEffect) + (Context context, boolean useHdr) -> + new PeriodicVignetteProcessor( + context, + useHdr, + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), + /* minInnerRadius= */ bundle.getFloat( + ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS), + /* maxInnerRadius= */ bundle.getFloat( + ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS), + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); } if (selectedEffects[3]) { effects.add(MatrixTransformationFactory.createSpin3dEffect()); } if (selectedEffects[4]) { - effects.add(BitmapOverlayProcessor::new); + effects.add((GlEffect) BitmapOverlayProcessor::new); } if (selectedEffects[5]) { effects.add(MatrixTransformationFactory.createZoomInTransition()); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java index 9d83cd5300..404feb0e49 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java @@ -295,7 +295,7 @@ public final class GlEffectsFrameProcessorPixelTest { "processData_withManyComposedMatrixTransformations_producesSameOutputAsCombinedTransformation"; Crop centerCrop = new Crop(/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f); - ImmutableList.Builder full10StepRotationAndCenterCrop = new ImmutableList.Builder<>(); + ImmutableList.Builder full10StepRotationAndCenterCrop = new ImmutableList.Builder<>(); for (int i = 0; i < 10; i++) { full10StepRotationAndCenterCrop.add(new Rotation(/* degrees= */ 36)); } @@ -336,7 +336,7 @@ public final class GlEffectsFrameProcessorPixelTest { setUpAndPrepareFirstFrame(pixelWidthHeightRatio, asList(effects)); } - private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, List effects) + private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, List effects) throws Exception { // Set up the extractor to read the first video frame and get its format. MediaExtractor mediaExtractor = new MediaExtractor(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Effect.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Effect.java new file mode 100644 index 0000000000..6d85e2f38d --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Effect.java @@ -0,0 +1,23 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import androidx.media3.common.util.UnstableApi; + +/** Marker interface for a video frame effect. */ +@UnstableApi +public interface Effect {} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java index e2eb0250c6..26a370b18f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java @@ -24,7 +24,7 @@ import java.util.List; /** * Interface for a frame processor that applies changes to individual video frames. * - *

The changes are specified by {@link GlEffect} instances passed to the {@link Factory}. + *

The changes are specified by {@link Effect} instances passed to the {@link Factory}. * *

The frame processor manages its input {@link Surface} which can be accessed via {@link * #getInputSurface()}. The output {@link Surface} must be set by the caller using {@link @@ -44,7 +44,7 @@ public interface FrameProcessor { * * @param context A {@link Context}. * @param listener A {@link Listener}. - * @param effects The {@link GlEffect} instances to apply to each frame. + * @param effects The {@link Effect} instances to apply to each frame. * @param debugViewProvider A {@link DebugViewProvider}. * @param useHdr Whether to process the input as an HDR signal. * @return A new instance. @@ -54,7 +54,7 @@ public interface FrameProcessor { FrameProcessor create( Context context, FrameProcessor.Listener listener, - List effects, + List effects, DebugViewProvider debugViewProvider, boolean useHdr) throws FrameProcessingException; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java index dde2b52012..aa69767be8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java @@ -26,7 +26,7 @@ import androidx.media3.common.util.UnstableApi; * the effect. */ @UnstableApi -public interface GlEffect { +public interface GlEffect extends Effect { /** * Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index aa1d6c5902..808215dc41 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -15,6 +15,7 @@ */ package androidx.media3.transformer; +import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static com.google.common.collect.Iterables.getLast; @@ -53,13 +54,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { /** * {@inheritDoc} * + *

All {@link Effect} instances must be {@link GlEffect} instances. + * *

Using HDR requires the {@code EXT_YUV_target} OpenGL extension. */ @Override public GlEffectsFrameProcessor create( Context context, FrameProcessor.Listener listener, - List effects, + List effects, DebugViewProvider debugViewProvider, boolean useHdr) throws FrameProcessingException { @@ -93,6 +96,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { * GlTextureProcessor} instances corresponding to the {@link GlEffect} instances, and returns a * new {@code GlEffectsFrameProcessor}. * + *

All {@link Effect} instances must be {@link GlEffect} instances. + * *

This method must be executed using the {@code singleThreadExecutorService}, as later OpenGL * commands will be called on that thread. */ @@ -100,7 +105,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { private static GlEffectsFrameProcessor createOpenGlObjectsAndFrameProcessor( Context context, FrameProcessor.Listener listener, - List effects, + List effects, DebugViewProvider debugViewProvider, boolean useHdr, ExecutorService singleThreadExecutorService) @@ -142,13 +147,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { * MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate * {@link GlTextureProcessor} instances. * + *

All {@link Effect} instances must be {@link GlEffect} instances. + * * @return A non-empty list of {@link GlTextureProcessor} instances to apply in the given order. * The first is an {@link ExternalTextureProcessor} and the last is a {@link * FinalMatrixTransformationProcessorWrapper}. */ private static ImmutableList getGlTextureProcessorsForGlEffects( Context context, - List effects, + List effects, EGLDisplay eglDisplay, EGLContext eglContext, FrameProcessor.Listener listener, @@ -161,9 +168,11 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { new ImmutableList.Builder<>(); boolean sampleFromExternalTexture = true; for (int i = 0; i < effects.size(); i++) { - GlEffect effect = effects.get(i); - if (effect instanceof GlMatrixTransformation) { - matrixTransformationListBuilder.add((GlMatrixTransformation) effect); + Effect effect = effects.get(i); + checkArgument(effect instanceof GlEffect, "GlEffectsFrameProcessor only supports GlEffects"); + GlEffect glEffect = (GlEffect) effect; + if (glEffect instanceof GlMatrixTransformation) { + matrixTransformationListBuilder.add((GlMatrixTransformation) glEffect); continue; } ImmutableList matrixTransformations = @@ -179,7 +188,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { matrixTransformationListBuilder = new ImmutableList.Builder<>(); sampleFromExternalTexture = false; } - textureProcessorListBuilder.add(effect.toGlTextureProcessor(context, useHdr)); + textureProcessorListBuilder.add(glEffect.toGlTextureProcessor(context, useHdr)); } textureProcessorListBuilder.add( new FinalMatrixTransformationProcessorWrapper( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index d35feefce1..cd8642b18c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -103,7 +103,7 @@ public final class Transformer { private boolean removeVideo; private String containerMimeType; private TransformationRequest transformationRequest; - private ImmutableList videoEffects; + private ImmutableList videoEffects; private FrameProcessor.Factory frameProcessorFactory; private ListenerSet listeners; private DebugViewProvider debugViewProvider; @@ -168,32 +168,36 @@ public final class Transformer { } /** - * Sets the {@linkplain GlEffect effects} to apply to each video frame. + * Sets the {@link Effect} instances to apply to each video frame. * - *

The {@linkplain GlEffect effects} are applied before any {@linkplain + *

The {@link Effect} instances are applied before any {@linkplain * TransformationRequest.Builder#setScale(float, float) scale}, {@linkplain * TransformationRequest.Builder#setRotationDegrees(float) rotation}, or {@linkplain * TransformationRequest.Builder#setResolution(int) resolution} changes specified in the {@link * #setTransformationRequest(TransformationRequest) TransformationRequest} but after {@linkplain * TransformationRequest.Builder#setFlattenForSlowMotion(boolean) slow-motion flattening}. * - * @param effects The {@linkplain GlEffect effects} to apply to each video frame. + *

The default {@link FrameProcessor} only supports {@link GlEffect} instances. To use other + * effects, call {@link #setFrameProcessorFactory(FrameProcessor.Factory)} with a custom {@link + * FrameProcessor.Factory}. + * + * @param effects The {@link Effect} instances to apply to each video frame. * @return This builder. */ @CanIgnoreReturnValue - public Builder setVideoEffects(List effects) { + public Builder setVideoEffects(List effects) { this.videoEffects = ImmutableList.copyOf(effects); return this; } /** * Sets the {@link FrameProcessor.Factory} for the {@link FrameProcessor} to use when applying - * {@linkplain GlEffect effects} to the video frames. + * {@linkplain Effect effects} to the video frames. * *

This factory will be used to create the {@link FrameProcessor} used for applying the - * {@link GlEffect} instances passed to {@link #setVideoEffects(List)} and any - * additional {@link GlMatrixTransformation} instances derived from the {@link - * TransformationRequest} set using {@link #setTransformationRequest(TransformationRequest)}. + * {@link Effect} instances passed to {@link #setVideoEffects(List)} and any additional + * {@link GlMatrixTransformation} instances derived from the {@link TransformationRequest} set + * using {@link #setTransformationRequest(TransformationRequest)}. * *

The default is {@link GlEffectsFrameProcessor.Factory}. * @@ -571,7 +575,7 @@ public final class Transformer { private final boolean removeVideo; private final String containerMimeType; private final TransformationRequest transformationRequest; - private final ImmutableList videoEffects; + private final ImmutableList videoEffects; private final FrameProcessor.Factory frameProcessorFactory; private final Looper looper; private final Clock clock; @@ -593,7 +597,7 @@ public final class Transformer { boolean removeVideo, String containerMimeType, TransformationRequest transformationRequest, - ImmutableList videoEffects, + ImmutableList videoEffects, FrameProcessor.Factory frameProcessorFactory, ListenerSet listeners, Looper looper, @@ -874,7 +878,7 @@ public final class Transformer { private final boolean removeVideo; private final TransformationRequest transformationRequest; private final boolean clippingStartsAtKeyFrame; - private final ImmutableList videoEffects; + private final ImmutableList videoEffects; private final FrameProcessor.Factory frameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; @@ -889,7 +893,7 @@ public final class Transformer { boolean removeVideo, TransformationRequest transformationRequest, boolean clippingStartsAtKeyFrame, - ImmutableList videoEffects, + ImmutableList videoEffects, FrameProcessor.Factory frameProcessorFactory, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index f1587042d5..91f4766c6c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -38,7 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final Context context; private final boolean clippingStartsAtKeyFrame; - private final ImmutableList effects; + private final ImmutableList effects; private final FrameProcessor.Factory frameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; @@ -53,7 +53,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformerMediaClock mediaClock, TransformationRequest transformationRequest, boolean clippingStartsAtKeyFrame, - ImmutableList effects, + ImmutableList effects, FrameProcessor.Factory frameProcessorFactory, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index c3496e0b3c..187eaef9ec 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -57,7 +57,7 @@ import org.checkerframework.dataflow.qual.Pure; Format inputFormat, long streamOffsetUs, TransformationRequest transformationRequest, - ImmutableList effects, + ImmutableList effects, FrameProcessor.Factory frameProcessorFactory, Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, @@ -78,8 +78,8 @@ import org.checkerframework.dataflow.qual.Pure; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; - ImmutableList.Builder effectsListBuilder = - new ImmutableList.Builder().addAll(effects); + ImmutableList.Builder effectsListBuilder = + new ImmutableList.Builder().addAll(effects); if (transformationRequest.scaleX != 1f || transformationRequest.scaleY != 1f || transformationRequest.rotationDegrees != 0f) { @@ -136,7 +136,7 @@ import org.checkerframework.dataflow.qual.Pure; debugViewProvider, // HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This // implies that the OpenGL EXT_YUV_target extension is supported and hence the - // GlEffectsFrameProcessor also supports HDR. + // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. /* useHdr= */ encoderWrapper.isHdrEditingEnabled()); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessingException(