From 5c7dcf42a0aab8c33d26dfa526516c6d7810756c Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 27 Jul 2022 22:40:20 +0000 Subject: [PATCH] Add Effect marker interface. This allows non-GL effects to be passed to custom FrameProcessor implementations. PiperOrigin-RevId: 463696384 (cherry picked from commit 66dde42989dbc8fedeee59897a05644d86240cf6) --- .../transformerdemo/TransformerActivity.java | 57 ++++++++++--------- .../GlEffectsFrameProcessorPixelTest.java | 4 +- .../exoplayer2/transformer/Effect.java | 20 +++++++ .../transformer/FrameProcessor.java | 6 +- .../exoplayer2/transformer/GlEffect.java | 2 +- .../transformer/GlEffectsFrameProcessor.java | 23 +++++--- .../exoplayer2/transformer/Transformer.java | 30 +++++----- .../transformer/TransformerVideoRenderer.java | 4 +- .../VideoTranscodingSamplePipeline.java | 8 +-- 9 files changed, 95 insertions(+), 59 deletions(-) create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Effect.java diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java index 144c804743..f7e27c89c3 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java @@ -41,6 +41,7 @@ import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.transformer.Contrast; import com.google.android.exoplayer2.transformer.DebugViewProvider; import com.google.android.exoplayer2.transformer.DefaultEncoderFactory; +import com.google.android.exoplayer2.transformer.Effect; import com.google.android.exoplayer2.transformer.GlEffect; import com.google.android.exoplayer2.transformer.GlTextureProcessor; import com.google.android.exoplayer2.transformer.ProgressHolder; @@ -265,7 +266,7 @@ public final class TransformerActivity extends AppCompatActivity { .setEnableFallback(bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK)) .build()); - ImmutableList.Builder effects = new ImmutableList.Builder<>(); + ImmutableList.Builder effects = new ImmutableList.Builder<>(); @Nullable boolean[] selectedEffects = bundle.getBooleanArray(ConfigurationActivity.DEMO_EFFECTS_SELECTIONS); @@ -281,43 +282,45 @@ public final class TransformerActivity extends AppCompatActivity { clazz.getConstructor( Context.class, Boolean.class, String.class, String.class, String.class); effects.add( - (Context context, boolean useHdr) -> { - try { - return (GlTextureProcessor) - constructor.newInstance( - context, - useHdr, - /* graphName= */ "edge_detector_mediapipe_graph.binarypb", - /* inputStreamName= */ "input_video", - /* outputStreamName= */ "output_video"); - } catch (Exception e) { - runOnUiThread(() -> showToast(R.string.no_media_pipe_error)); - throw new RuntimeException("Failed to load MediaPipe processor", e); - } - }); + (GlEffect) + (Context context, boolean useHdr) -> { + try { + return (GlTextureProcessor) + constructor.newInstance( + context, + useHdr, + /* graphName= */ "edge_detector_mediapipe_graph.binarypb", + /* inputStreamName= */ "input_video", + /* outputStreamName= */ "output_video"); + } catch (Exception e) { + runOnUiThread(() -> showToast(R.string.no_media_pipe_error)); + throw new RuntimeException("Failed to load MediaPipe processor", e); + } + }); } catch (Exception e) { showToast(R.string.no_media_pipe_error); } } if (selectedEffects[2]) { effects.add( - (Context context, boolean useHdr) -> - new PeriodicVignetteProcessor( - context, - useHdr, - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), - /* minInnerRadius= */ bundle.getFloat( - ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS), - /* maxInnerRadius= */ bundle.getFloat( - ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS), - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); + (GlEffect) + (Context context, boolean useHdr) -> + new PeriodicVignetteProcessor( + context, + useHdr, + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), + /* minInnerRadius= */ bundle.getFloat( + ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS), + /* maxInnerRadius= */ bundle.getFloat( + ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS), + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); } if (selectedEffects[3]) { effects.add(MatrixTransformationFactory.createSpin3dEffect()); } if (selectedEffects[4]) { - effects.add(BitmapOverlayProcessor::new); + effects.add((GlEffect) BitmapOverlayProcessor::new); } if (selectedEffects[5]) { effects.add(MatrixTransformationFactory.createZoomInTransition()); diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java index 314cd1eac8..ac473380bd 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java @@ -295,7 +295,7 @@ public final class GlEffectsFrameProcessorPixelTest { "processData_withManyComposedMatrixTransformations_producesSameOutputAsCombinedTransformation"; Crop centerCrop = new Crop(/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f); - ImmutableList.Builder full10StepRotationAndCenterCrop = new ImmutableList.Builder<>(); + ImmutableList.Builder full10StepRotationAndCenterCrop = new ImmutableList.Builder<>(); for (int i = 0; i < 10; i++) { full10StepRotationAndCenterCrop.add(new Rotation(/* degrees= */ 36)); } @@ -336,7 +336,7 @@ public final class GlEffectsFrameProcessorPixelTest { setUpAndPrepareFirstFrame(pixelWidthHeightRatio, asList(effects)); } - private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, List effects) + private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, List effects) throws Exception { // Set up the extractor to read the first video frame and get its format. MediaExtractor mediaExtractor = new MediaExtractor(); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Effect.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Effect.java new file mode 100644 index 0000000000..af38cd1cdc --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Effect.java @@ -0,0 +1,20 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +/** Marker interface for a video frame effect. */ +public interface Effect {} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java index 4371996e0e..fd4c4822ca 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java @@ -23,7 +23,7 @@ import java.util.List; /** * Interface for a frame processor that applies changes to individual video frames. * - *

The changes are specified by {@link GlEffect} instances passed to the {@link Factory}. + *

The changes are specified by {@link Effect} instances passed to the {@link Factory}. * *

The frame processor manages its input {@link Surface} which can be accessed via {@link * #getInputSurface()}. The output {@link Surface} must be set by the caller using {@link @@ -42,7 +42,7 @@ public interface FrameProcessor { * * @param context A {@link Context}. * @param listener A {@link Listener}. - * @param effects The {@link GlEffect} instances to apply to each frame. + * @param effects The {@link Effect} instances to apply to each frame. * @param debugViewProvider A {@link DebugViewProvider}. * @param useHdr Whether to process the input as an HDR signal. * @return A new instance. @@ -52,7 +52,7 @@ public interface FrameProcessor { FrameProcessor create( Context context, FrameProcessor.Listener listener, - List effects, + List effects, DebugViewProvider debugViewProvider, boolean useHdr) throws FrameProcessingException; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java index 1394e0cd5a..5f82530246 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java @@ -24,7 +24,7 @@ import android.content.Context; * #toGlTextureProcessor(Context, boolean) converted} to a {@link GlTextureProcessor} which applies * the effect. */ -public interface GlEffect { +public interface GlEffect extends Effect { /** * Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java index 5c5c4ad607..27d3c9d269 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java @@ -15,6 +15,7 @@ */ package com.google.android.exoplayer2.transformer; +import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.common.collect.Iterables.getLast; @@ -51,13 +52,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { /** * {@inheritDoc} * + *

All {@link Effect} instances must be {@link GlEffect} instances. + * *

Using HDR requires the {@code EXT_YUV_target} OpenGL extension. */ @Override public GlEffectsFrameProcessor create( Context context, FrameProcessor.Listener listener, - List effects, + List effects, DebugViewProvider debugViewProvider, boolean useHdr) throws FrameProcessingException { @@ -91,6 +94,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { * GlTextureProcessor} instances corresponding to the {@link GlEffect} instances, and returns a * new {@code GlEffectsFrameProcessor}. * + *

All {@link Effect} instances must be {@link GlEffect} instances. + * *

This method must be executed using the {@code singleThreadExecutorService}, as later OpenGL * commands will be called on that thread. */ @@ -98,7 +103,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { private static GlEffectsFrameProcessor createOpenGlObjectsAndFrameProcessor( Context context, FrameProcessor.Listener listener, - List effects, + List effects, DebugViewProvider debugViewProvider, boolean useHdr, ExecutorService singleThreadExecutorService) @@ -140,13 +145,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { * MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate * {@link GlTextureProcessor} instances. * + *

All {@link Effect} instances must be {@link GlEffect} instances. + * * @return A non-empty list of {@link GlTextureProcessor} instances to apply in the given order. * The first is an {@link ExternalTextureProcessor} and the last is a {@link * FinalMatrixTransformationProcessorWrapper}. */ private static ImmutableList getGlTextureProcessorsForGlEffects( Context context, - List effects, + List effects, EGLDisplay eglDisplay, EGLContext eglContext, FrameProcessor.Listener listener, @@ -159,9 +166,11 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { new ImmutableList.Builder<>(); boolean sampleFromExternalTexture = true; for (int i = 0; i < effects.size(); i++) { - GlEffect effect = effects.get(i); - if (effect instanceof GlMatrixTransformation) { - matrixTransformationListBuilder.add((GlMatrixTransformation) effect); + Effect effect = effects.get(i); + checkArgument(effect instanceof GlEffect, "GlEffectsFrameProcessor only supports GlEffects"); + GlEffect glEffect = (GlEffect) effect; + if (glEffect instanceof GlMatrixTransformation) { + matrixTransformationListBuilder.add((GlMatrixTransformation) glEffect); continue; } ImmutableList matrixTransformations = @@ -177,7 +186,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { matrixTransformationListBuilder = new ImmutableList.Builder<>(); sampleFromExternalTexture = false; } - textureProcessorListBuilder.add(effect.toGlTextureProcessor(context, useHdr)); + textureProcessorListBuilder.add(glEffect.toGlTextureProcessor(context, useHdr)); } textureProcessorListBuilder.add( new FinalMatrixTransformationProcessorWrapper( diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 693b20dafe..ca71fc3d4b 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -101,7 +101,7 @@ public final class Transformer { private boolean removeVideo; private String containerMimeType; private TransformationRequest transformationRequest; - private ImmutableList videoEffects; + private ImmutableList videoEffects; private FrameProcessor.Factory frameProcessorFactory; private ListenerSet listeners; private DebugViewProvider debugViewProvider; @@ -166,32 +166,36 @@ public final class Transformer { } /** - * Sets the {@linkplain GlEffect effects} to apply to each video frame. + * Sets the {@link Effect} instances to apply to each video frame. * - *

The {@linkplain GlEffect effects} are applied before any {@linkplain + *

The {@link Effect} instances are applied before any {@linkplain * TransformationRequest.Builder#setScale(float, float) scale}, {@linkplain * TransformationRequest.Builder#setRotationDegrees(float) rotation}, or {@linkplain * TransformationRequest.Builder#setResolution(int) resolution} changes specified in the {@link * #setTransformationRequest(TransformationRequest) TransformationRequest} but after {@linkplain * TransformationRequest.Builder#setFlattenForSlowMotion(boolean) slow-motion flattening}. * - * @param effects The {@linkplain GlEffect effects} to apply to each video frame. + *

The default {@link FrameProcessor} only supports {@link GlEffect} instances. To use other + * effects, call {@link #setFrameProcessorFactory(FrameProcessor.Factory)} with a custom {@link + * FrameProcessor.Factory}. + * + * @param effects The {@link Effect} instances to apply to each video frame. * @return This builder. */ @CanIgnoreReturnValue - public Builder setVideoEffects(List effects) { + public Builder setVideoEffects(List effects) { this.videoEffects = ImmutableList.copyOf(effects); return this; } /** * Sets the {@link FrameProcessor.Factory} for the {@link FrameProcessor} to use when applying - * {@linkplain GlEffect effects} to the video frames. + * {@linkplain Effect effects} to the video frames. * *

This factory will be used to create the {@link FrameProcessor} used for applying the - * {@link GlEffect} instances passed to {@link #setVideoEffects(List)} and any - * additional {@link GlMatrixTransformation} instances derived from the {@link - * TransformationRequest} set using {@link #setTransformationRequest(TransformationRequest)}. + * {@link Effect} instances passed to {@link #setVideoEffects(List)} and any additional + * {@link GlMatrixTransformation} instances derived from the {@link TransformationRequest} set + * using {@link #setTransformationRequest(TransformationRequest)}. * *

The default is {@link GlEffectsFrameProcessor.Factory}. * @@ -569,7 +573,7 @@ public final class Transformer { private final boolean removeVideo; private final String containerMimeType; private final TransformationRequest transformationRequest; - private final ImmutableList videoEffects; + private final ImmutableList videoEffects; private final FrameProcessor.Factory frameProcessorFactory; private final Looper looper; private final Clock clock; @@ -591,7 +595,7 @@ public final class Transformer { boolean removeVideo, String containerMimeType, TransformationRequest transformationRequest, - ImmutableList videoEffects, + ImmutableList videoEffects, FrameProcessor.Factory frameProcessorFactory, ListenerSet listeners, Looper looper, @@ -872,7 +876,7 @@ public final class Transformer { private final boolean removeVideo; private final TransformationRequest transformationRequest; private final boolean clippingStartsAtKeyFrame; - private final ImmutableList videoEffects; + private final ImmutableList videoEffects; private final FrameProcessor.Factory frameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; @@ -887,7 +891,7 @@ public final class Transformer { boolean removeVideo, TransformationRequest transformationRequest, boolean clippingStartsAtKeyFrame, - ImmutableList videoEffects, + ImmutableList videoEffects, FrameProcessor.Factory frameProcessorFactory, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java index 032adda9fb..daa79b8665 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java @@ -38,7 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final Context context; private final boolean clippingStartsAtKeyFrame; - private final ImmutableList effects; + private final ImmutableList effects; private final FrameProcessor.Factory frameProcessorFactory; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; @@ -53,7 +53,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformerMediaClock mediaClock, TransformationRequest transformationRequest, boolean clippingStartsAtKeyFrame, - ImmutableList effects, + ImmutableList effects, FrameProcessor.Factory frameProcessorFactory, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index 4887dc7873..d169c9a023 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -57,7 +57,7 @@ import org.checkerframework.dataflow.qual.Pure; Format inputFormat, long streamOffsetUs, TransformationRequest transformationRequest, - ImmutableList effects, + ImmutableList effects, FrameProcessor.Factory frameProcessorFactory, Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, @@ -78,8 +78,8 @@ import org.checkerframework.dataflow.qual.Pure; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; - ImmutableList.Builder effectsListBuilder = - new ImmutableList.Builder().addAll(effects); + ImmutableList.Builder effectsListBuilder = + new ImmutableList.Builder().addAll(effects); if (transformationRequest.scaleX != 1f || transformationRequest.scaleY != 1f || transformationRequest.rotationDegrees != 0f) { @@ -136,7 +136,7 @@ import org.checkerframework.dataflow.qual.Pure; debugViewProvider, // HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This // implies that the OpenGL EXT_YUV_target extension is supported and hence the - // GlEffectsFrameProcessor also supports HDR. + // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. /* useHdr= */ encoderWrapper.isHdrEditingEnabled()); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessingException(