loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-16 20:19:57 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add Effect marker interface.

Browse Source 
This allows non-GL effects to be passed to custom FrameProcessor
implementations.

PiperOrigin-RevId: 463696384
(cherry picked from commit 580e44fc47ec5b62da5a133d12048a20d716e94f)
This commit is contained in:
hschlueter 2022-07-27 22:40:20 +00:00 committed by microkatz
parent 49a6511696
commit 6e390771a0
9 changed files with 98 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -44,6 +44,7 @@ import androidx.media3.exoplayer.util.DebugTextViewHelper;
import androidx.media3.transformer.Contrast;
import androidx.media3.transformer.DebugViewProvider;
import androidx.media3.transformer.DefaultEncoderFactory;
import androidx.media3.transformer.Effect;
import androidx.media3.transformer.GlEffect;
import androidx.media3.transformer.GlTextureProcessor;
import androidx.media3.transformer.ProgressHolder;
@ -265,7 +266,7 @@ public final class TransformerActivity extends AppCompatActivity {
.setEnableFallback(bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK))
.build());
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
ImmutableList.Builder<Effect> effects = new ImmutableList.Builder<>();
@Nullable
boolean[] selectedEffects =
bundle.getBooleanArray(ConfigurationActivity.DEMO_EFFECTS_SELECTIONS);
@ -280,43 +281,45 @@ public final class TransformerActivity extends AppCompatActivity {
clazz.getConstructor(
Context.class, Boolean.class, String.class, String.class, String.class);
effects.add(
(Context context, boolean useHdr) -> {
try {
return (GlTextureProcessor)
constructor.newInstance(
context,
useHdr,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb",
/* inputStreamName= */ "input_video",
/* outputStreamName= */ "output_video");
} catch (Exception e) {
runOnUiThread(() -> showToast(R.string.no_media_pipe_error));
throw new RuntimeException("Failed to load MediaPipe processor", e);
}
});
(GlEffect)
(Context context, boolean useHdr) -> {
try {
return (GlTextureProcessor)
constructor.newInstance(
context,
useHdr,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb",
/* inputStreamName= */ "input_video",
/* outputStreamName= */ "output_video");
} catch (Exception e) {
runOnUiThread(() -> showToast(R.string.no_media_pipe_error));
throw new RuntimeException("Failed to load MediaPipe processor", e);
}
});
} catch (Exception e) {
showToast(R.string.no_media_pipe_error);
}
}
if (selectedEffects[2]) {
effects.add(
(Context context, boolean useHdr) ->
new PeriodicVignetteProcessor(
context,
useHdr,
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS),
/* maxInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS)));
(GlEffect)
(Context context, boolean useHdr) ->
new PeriodicVignetteProcessor(
context,
useHdr,
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS),
/* maxInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS)));
}
if (selectedEffects[3]) {
effects.add(MatrixTransformationFactory.createSpin3dEffect());
}
if (selectedEffects[4]) {
effects.add(BitmapOverlayProcessor::new);
effects.add((GlEffect) BitmapOverlayProcessor::new);
}
if (selectedEffects[5]) {
effects.add(MatrixTransformationFactory.createZoomInTransition());

4
libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
View File

@ -295,7 +295,7 @@ public final class GlEffectsFrameProcessorPixelTest {
"processData_withManyComposedMatrixTransformations_producesSameOutputAsCombinedTransformation";
Crop centerCrop =
new Crop(/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f);
ImmutableList.Builder<GlEffect> full10StepRotationAndCenterCrop = new ImmutableList.Builder<>();
ImmutableList.Builder<Effect> full10StepRotationAndCenterCrop = new ImmutableList.Builder<>();
for (int i = 0; i < 10; i++) {
full10StepRotationAndCenterCrop.add(new Rotation(/* degrees= */ 36));
}
@ -336,7 +336,7 @@ public final class GlEffectsFrameProcessorPixelTest {
setUpAndPrepareFirstFrame(pixelWidthHeightRatio, asList(effects));
}
private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, List<GlEffect> effects)
private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, List<Effect> effects)
throws Exception {
// Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor();

23
libraries/transformer/src/main/java/androidx/media3/transformer/Effect.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import androidx.media3.common.util.UnstableApi;
/** Marker interface for a video frame effect. */
@UnstableApi
public interface Effect {}

6
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java
View File

@ -24,7 +24,7 @@ import java.util.List;
/**
* Interface for a frame processor that applies changes to individual video frames.
*
* <p>The changes are specified by {@link GlEffect} instances passed to the {@link Factory}.
* <p>The changes are specified by {@link Effect} instances passed to the {@link Factory}.
*
* <p>The frame processor manages its input {@link Surface} which can be accessed via {@link
* #getInputSurface()}. The output {@link Surface} must be set by the caller using {@link
@ -44,7 +44,7 @@ public interface FrameProcessor {
*
* @param context A {@link Context}.
* @param listener A {@link Listener}.
* @param effects The {@link GlEffect} instances to apply to each frame.
* @param effects The {@link Effect} instances to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}.
* @param useHdr Whether to process the input as an HDR signal.
* @return A new instance.
@ -54,7 +54,7 @@ public interface FrameProcessor {
FrameProcessor create(
Context context,
FrameProcessor.Listener listener,
List<GlEffect> effects,
List<Effect> effects,
DebugViewProvider debugViewProvider,
boolean useHdr)
throws FrameProcessingException;

2
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java
View File

@ -26,7 +26,7 @@ import androidx.media3.common.util.UnstableApi;
* the effect.
*/
@UnstableApi
public interface GlEffect {
public interface GlEffect extends Effect {
/**
* Returns a {@link SingleFrameGlTextureProcessor} that applies the effect.

23
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static com.google.common.collect.Iterables.getLast;
@ -53,13 +54,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
/**
* {@inheritDoc}
*
* <p>All {@link Effect} instances must be {@link GlEffect} instances.
*
* <p>Using HDR requires the {@code EXT_YUV_target} OpenGL extension.
*/
@Override
public GlEffectsFrameProcessor create(
Context context,
FrameProcessor.Listener listener,
List<GlEffect> effects,
List<Effect> effects,
DebugViewProvider debugViewProvider,
boolean useHdr)
throws FrameProcessingException {
@ -93,6 +96,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
* GlTextureProcessor} instances corresponding to the {@link GlEffect} instances, and returns a
* new {@code GlEffectsFrameProcessor}.
*
* <p>All {@link Effect} instances must be {@link GlEffect} instances.
*
* <p>This method must be executed using the {@code singleThreadExecutorService}, as later OpenGL
* commands will be called on that thread.
*/
@ -100,7 +105,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
private static GlEffectsFrameProcessor createOpenGlObjectsAndFrameProcessor(
Context context,
FrameProcessor.Listener listener,
List<GlEffect> effects,
List<Effect> effects,
DebugViewProvider debugViewProvider,
boolean useHdr,
ExecutorService singleThreadExecutorService)
@ -142,13 +147,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
* MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate
* {@link GlTextureProcessor} instances.
*
* <p>All {@link Effect} instances must be {@link GlEffect} instances.
*
* @return A non-empty list of {@link GlTextureProcessor} instances to apply in the given order.
* The first is an {@link ExternalTextureProcessor} and the last is a {@link
* FinalMatrixTransformationProcessorWrapper}.
*/
private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffects(
Context context,
List<GlEffect> effects,
List<Effect> effects,
EGLDisplay eglDisplay,
EGLContext eglContext,
FrameProcessor.Listener listener,
@ -161,9 +168,11 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
new ImmutableList.Builder<>();
boolean sampleFromExternalTexture = true;
for (int i = 0; i < effects.size(); i++) {
GlEffect effect = effects.get(i);
if (effect instanceof GlMatrixTransformation) {
matrixTransformationListBuilder.add((GlMatrixTransformation) effect);
Effect effect = effects.get(i);
checkArgument(effect instanceof GlEffect, "GlEffectsFrameProcessor only supports GlEffects");
GlEffect glEffect = (GlEffect) effect;
if (glEffect instanceof GlMatrixTransformation) {
matrixTransformationListBuilder.add((GlMatrixTransformation) glEffect);
continue;
}
ImmutableList<GlMatrixTransformation> matrixTransformations =
@ -179,7 +188,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
}
textureProcessorListBuilder.add(effect.toGlTextureProcessor(context, useHdr));
textureProcessorListBuilder.add(glEffect.toGlTextureProcessor(context, useHdr));
}
textureProcessorListBuilder.add(
new FinalMatrixTransformationProcessorWrapper(

30
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -103,7 +103,7 @@ public final class Transformer {
private boolean removeVideo;
private String containerMimeType;
private TransformationRequest transformationRequest;
private ImmutableList<GlEffect> videoEffects;
private ImmutableList<Effect> videoEffects;
private FrameProcessor.Factory frameProcessorFactory;
private ListenerSet<Transformer.Listener> listeners;
private DebugViewProvider debugViewProvider;
@ -168,32 +168,36 @@ public final class Transformer {
}
/**
* Sets the {@linkplain GlEffect effects} to apply to each video frame.
* Sets the {@link Effect} instances to apply to each video frame.
*
* <p>The {@linkplain GlEffect effects} are applied before any {@linkplain
* <p>The {@link Effect} instances are applied before any {@linkplain
* TransformationRequest.Builder#setScale(float, float) scale}, {@linkplain
* TransformationRequest.Builder#setRotationDegrees(float) rotation}, or {@linkplain
* TransformationRequest.Builder#setResolution(int) resolution} changes specified in the {@link
* #setTransformationRequest(TransformationRequest) TransformationRequest} but after {@linkplain
* TransformationRequest.Builder#setFlattenForSlowMotion(boolean) slow-motion flattening}.
*
* @param effects The {@linkplain GlEffect effects} to apply to each video frame.
* <p>The default {@link FrameProcessor} only supports {@link GlEffect} instances. To use other
* effects, call {@link #setFrameProcessorFactory(FrameProcessor.Factory)} with a custom {@link
* FrameProcessor.Factory}.
*
* @param effects The {@link Effect} instances to apply to each video frame.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setVideoEffects(List<GlEffect> effects) {
public Builder setVideoEffects(List<Effect> effects) {
this.videoEffects = ImmutableList.copyOf(effects);
return this;
}
/**
* Sets the {@link FrameProcessor.Factory} for the {@link FrameProcessor} to use when applying
* {@linkplain GlEffect effects} to the video frames.
* {@linkplain Effect effects} to the video frames.
*
* <p>This factory will be used to create the {@link FrameProcessor} used for applying the
* {@link GlEffect} instances passed to {@link #setVideoEffects(List<GlEffect>)} and any
* additional {@link GlMatrixTransformation} instances derived from the {@link
* TransformationRequest} set using {@link #setTransformationRequest(TransformationRequest)}.
* {@link Effect} instances passed to {@link #setVideoEffects(List<Effect>)} and any additional
* {@link GlMatrixTransformation} instances derived from the {@link TransformationRequest} set
* using {@link #setTransformationRequest(TransformationRequest)}.
*
* <p>The default is {@link GlEffectsFrameProcessor.Factory}.
*
@ -571,7 +575,7 @@ public final class Transformer {
private final boolean removeVideo;
private final String containerMimeType;
private final TransformationRequest transformationRequest;
private final ImmutableList<GlEffect> videoEffects;
private final ImmutableList<Effect> videoEffects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Looper looper;
private final Clock clock;
@ -593,7 +597,7 @@ public final class Transformer {
boolean removeVideo,
String containerMimeType,
TransformationRequest transformationRequest,
ImmutableList<GlEffect> videoEffects,
ImmutableList<Effect> videoEffects,
FrameProcessor.Factory frameProcessorFactory,
ListenerSet<Transformer.Listener> listeners,
Looper looper,
@ -874,7 +878,7 @@ public final class Transformer {
private final boolean removeVideo;
private final TransformationRequest transformationRequest;
private final boolean clippingStartsAtKeyFrame;
private final ImmutableList<GlEffect> videoEffects;
private final ImmutableList<Effect> videoEffects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
@ -889,7 +893,7 @@ public final class Transformer {
boolean removeVideo,
TransformationRequest transformationRequest,
boolean clippingStartsAtKeyFrame,
ImmutableList<GlEffect> videoEffects,
ImmutableList<Effect> videoEffects,
FrameProcessor.Factory frameProcessorFactory,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,

4
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
View File

@ -38,7 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final Context context;
private final boolean clippingStartsAtKeyFrame;
private final ImmutableList<GlEffect> effects;
private final ImmutableList<Effect> effects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
@ -53,7 +53,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
TransformerMediaClock mediaClock,
TransformationRequest transformationRequest,
boolean clippingStartsAtKeyFrame,
ImmutableList<GlEffect> effects,
ImmutableList<Effect> effects,
FrameProcessor.Factory frameProcessorFactory,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,

8
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -57,7 +57,7 @@ import org.checkerframework.dataflow.qual.Pure;
Format inputFormat,
long streamOffsetUs,
TransformationRequest transformationRequest,
ImmutableList<GlEffect> effects,
ImmutableList<Effect> effects,
FrameProcessor.Factory frameProcessorFactory,
Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
@ -78,8 +78,8 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
ImmutableList.Builder<GlEffect> effectsListBuilder =
new ImmutableList.Builder<GlEffect>().addAll(effects);
ImmutableList.Builder<Effect> effectsListBuilder =
new ImmutableList.Builder<Effect>().addAll(effects);
if (transformationRequest.scaleX != 1f
|| transformationRequest.scaleY != 1f
|| transformationRequest.rotationDegrees != 0f) {
@ -136,7 +136,7 @@ import org.checkerframework.dataflow.qual.Pure;
debugViewProvider,
// HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This
// implies that the OpenGL EXT_YUV_target extension is supported and hence the
// GlEffectsFrameProcessor also supports HDR.
// default FrameProcessor, GlEffectsFrameProcessor, also supports HDR.
/* useHdr= */ encoderWrapper.isHdrEditingEnabled());
} catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessingException(