From e6518126ac96824b54cc12ac8c1f14b8cb4a11cb Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Wed, 4 May 2022 13:48:56 +0100 Subject: [PATCH] Add frame processor based on MediaPipe to demo PiperOrigin-RevId: 446432695 --- demos/transformer/BUILD.bazel | 22 +++ demos/transformer/README.md | 56 ++++++ demos/transformer/build.gradle | 18 ++ .../transformer/ConfigurationActivity.java | 9 +- .../demo/transformer/TransformerActivity.java | 36 +++- .../src/main/res/values/strings.xml | 1 + .../src/withMediaPipe/AndroidManifest.xml | 19 ++ .../edge_detector_mediapipe_graph.pbtxt | 13 ++ .../transformer/MediaPipeFrameProcessor.java | 164 ++++++++++++++++++ 9 files changed, 333 insertions(+), 5 deletions(-) create mode 100644 demos/transformer/BUILD.bazel create mode 100644 demos/transformer/src/withMediaPipe/AndroidManifest.xml create mode 100644 demos/transformer/src/withMediaPipe/assets/edge_detector_mediapipe_graph.pbtxt create mode 100644 demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java diff --git a/demos/transformer/BUILD.bazel b/demos/transformer/BUILD.bazel new file mode 100644 index 0000000000..dba4a65315 --- /dev/null +++ b/demos/transformer/BUILD.bazel @@ -0,0 +1,22 @@ +# Build targets for a demo MediaPipe graph. +# See README.md for instructions on using MediaPipe in the demo. + +load("//mediapipe/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_aar") +load( + "//mediapipe/framework/tool:mediapipe_graph.bzl", + "mediapipe_binary_graph", +) + +mediapipe_aar( + name = "edge_detector_mediapipe_aar", + calculators = [ + "//mediapipe/calculators/image:luminance_calculator", + "//mediapipe/calculators/image:sobel_edges_calculator", + ], +) + +mediapipe_binary_graph( + name = "edge_detector_binary_graph", + graph = "edge_detector_mediapipe_graph.pbtxt", + output_name = "edge_detector_mediapipe_graph.binarypb", +) diff --git a/demos/transformer/README.md b/demos/transformer/README.md index fb2657001e..3a53f71dc7 100644 --- a/demos/transformer/README.md +++ b/demos/transformer/README.md @@ -6,4 +6,60 @@ example by removing audio or video. See the [demos README](../README.md) for instructions on how to build and run this demo. +## MediaPipe frame processing demo + +Building the demo app with [MediaPipe][] integration enabled requires some extra +manual steps. + +1. Follow the + [instructions](https://google.github.io/mediapipe/getting_started/install.html) + to install MediaPipe. +1. Copy the Transformer demo's build configuration and MediaPipe graph text + protocol buffer under the MediaPipe source tree. This makes it easy to + [build an AAR][] with bazel by reusing MediaPipe's workspace. + + ```sh + cd "" + MEDIAPIPE_ROOT="$(pwd)" + MEDIAPIPE_TRANSFORMER_ROOT="${MEDIAPIPE_ROOT}/mediapipe/java/com/google/mediapipe/transformer" + cd "" + TRANSFORMER_DEMO_ROOT="$(pwd)" + mkdir -p "${MEDIAPIPE_TRANSFORMER_ROOT}" + mkdir -p "${TRANSFORMER_DEMO_ROOT}/libs" + cp ${TRANSFORMER_DEMO_ROOT}/BUILD.bazel ${MEDIAPIPE_TRANSFORMER_ROOT}/BUILD + cp ${TRANSFORMER_DEMO_ROOT}/src/withMediaPipe/assets/edge_detector_mediapipe_graph.pbtxt \ + ${MEDIAPIPE_TRANSFORMER_ROOT} + ``` + +1. Build the AAR and the binary proto for the demo's MediaPipe graph, then copy + them to Transformer. + + ```sh + cd ${MEDIAPIPE_ROOT} + bazel build -c opt --strip=ALWAYS \ + --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ + --fat_apk_cpu=arm64-v8a,armeabi-v7a \ + --legacy_whole_archive=0 \ + --features=-legacy_whole_archive \ + --copt=-fvisibility=hidden \ + --copt=-ffunction-sections \ + --copt=-fdata-sections \ + --copt=-fstack-protector \ + --copt=-Oz \ + --copt=-fomit-frame-pointer \ + --copt=-DABSL_MIN_LOG_LEVEL=2 \ + --linkopt=-Wl,--gc-sections,--strip-all \ + mediapipe/java/com/google/mediapipe/transformer:edge_detector_mediapipe_aar.aar + cp bazel-bin/mediapipe/java/com/google/mediapipe/transformer/edge_detector_mediapipe_aar.aar \ + ${TRANSFORMER_DEMO_ROOT}/libs + bazel build mediapipe/java/com/google/mediapipe/transformer:edge_detector_binary_graph + cp bazel-bin/mediapipe/java/com/google/mediapipe/transformer/edge_detector_mediapipe_graph.binarypb \ + ${TRANSFORMER_DEMO_ROOT}/src/withMediaPipe/assets + ``` + +1. Select the `withMediaPipe` build variant in Android Studio, then build and + run the demo app and select a MediaPipe-based effect. + [Transformer]: https://exoplayer.dev/transforming-media.html +[MediaPipe]: https://google.github.io/mediapipe/ +[build an AAR]: https://google.github.io/mediapipe/getting_started/android_archive_library.html diff --git a/demos/transformer/build.gradle b/demos/transformer/build.gradle index a40bebff6f..6f6ff50908 100644 --- a/demos/transformer/build.gradle +++ b/demos/transformer/build.gradle @@ -45,6 +45,17 @@ android { // This demo app isn't indexed and doesn't have translations. disable 'GoogleAppIndexingWarning','MissingTranslation' } + + flavorDimensions "mediaPipe" + + productFlavors { + noMediaPipe { + dimension "mediaPipe" + } + withMediaPipe { + dimension "mediaPipe" + } + } } dependencies { @@ -59,4 +70,11 @@ dependencies { implementation project(modulePrefix + 'lib-exoplayer-dash') implementation project(modulePrefix + 'lib-transformer') implementation project(modulePrefix + 'lib-ui') + + // For MediaPipe and its dependencies: + withMediaPipeImplementation fileTree(dir: 'libs', include: ['*.aar']) + withMediaPipeImplementation 'com.google.flogger:flogger:latest.release' + withMediaPipeImplementation 'com.google.flogger:flogger-system-backend:latest.release' + withMediaPipeImplementation 'com.google.code.findbugs:jsr305:latest.release' + withMediaPipeImplementation 'com.google.protobuf:protobuf-javalite:3.19.1' } diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index 901c56299f..c4cec6e845 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -92,9 +92,14 @@ public final class ConfigurationActivity extends AppCompatActivity { "MP4 with HDR (HDR10) H265 video (encoding may fail)", }; private static final String[] DEMO_EFFECTS = { - "Dizzy crop", "Periodic vignette", "3D spin", "Overlay logo & timer", "Zoom in start" + "Dizzy crop", + "Edge detector (Media Pipe)", + "Periodic vignette", + "3D spin", + "Overlay logo & timer", + "Zoom in start", }; - private static final int PERIODIC_VIGNETTE_INDEX = 1; + private static final int PERIODIC_VIGNETTE_INDEX = 2; private static final String SAME_AS_INPUT_OPTION = "same as input"; private static final float HALF_DIAGONAL = 1f / (float) Math.sqrt(2); diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index cd516c94ef..244c86baed 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -31,6 +31,7 @@ import android.view.ViewGroup; import android.widget.TextView; import android.widget.Toast; import androidx.annotation.Nullable; +import androidx.annotation.StringRes; import androidx.appcompat.app.AppCompatActivity; import androidx.media3.common.C; import androidx.media3.common.MediaItem; @@ -41,6 +42,7 @@ import androidx.media3.exoplayer.util.DebugTextViewHelper; import androidx.media3.transformer.DefaultEncoderFactory; import androidx.media3.transformer.EncoderSelector; import androidx.media3.transformer.GlEffect; +import androidx.media3.transformer.GlFrameProcessor; import androidx.media3.transformer.ProgressHolder; import androidx.media3.transformer.TransformationException; import androidx.media3.transformer.TransformationRequest; @@ -54,6 +56,7 @@ import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; import java.io.File; import java.io.IOException; +import java.lang.reflect.Constructor; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -249,6 +252,29 @@ public final class TransformerActivity extends AppCompatActivity { effects.add(MatrixTransformationFactory.createDizzyCropEffect()); } if (selectedEffects[1]) { + try { + Class clazz = + Class.forName("androidx.media3.demo.transformer.MediaPipeFrameProcessor"); + Constructor constructor = + clazz.getConstructor(String.class, String.class, String.class); + effects.add( + () -> { + try { + return (GlFrameProcessor) + constructor.newInstance( + /* graphName= */ "edge_detector_mediapipe_graph.binarypb", + /* inputStreamName= */ "input_video", + /* outputStreamName= */ "output_video"); + } catch (Exception e) { + showToast(R.string.no_media_pipe_error); + throw new RuntimeException("Failed to load MediaPipe processor", e); + } + }); + } catch (Exception e) { + showToast(R.string.no_media_pipe_error); + } + } + if (selectedEffects[2]) { effects.add( () -> new PeriodicVignetteFrameProcessor( @@ -260,13 +286,13 @@ public final class TransformerActivity extends AppCompatActivity { ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS), bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); } - if (selectedEffects[2]) { + if (selectedEffects[3]) { effects.add(MatrixTransformationFactory.createSpin3dEffect()); } - if (selectedEffects[3]) { + if (selectedEffects[4]) { effects.add(BitmapOverlayFrameProcessor::new); } - if (selectedEffects[4]) { + if (selectedEffects[5]) { effects.add(MatrixTransformationFactory.createZoomInTransition()); } transformerBuilder.setVideoFrameEffects(effects.build()); @@ -363,6 +389,10 @@ public final class TransformerActivity extends AppCompatActivity { } } + private void showToast(@StringRes int messageResource) { + Toast.makeText(getApplicationContext(), getString(messageResource), Toast.LENGTH_LONG).show(); + } + private final class DemoDebugViewProvider implements Transformer.DebugViewProvider { @Nullable diff --git a/demos/transformer/src/main/res/values/strings.xml b/demos/transformer/src/main/res/values/strings.xml index 27ebb2b3ab..98dc42ecb8 100644 --- a/demos/transformer/src/main/res/values/strings.xml +++ b/demos/transformer/src/main/res/values/strings.xml @@ -31,6 +31,7 @@ [Experimental] HDR editing Add demo effects Periodic vignette options + Failed to load MediaPipe processor. Check the README for instructions. Transform Debug preview: No debug preview available. diff --git a/demos/transformer/src/withMediaPipe/AndroidManifest.xml b/demos/transformer/src/withMediaPipe/AndroidManifest.xml new file mode 100644 index 0000000000..683edd4489 --- /dev/null +++ b/demos/transformer/src/withMediaPipe/AndroidManifest.xml @@ -0,0 +1,19 @@ + + + + + diff --git a/demos/transformer/src/withMediaPipe/assets/edge_detector_mediapipe_graph.pbtxt b/demos/transformer/src/withMediaPipe/assets/edge_detector_mediapipe_graph.pbtxt new file mode 100644 index 0000000000..b32a40d940 --- /dev/null +++ b/demos/transformer/src/withMediaPipe/assets/edge_detector_mediapipe_graph.pbtxt @@ -0,0 +1,13 @@ +# Demo MediaPipe graph that shows edges using a SobelEdgesCalculator. +input_stream: "input_video" +output_stream: "output_video" +node: { + calculator: "LuminanceCalculator" + input_stream: "input_video" + output_stream: "luma_video" +} +node: { + calculator: "SobelEdgesCalculator" + input_stream: "luma_video" + output_stream: "output_video" +} diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java new file mode 100644 index 0000000000..9d9b0b9dd3 --- /dev/null +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java @@ -0,0 +1,164 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.demo.transformer; + +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.opengl.EGL14; +import android.opengl.GLES20; +import android.util.Size; +import androidx.media3.common.util.ConditionVariable; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.LibraryLoader; +import androidx.media3.transformer.GlFrameProcessor; +import com.google.mediapipe.components.FrameProcessor; +import com.google.mediapipe.framework.AppTextureFrame; +import com.google.mediapipe.framework.TextureFrame; +import com.google.mediapipe.glutil.EglManager; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that + * can immediately produce one output frame per input frame. + */ +/* package */ final class MediaPipeFrameProcessor implements GlFrameProcessor { + + private static final LibraryLoader LOADER = + new LibraryLoader("mediapipe_jni") { + @Override + protected void loadLibrary(String name) { + System.loadLibrary(name); + } + }; + + private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl"; + private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl"; + + private final String graphName; + private final String inputStreamName; + private final String outputStreamName; + private final ConditionVariable frameProcessorConditionVariable; + + private @MonotonicNonNull FrameProcessor frameProcessor; + private int inputWidth; + private int inputHeight; + private int inputTexId; + private @MonotonicNonNull GlProgram glProgram; + private @MonotonicNonNull TextureFrame outputFrame; + private @MonotonicNonNull RuntimeException frameProcessorPendingError; + + /** + * Creates a new frame processor that wraps a MediaPipe graph. + * + * @param graphName Name of a MediaPipe graph asset to load. + * @param inputStreamName Name of the input video stream in the graph. + * @param outputStreamName Name of the input video stream in the graph. + */ + public MediaPipeFrameProcessor( + String graphName, String inputStreamName, String outputStreamName) { + checkState(LOADER.isAvailable()); + this.graphName = graphName; + this.inputStreamName = inputStreamName; + this.outputStreamName = outputStreamName; + frameProcessorConditionVariable = new ConditionVariable(); + } + + @Override + public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) + throws IOException { + this.inputTexId = inputTexId; + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME); + + EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); + frameProcessor = + new FrameProcessor( + context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName); + + // Unblock drawFrame when there is an output frame or an error. + frameProcessor.setConsumer( + frame -> { + outputFrame = frame; + frameProcessorConditionVariable.open(); + }); + frameProcessor.setAsynchronousErrorListener( + error -> { + frameProcessorPendingError = error; + frameProcessorConditionVariable.open(); + }); + } + + @Override + public Size getOutputSize() { + return new Size(inputWidth, inputHeight); + } + + @Override + public void drawFrame(long presentationTimeUs) { + frameProcessorConditionVariable.close(); + + // Pass the input frame to MediaPipe. + AppTextureFrame appTextureFrame = new AppTextureFrame(inputTexId, inputWidth, inputHeight); + appTextureFrame.setTimestamp(presentationTimeUs); + checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame); + + // Wait for output to be passed to the consumer. + try { + frameProcessorConditionVariable.block(); + } catch (InterruptedException e) { + // Propagate the interrupted flag so the next blocking operation will throw. + // TODO(b/230469581): The next processor that runs will not have valid input due to returning + // early here. This could be fixed by checking for interruption in the outer loop that runs + // through the frame processors. + Thread.currentThread().interrupt(); + return; + } + + if (frameProcessorPendingError != null) { + throw new IllegalStateException(frameProcessorPendingError); + } + + // Copy from MediaPipe's output texture to the current output. + try { + checkStateNotNull(glProgram).use(); + glProgram.setSamplerTexIdUniform( + "uTexSampler", checkStateNotNull(outputFrame).getTextureName(), /* texUnitIndex= */ 0); + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.setBufferAttribute( + "aTexSamplingCoord", + GlUtil.getTextureCoordinateBounds(), + GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.bindAttributesAndUniforms(); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); + } finally { + checkStateNotNull(outputFrame).release(); + } + } + + @Override + public void release() { + checkStateNotNull(frameProcessor).close(); + } +}