Use GlTextureProcessor to avoid redundant copy in MediaPipeProcessor.

After this change GlEffects can use any GlTextureProcessor not just
SingleFrameGlTextureProcessor.
MediaPipeProcessor now implements GlTextureProcessor directly which
allows it to reuse MediaPipe's output texture for its output texture
and avoids an extra copy shader step.

PiperOrigin-RevId: 456530718
(cherry picked from commit 69ab79418ef21033ca25cebdf5a5e80752818ab5)
This commit is contained in:
hschlueter 2022-06-22 17:16:54 +01:00 committed by microkatz
parent 0007a47365
commit 26ee3d32b2
6 changed files with 97 additions and 91 deletions

View File

@ -44,8 +44,8 @@ import androidx.media3.exoplayer.util.DebugTextViewHelper;
import androidx.media3.transformer.DefaultEncoderFactory; import androidx.media3.transformer.DefaultEncoderFactory;
import androidx.media3.transformer.EncoderSelector; import androidx.media3.transformer.EncoderSelector;
import androidx.media3.transformer.GlEffect; import androidx.media3.transformer.GlEffect;
import androidx.media3.transformer.GlTextureProcessor;
import androidx.media3.transformer.ProgressHolder; import androidx.media3.transformer.ProgressHolder;
import androidx.media3.transformer.SingleFrameGlTextureProcessor;
import androidx.media3.transformer.TransformationException; import androidx.media3.transformer.TransformationException;
import androidx.media3.transformer.TransformationRequest; import androidx.media3.transformer.TransformationRequest;
import androidx.media3.transformer.TransformationResult; import androidx.media3.transformer.TransformationResult;
@ -281,7 +281,7 @@ public final class TransformerActivity extends AppCompatActivity {
effects.add( effects.add(
(Context context) -> { (Context context) -> {
try { try {
return (SingleFrameGlTextureProcessor) return (GlTextureProcessor)
constructor.newInstance( constructor.newInstance(
context, context,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb", /* graphName= */ "edge_detector_mediapipe_graph.binarypb",

View File

@ -20,27 +20,24 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context; import android.content.Context;
import android.opengl.EGL14; import android.opengl.EGL14;
import android.opengl.GLES20; import android.os.Build;
import android.util.Size; import androidx.annotation.ChecksSdkIntAtLeast;
import androidx.media3.common.util.ConditionVariable; import androidx.annotation.Nullable;
import androidx.media3.common.util.GlProgram; import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.LibraryLoader; import androidx.media3.common.util.LibraryLoader;
import androidx.media3.common.util.Util;
import androidx.media3.transformer.FrameProcessingException; import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.SingleFrameGlTextureProcessor; import androidx.media3.transformer.GlTextureProcessor;
import androidx.media3.transformer.TextureInfo;
import com.google.mediapipe.components.FrameProcessor; import com.google.mediapipe.components.FrameProcessor;
import com.google.mediapipe.framework.AndroidAssetUtil;
import com.google.mediapipe.framework.AppTextureFrame; import com.google.mediapipe.framework.AppTextureFrame;
import com.google.mediapipe.framework.TextureFrame; import com.google.mediapipe.framework.TextureFrame;
import com.google.mediapipe.glutil.EglManager; import com.google.mediapipe.glutil.EglManager;
import java.io.IOException; import java.util.concurrent.ConcurrentHashMap;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /** Runs a MediaPipe graph on input frames. */
* Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that /* package */ final class MediaPipeProcessor implements GlTextureProcessor {
* can immediately produce one output frame per input frame.
*/
/* package */ final class MediaPipeProcessor extends SingleFrameGlTextureProcessor {
private static final LibraryLoader LOADER = private static final LibraryLoader LOADER =
new LibraryLoader("mediapipe_jni") { new LibraryLoader("mediapipe_jni") {
@ -60,17 +57,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
} }
private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl";
private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl";
private final ConditionVariable frameProcessorConditionVariable;
private final FrameProcessor frameProcessor; private final FrameProcessor frameProcessor;
private final GlProgram glProgram; private volatile GlTextureProcessor.@MonotonicNonNull Listener listener;
private volatile boolean acceptedFrame;
private int inputWidth; // Only available from API 23 and above.
private int inputHeight; @Nullable private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
private @MonotonicNonNull TextureFrame outputFrame; // Used instead for API 21 and 22.
private @MonotonicNonNull RuntimeException frameProcessorPendingError; @Nullable private volatile TextureInfo outputTexture;
@Nullable private volatile TextureFrame outputFrame;
/** /**
* Creates a new texture processor that wraps a MediaPipe graph. * Creates a new texture processor that wraps a MediaPipe graph.
@ -79,92 +73,103 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param graphName Name of a MediaPipe graph asset to load. * @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph. * @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph.
* @throws FrameProcessingException If a problem occurs while reading shader files or initializing
* MediaPipe resources.
*/ */
@SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23.
public MediaPipeProcessor( public MediaPipeProcessor(
Context context, String graphName, String inputStreamName, String outputStreamName) Context context, String graphName, String inputStreamName, String outputStreamName) {
throws FrameProcessingException {
checkState(LOADER.isAvailable()); checkState(LOADER.isAvailable());
frameProcessorConditionVariable = new ConditionVariable();
AndroidAssetUtil.initializeNativeAssetManager(context);
EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor = frameProcessor =
new FrameProcessor( new FrameProcessor(
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName); context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
// Unblock drawFrame when there is an output frame or an error. outputFrames = areMultipleOutputFramesSupported() ? new ConcurrentHashMap<>() : null;
frameProcessor.setConsumer( frameProcessor.setConsumer(
frame -> { frame -> {
outputFrame = frame; TextureInfo texture =
frameProcessorConditionVariable.open(); new TextureInfo(
frame.getTextureName(),
/* fboId= */ C.INDEX_UNSET,
frame.getWidth(),
frame.getHeight());
if (areMultipleOutputFramesSupported()) {
checkStateNotNull(outputFrames).put(texture, frame);
} else {
outputFrame = frame;
outputTexture = texture;
}
if (listener != null) {
listener.onOutputFrameAvailable(texture, frame.getTimestamp());
}
}); });
frameProcessor.setAsynchronousErrorListener( frameProcessor.setAsynchronousErrorListener(
error -> { error -> {
frameProcessorPendingError = error; if (listener != null) {
frameProcessorConditionVariable.open(); listener.onFrameProcessingError(new FrameProcessingException(error));
}
}); });
try { frameProcessor.setOnWillAddFrameListener((long timestamp) -> acceptedFrame = true);
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME); }
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e); @Override
public void setListener(GlTextureProcessor.Listener listener) {
this.listener = listener;
}
@Override
public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
if (!areMultipleOutputFramesSupported() && outputTexture != null) {
return false;
} }
}
@Override acceptedFrame = false;
public Size configure(int inputWidth, int inputHeight) { AppTextureFrame appTextureFrame =
this.inputWidth = inputWidth; new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height);
this.inputHeight = inputHeight;
return new Size(inputWidth, inputHeight);
}
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
frameProcessorConditionVariable.close();
// Pass the input frame to MediaPipe.
AppTextureFrame appTextureFrame = new AppTextureFrame(inputTexId, inputWidth, inputHeight);
appTextureFrame.setTimestamp(presentationTimeUs); appTextureFrame.setTimestamp(presentationTimeUs);
checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame); checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame);
// Wait for output to be passed to the consumer.
try { try {
frameProcessorConditionVariable.block(); appTextureFrame.waitUntilReleasedWithGpuSync();
} catch (InterruptedException e) { } catch (InterruptedException e) {
// Propagate the interrupted flag so the next blocking operation will throw.
// TODO(b/230469581): The next processor that runs will not have valid input due to returning
// early here. This could be fixed by checking for interruption in the outer loop that runs
// through the texture processors.
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
return; if (listener != null) {
listener.onFrameProcessingError(new FrameProcessingException(e));
}
} }
if (listener != null) {
if (frameProcessorPendingError != null) { listener.onInputFrameProcessed(inputTexture);
throw new FrameProcessingException(frameProcessorPendingError);
} }
return acceptedFrame;
}
// Copy from MediaPipe's output texture to the current output. @Override
try { public void releaseOutputFrame(TextureInfo outputTexture) {
checkStateNotNull(glProgram).use(); if (areMultipleOutputFramesSupported()) {
glProgram.setSamplerTexIdUniform( checkStateNotNull(checkStateNotNull(outputFrames).get(outputTexture)).release();
"uTexSampler", checkStateNotNull(outputFrame).getTextureName(), /* texUnitIndex= */ 0); } else {
glProgram.setBufferAttribute( checkState(Util.areEqual(outputTexture, this.outputTexture));
"aFramePosition", this.outputTexture = null;
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
} finally {
checkStateNotNull(outputFrame).release(); checkStateNotNull(outputFrame).release();
outputFrame = null;
} }
} }
@Override @Override
public void release() throws FrameProcessingException { public void release() {
super.release();
checkStateNotNull(frameProcessor).close(); checkStateNotNull(frameProcessor).close();
} }
@Override
public final void signalEndOfInputStream() {
frameProcessor.waitUntilIdle();
if (listener != null) {
listener.onOutputStreamEnded();
}
}
@ChecksSdkIntAtLeast(api = Build.VERSION_CODES.M)
private static boolean areMultipleOutputFramesSupported() {
// Android devices running Lollipop (API 21/22) have a bug in ConcurrentHashMap that can result
// in lost updates, so we only allow one output frame to be pending at a time to avoid using
// ConcurrentHashMap.
return Util.SDK_INT >= 23;
}
} }

View File

@ -491,7 +491,7 @@ public final class FrameProcessorChainPixelTest {
} }
@Override @Override
public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) public GlTextureProcessor toGlTextureProcessor(Context context)
throws FrameProcessingException { throws FrameProcessingException {
return effect.toGlTextureProcessor(context); return effect.toGlTextureProcessor(context);
} }

View File

@ -23,6 +23,7 @@ import android.opengl.EGLContext;
import android.opengl.EGLDisplay; import android.opengl.EGLDisplay;
import android.opengl.EGLExt; import android.opengl.EGLExt;
import android.opengl.EGLSurface; import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.util.Size; import android.util.Size;
import android.view.Surface; import android.view.Surface;
import android.view.SurfaceHolder; import android.view.SurfaceHolder;
@ -314,6 +315,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height); GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
renderingTask.run(); renderingTask.run();
EGL14.eglSwapBuffers(eglDisplay, eglSurface); EGL14.eglSwapBuffers(eglDisplay, eglSurface);
// Prevents white flashing on the debug SurfaceView when frames are rendered too fast.
GLES20.glFinish();
} }
@Override @Override

View File

@ -19,17 +19,15 @@ import android.content.Context;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
/** /**
* Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation. * Interface for a video frame effect with a {@link GlTextureProcessor} implementation.
* *
* <p>Implementations contain information specifying the effect and can be {@linkplain * <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlTextureProcessor(Context) converted} to a {@link SingleFrameGlTextureProcessor} which * #toGlTextureProcessor(Context) converted} to a {@link GlTextureProcessor} which applies the
* applies the effect. * effect.
*/ */
@UnstableApi @UnstableApi
public interface GlEffect { public interface GlEffect {
/** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */ /** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */
// TODO(b/227625423): use GlTextureProcessor here once this interface exists. GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException;
SingleFrameGlTextureProcessor toGlTextureProcessor(Context context)
throws FrameProcessingException;
} }

View File

@ -19,7 +19,7 @@ import androidx.media3.common.util.UnstableApi;
/** Contains information describing an OpenGL texture. */ /** Contains information describing an OpenGL texture. */
@UnstableApi @UnstableApi
/* package */ final class TextureInfo { public final class TextureInfo {
/** The OpenGL texture identifier. */ /** The OpenGL texture identifier. */
public final int texId; public final int texId;
/** Identifier of a framebuffer object associated with the texture. */ /** Identifier of a framebuffer object associated with the texture. */