diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 2c0b38fbf9..0c41d8c1f4 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -44,8 +44,8 @@ import androidx.media3.exoplayer.util.DebugTextViewHelper; import androidx.media3.transformer.DefaultEncoderFactory; import androidx.media3.transformer.EncoderSelector; import androidx.media3.transformer.GlEffect; +import androidx.media3.transformer.GlTextureProcessor; import androidx.media3.transformer.ProgressHolder; -import androidx.media3.transformer.SingleFrameGlTextureProcessor; import androidx.media3.transformer.TransformationException; import androidx.media3.transformer.TransformationRequest; import androidx.media3.transformer.TransformationResult; @@ -281,7 +281,7 @@ public final class TransformerActivity extends AppCompatActivity { effects.add( (Context context) -> { try { - return (SingleFrameGlTextureProcessor) + return (GlTextureProcessor) constructor.newInstance( context, /* graphName= */ "edge_detector_mediapipe_graph.binarypb", diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java index 2ceb8cf12f..3fcd65e9fa 100644 --- a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java @@ -20,27 +20,24 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.EGL14; -import android.opengl.GLES20; -import android.util.Size; -import androidx.media3.common.util.ConditionVariable; -import androidx.media3.common.util.GlProgram; -import androidx.media3.common.util.GlUtil; +import android.os.Build; +import androidx.annotation.ChecksSdkIntAtLeast; +import androidx.annotation.Nullable; +import androidx.media3.common.C; import androidx.media3.common.util.LibraryLoader; +import androidx.media3.common.util.Util; import androidx.media3.transformer.FrameProcessingException; -import androidx.media3.transformer.SingleFrameGlTextureProcessor; +import androidx.media3.transformer.GlTextureProcessor; +import androidx.media3.transformer.TextureInfo; import com.google.mediapipe.components.FrameProcessor; -import com.google.mediapipe.framework.AndroidAssetUtil; import com.google.mediapipe.framework.AppTextureFrame; import com.google.mediapipe.framework.TextureFrame; import com.google.mediapipe.glutil.EglManager; -import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** - * Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that - * can immediately produce one output frame per input frame. - */ -/* package */ final class MediaPipeProcessor extends SingleFrameGlTextureProcessor { +/** Runs a MediaPipe graph on input frames. */ +/* package */ final class MediaPipeProcessor implements GlTextureProcessor { private static final LibraryLoader LOADER = new LibraryLoader("mediapipe_jni") { @@ -60,17 +57,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl"; - private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl"; - - private final ConditionVariable frameProcessorConditionVariable; private final FrameProcessor frameProcessor; - private final GlProgram glProgram; - - private int inputWidth; - private int inputHeight; - private @MonotonicNonNull TextureFrame outputFrame; - private @MonotonicNonNull RuntimeException frameProcessorPendingError; + private volatile GlTextureProcessor.@MonotonicNonNull Listener listener; + private volatile boolean acceptedFrame; + // Only available from API 23 and above. + @Nullable private final ConcurrentHashMap outputFrames; + // Used instead for API 21 and 22. + @Nullable private volatile TextureInfo outputTexture; + @Nullable private volatile TextureFrame outputFrame; /** * Creates a new texture processor that wraps a MediaPipe graph. @@ -79,92 +73,103 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param graphName Name of a MediaPipe graph asset to load. * @param inputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph. - * @throws FrameProcessingException If a problem occurs while reading shader files or initializing - * MediaPipe resources. */ + @SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23. public MediaPipeProcessor( - Context context, String graphName, String inputStreamName, String outputStreamName) - throws FrameProcessingException { + Context context, String graphName, String inputStreamName, String outputStreamName) { checkState(LOADER.isAvailable()); - - frameProcessorConditionVariable = new ConditionVariable(); - AndroidAssetUtil.initializeNativeAssetManager(context); EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); frameProcessor = new FrameProcessor( context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName); - // Unblock drawFrame when there is an output frame or an error. + outputFrames = areMultipleOutputFramesSupported() ? new ConcurrentHashMap<>() : null; frameProcessor.setConsumer( frame -> { - outputFrame = frame; - frameProcessorConditionVariable.open(); + TextureInfo texture = + new TextureInfo( + frame.getTextureName(), + /* fboId= */ C.INDEX_UNSET, + frame.getWidth(), + frame.getHeight()); + if (areMultipleOutputFramesSupported()) { + checkStateNotNull(outputFrames).put(texture, frame); + } else { + outputFrame = frame; + outputTexture = texture; + } + if (listener != null) { + listener.onOutputFrameAvailable(texture, frame.getTimestamp()); + } }); frameProcessor.setAsynchronousErrorListener( error -> { - frameProcessorPendingError = error; - frameProcessorConditionVariable.open(); + if (listener != null) { + listener.onFrameProcessingError(new FrameProcessingException(error)); + } }); - try { - glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME); - } catch (IOException | GlUtil.GlException e) { - throw new FrameProcessingException(e); + frameProcessor.setOnWillAddFrameListener((long timestamp) -> acceptedFrame = true); + } + + @Override + public void setListener(GlTextureProcessor.Listener listener) { + this.listener = listener; + } + + @Override + public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + if (!areMultipleOutputFramesSupported() && outputTexture != null) { + return false; } - } - @Override - public Size configure(int inputWidth, int inputHeight) { - this.inputWidth = inputWidth; - this.inputHeight = inputHeight; - return new Size(inputWidth, inputHeight); - } - - @Override - public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { - frameProcessorConditionVariable.close(); - - // Pass the input frame to MediaPipe. - AppTextureFrame appTextureFrame = new AppTextureFrame(inputTexId, inputWidth, inputHeight); + acceptedFrame = false; + AppTextureFrame appTextureFrame = + new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height); appTextureFrame.setTimestamp(presentationTimeUs); checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame); - - // Wait for output to be passed to the consumer. try { - frameProcessorConditionVariable.block(); + appTextureFrame.waitUntilReleasedWithGpuSync(); } catch (InterruptedException e) { - // Propagate the interrupted flag so the next blocking operation will throw. - // TODO(b/230469581): The next processor that runs will not have valid input due to returning - // early here. This could be fixed by checking for interruption in the outer loop that runs - // through the texture processors. Thread.currentThread().interrupt(); - return; + if (listener != null) { + listener.onFrameProcessingError(new FrameProcessingException(e)); + } } - - if (frameProcessorPendingError != null) { - throw new FrameProcessingException(frameProcessorPendingError); + if (listener != null) { + listener.onInputFrameProcessed(inputTexture); } + return acceptedFrame; + } - // Copy from MediaPipe's output texture to the current output. - try { - checkStateNotNull(glProgram).use(); - glProgram.setSamplerTexIdUniform( - "uTexSampler", checkStateNotNull(outputFrame).getTextureName(), /* texUnitIndex= */ 0); - glProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - glProgram.bindAttributesAndUniforms(); - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - GlUtil.checkGlError(); - } catch (GlUtil.GlException e) { - throw new FrameProcessingException(e, presentationTimeUs); - } finally { + @Override + public void releaseOutputFrame(TextureInfo outputTexture) { + if (areMultipleOutputFramesSupported()) { + checkStateNotNull(checkStateNotNull(outputFrames).get(outputTexture)).release(); + } else { + checkState(Util.areEqual(outputTexture, this.outputTexture)); + this.outputTexture = null; checkStateNotNull(outputFrame).release(); + outputFrame = null; } } @Override - public void release() throws FrameProcessingException { - super.release(); + public void release() { checkStateNotNull(frameProcessor).close(); } + + @Override + public final void signalEndOfInputStream() { + frameProcessor.waitUntilIdle(); + if (listener != null) { + listener.onOutputStreamEnded(); + } + } + + @ChecksSdkIntAtLeast(api = Build.VERSION_CODES.M) + private static boolean areMultipleOutputFramesSupported() { + // Android devices running Lollipop (API 21/22) have a bug in ConcurrentHashMap that can result + // in lost updates, so we only allow one output frame to be pending at a time to avoid using + // ConcurrentHashMap. + return Util.SDK_INT >= 23; + } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index 263714eca3..880c36bdf3 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -491,7 +491,7 @@ public final class FrameProcessorChainPixelTest { } @Override - public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) + public GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException { return effect.toGlTextureProcessor(context); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java index e582af3309..84040ab6e1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java @@ -23,6 +23,7 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; +import android.opengl.GLES20; import android.util.Size; import android.view.Surface; import android.view.SurfaceHolder; @@ -314,6 +315,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height); renderingTask.run(); EGL14.eglSwapBuffers(eglDisplay, eglSurface); + // Prevents white flashing on the debug SurfaceView when frames are rendered too fast. + GLES20.glFinish(); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java index 10a587a026..0d4b77a388 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java @@ -19,17 +19,15 @@ import android.content.Context; import androidx.media3.common.util.UnstableApi; /** - * Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation. + * Interface for a video frame effect with a {@link GlTextureProcessor} implementation. * *

Implementations contain information specifying the effect and can be {@linkplain - * #toGlTextureProcessor(Context) converted} to a {@link SingleFrameGlTextureProcessor} which - * applies the effect. + * #toGlTextureProcessor(Context) converted} to a {@link GlTextureProcessor} which applies the + * effect. */ @UnstableApi public interface GlEffect { /** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */ - // TODO(b/227625423): use GlTextureProcessor here once this interface exists. - SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) - throws FrameProcessingException; + GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TextureInfo.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TextureInfo.java index f81f99d2c0..9e96c8e923 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TextureInfo.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TextureInfo.java @@ -19,7 +19,7 @@ import androidx.media3.common.util.UnstableApi; /** Contains information describing an OpenGL texture. */ @UnstableApi -/* package */ final class TextureInfo { +public final class TextureInfo { /** The OpenGL texture identifier. */ public final int texId; /** Identifier of a framebuffer object associated with the texture. */