loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use multiple output frames in transformer MediaPipe demo pre API 23.

Browse Source 
PiperOrigin-RevId: 465545764
This commit is contained in:
hschlueter 2022-08-05 12:58:48 +00:00 committed by tonihei
parent a76dbfd363
commit 51a1a879fa
2 changed files with 9 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
demos/transformer/build.gradle
View File

@ -20,6 +20,7 @@ android {
compileSdkVersion project.ext.compileSdkVersion
compileOptions {
coreLibraryDesugaringEnabled true
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
@ -82,6 +83,8 @@ dependencies {
implementation project(modulePrefix + 'lib-transformer')
implementation project(modulePrefix + 'lib-ui')
coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:1.1.5'
// For MediaPipe and its dependencies:
withMediaPipeImplementation fileTree(dir: 'libs', include: ['*.aar'])
withMediaPipeImplementation 'com.google.flogger:flogger:latest.release'

44
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
View File

@ -21,13 +21,9 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.opengl.EGL14;
import android.os.Build;
import androidx.annotation.ChecksSdkIntAtLeast;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.LibraryLoader;
import androidx.media3.common.util.Util;
import androidx.media3.effect.GlTextureProcessor;
import androidx.media3.effect.TextureInfo;
import com.google.mediapipe.components.FrameProcessor;
@ -61,11 +57,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final FrameProcessor frameProcessor;
private volatile GlTextureProcessor.@MonotonicNonNull Listener listener;
private volatile boolean acceptedFrame;
// Only available from API 23 and above.
@Nullable private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
// Used instead for API 21 and 22.
@Nullable private volatile TextureInfo outputTexture;
@Nullable private volatile TextureFrame outputFrame;
private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
/**
* Creates a new texture processor that wraps a MediaPipe graph.
@ -91,7 +83,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
frameProcessor =
new FrameProcessor(
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
outputFrames = areMultipleOutputFramesSupported() ? new ConcurrentHashMap<>() : null;
outputFrames = new ConcurrentHashMap<>();
frameProcessor.setConsumer(
frame -> {
TextureInfo texture =
@ -100,12 +92,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* fboId= */ C.INDEX_UNSET,
frame.getWidth(),
frame.getHeight());
if (areMultipleOutputFramesSupported()) {
checkStateNotNull(outputFrames).put(texture, frame);
} else {
outputFrame = frame;
outputTexture = texture;
}
outputFrames.put(texture, frame);
if (listener != null) {
listener.onOutputFrameAvailable(texture, frame.getTimestamp());
}
@ -126,17 +113,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
if (!areMultipleOutputFramesSupported() && outputTexture != null) {
return false;
}
acceptedFrame = false;
AppTextureFrame appTextureFrame =
new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height);
// TODO(b/238302213): Handle timestamps restarting from 0 when applying effects to a playlist.
// MediaPipe will fail if the timestamps are not monotonically increasing.
appTextureFrame.setTimestamp(presentationTimeUs);
checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame);
frameProcessor.onNewFrame(appTextureFrame);
try {
appTextureFrame.waitUntilReleasedWithGpuSync();
} catch (InterruptedException e) {
@ -153,19 +136,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void releaseOutputFrame(TextureInfo outputTexture) {
if (areMultipleOutputFramesSupported()) {
checkStateNotNull(checkStateNotNull(outputFrames).get(outputTexture)).release();
} else {
checkState(Util.areEqual(outputTexture, this.outputTexture));
this.outputTexture = null;
checkStateNotNull(outputFrame).release();
outputFrame = null;
}
checkStateNotNull(outputFrames.get(outputTexture)).release();
}
@Override
public void release() {
checkStateNotNull(frameProcessor).close();
frameProcessor.close();
}
@Override
@ -175,12 +151,4 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
listener.onCurrentOutputStreamEnded();
}
}
@ChecksSdkIntAtLeast(api = Build.VERSION_CODES.M)
private static boolean areMultipleOutputFramesSupported() {
// Android devices running Lollipop (API 21/22) have a bug in ConcurrentHashMap that can result
// in lost updates, so we only allow one output frame to be pending at a time to avoid using
// ConcurrentHashMap.
return Util.SDK_INT >= 23;
}
}