mirror of
https://github.com/androidx/media.git
synced 2025-04-30 06:46:50 +08:00
Use multiple output frames in transformer MediaPipe demo pre API 23.
PiperOrigin-RevId: 465545764 (cherry picked from commit 51a1a879faadf749348902c12b382b42d54cd202)
This commit is contained in:
parent
d34ed7a5e1
commit
fa1a7eb51e
@ -20,6 +20,7 @@ android {
|
|||||||
compileSdkVersion project.ext.compileSdkVersion
|
compileSdkVersion project.ext.compileSdkVersion
|
||||||
|
|
||||||
compileOptions {
|
compileOptions {
|
||||||
|
coreLibraryDesugaringEnabled true
|
||||||
sourceCompatibility JavaVersion.VERSION_1_8
|
sourceCompatibility JavaVersion.VERSION_1_8
|
||||||
targetCompatibility JavaVersion.VERSION_1_8
|
targetCompatibility JavaVersion.VERSION_1_8
|
||||||
}
|
}
|
||||||
@ -82,6 +83,8 @@ dependencies {
|
|||||||
implementation project(modulePrefix + 'lib-transformer')
|
implementation project(modulePrefix + 'lib-transformer')
|
||||||
implementation project(modulePrefix + 'lib-ui')
|
implementation project(modulePrefix + 'lib-ui')
|
||||||
|
|
||||||
|
coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:1.1.5'
|
||||||
|
|
||||||
// For MediaPipe and its dependencies:
|
// For MediaPipe and its dependencies:
|
||||||
withMediaPipeImplementation fileTree(dir: 'libs', include: ['*.aar'])
|
withMediaPipeImplementation fileTree(dir: 'libs', include: ['*.aar'])
|
||||||
withMediaPipeImplementation 'com.google.flogger:flogger:latest.release'
|
withMediaPipeImplementation 'com.google.flogger:flogger:latest.release'
|
||||||
|
@ -21,13 +21,9 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
|||||||
|
|
||||||
import android.content.Context;
|
import android.content.Context;
|
||||||
import android.opengl.EGL14;
|
import android.opengl.EGL14;
|
||||||
import android.os.Build;
|
|
||||||
import androidx.annotation.ChecksSdkIntAtLeast;
|
|
||||||
import androidx.annotation.Nullable;
|
|
||||||
import androidx.media3.common.C;
|
import androidx.media3.common.C;
|
||||||
import androidx.media3.common.FrameProcessingException;
|
import androidx.media3.common.FrameProcessingException;
|
||||||
import androidx.media3.common.util.LibraryLoader;
|
import androidx.media3.common.util.LibraryLoader;
|
||||||
import androidx.media3.common.util.Util;
|
|
||||||
import androidx.media3.effect.GlTextureProcessor;
|
import androidx.media3.effect.GlTextureProcessor;
|
||||||
import androidx.media3.effect.TextureInfo;
|
import androidx.media3.effect.TextureInfo;
|
||||||
import com.google.mediapipe.components.FrameProcessor;
|
import com.google.mediapipe.components.FrameProcessor;
|
||||||
@ -61,11 +57,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
private final FrameProcessor frameProcessor;
|
private final FrameProcessor frameProcessor;
|
||||||
private volatile GlTextureProcessor.@MonotonicNonNull Listener listener;
|
private volatile GlTextureProcessor.@MonotonicNonNull Listener listener;
|
||||||
private volatile boolean acceptedFrame;
|
private volatile boolean acceptedFrame;
|
||||||
// Only available from API 23 and above.
|
private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
|
||||||
@Nullable private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
|
|
||||||
// Used instead for API 21 and 22.
|
|
||||||
@Nullable private volatile TextureInfo outputTexture;
|
|
||||||
@Nullable private volatile TextureFrame outputFrame;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a new texture processor that wraps a MediaPipe graph.
|
* Creates a new texture processor that wraps a MediaPipe graph.
|
||||||
@ -91,7 +83,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
frameProcessor =
|
frameProcessor =
|
||||||
new FrameProcessor(
|
new FrameProcessor(
|
||||||
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
|
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
|
||||||
outputFrames = areMultipleOutputFramesSupported() ? new ConcurrentHashMap<>() : null;
|
outputFrames = new ConcurrentHashMap<>();
|
||||||
frameProcessor.setConsumer(
|
frameProcessor.setConsumer(
|
||||||
frame -> {
|
frame -> {
|
||||||
TextureInfo texture =
|
TextureInfo texture =
|
||||||
@ -100,12 +92,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
/* fboId= */ C.INDEX_UNSET,
|
/* fboId= */ C.INDEX_UNSET,
|
||||||
frame.getWidth(),
|
frame.getWidth(),
|
||||||
frame.getHeight());
|
frame.getHeight());
|
||||||
if (areMultipleOutputFramesSupported()) {
|
outputFrames.put(texture, frame);
|
||||||
checkStateNotNull(outputFrames).put(texture, frame);
|
|
||||||
} else {
|
|
||||||
outputFrame = frame;
|
|
||||||
outputTexture = texture;
|
|
||||||
}
|
|
||||||
if (listener != null) {
|
if (listener != null) {
|
||||||
listener.onOutputFrameAvailable(texture, frame.getTimestamp());
|
listener.onOutputFrameAvailable(texture, frame.getTimestamp());
|
||||||
}
|
}
|
||||||
@ -126,17 +113,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
||||||
if (!areMultipleOutputFramesSupported() && outputTexture != null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
acceptedFrame = false;
|
acceptedFrame = false;
|
||||||
AppTextureFrame appTextureFrame =
|
AppTextureFrame appTextureFrame =
|
||||||
new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height);
|
new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height);
|
||||||
// TODO(b/238302213): Handle timestamps restarting from 0 when applying effects to a playlist.
|
// TODO(b/238302213): Handle timestamps restarting from 0 when applying effects to a playlist.
|
||||||
// MediaPipe will fail if the timestamps are not monotonically increasing.
|
// MediaPipe will fail if the timestamps are not monotonically increasing.
|
||||||
appTextureFrame.setTimestamp(presentationTimeUs);
|
appTextureFrame.setTimestamp(presentationTimeUs);
|
||||||
checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame);
|
frameProcessor.onNewFrame(appTextureFrame);
|
||||||
try {
|
try {
|
||||||
appTextureFrame.waitUntilReleasedWithGpuSync();
|
appTextureFrame.waitUntilReleasedWithGpuSync();
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
@ -153,19 +136,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void releaseOutputFrame(TextureInfo outputTexture) {
|
public void releaseOutputFrame(TextureInfo outputTexture) {
|
||||||
if (areMultipleOutputFramesSupported()) {
|
checkStateNotNull(outputFrames.get(outputTexture)).release();
|
||||||
checkStateNotNull(checkStateNotNull(outputFrames).get(outputTexture)).release();
|
|
||||||
} else {
|
|
||||||
checkState(Util.areEqual(outputTexture, this.outputTexture));
|
|
||||||
this.outputTexture = null;
|
|
||||||
checkStateNotNull(outputFrame).release();
|
|
||||||
outputFrame = null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void release() {
|
public void release() {
|
||||||
checkStateNotNull(frameProcessor).close();
|
frameProcessor.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -175,12 +151,4 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
listener.onCurrentOutputStreamEnded();
|
listener.onCurrentOutputStreamEnded();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ChecksSdkIntAtLeast(api = Build.VERSION_CODES.M)
|
|
||||||
private static boolean areMultipleOutputFramesSupported() {
|
|
||||||
// Android devices running Lollipop (API 21/22) have a bug in ConcurrentHashMap that can result
|
|
||||||
// in lost updates, so we only allow one output frame to be pending at a time to avoid using
|
|
||||||
// ConcurrentHashMap.
|
|
||||||
return Util.SDK_INT >= 23;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user