Fix ExternalTextureManager: repeated queueing input frame in preview
TL;DR: we should check if there are new frames available to queue to the ExternalTextureProcessor before actually queueing a frame. The overall flow on the external texture processor: - `SurfaceTexture.onFrameAvailable` is called on `ExtTexMgr`, and - it calls `updateTexImage()`, and sets `frame` - it calls `maybeQueueFrameToExtTexProc()` - the frame is queued to `ExtTexProc` if `frame` is set - From `ExtTexProc.queueInputFrame()`: - notifies the `frameProcessorListener` of available frame - notifies the `inputListener` of `onReadyToAcceptInputFrame` - (`ExtTexMgr` is the listener), it calls `maybeQueueFrameToExtTexProc()` again -- Parallelly -- - `ExtTexProc` calls `inputListener.onInputFrameProcessed`, when the frame is released - (`ExtTexMgr` is the listener), sets `frame` to `null` *Problem* This logic relies on `frame` to be cleared at the right time. In transformer, it's OK b/c `ExtTexProc` release the frame immediately in `queueInputFrame()` and calls `onInputFrameProcessed` which also reset `frame` But in previewing, the frame is not released for a while, up to 10 ms. In this case, `frame` will not reset in this 10 ms, and `maybeQueueFrameToExtTexProc()` is repeatedly queueing the same input frame. PiperOrigin-RevId: 470211620 (cherry picked from commit a8c54dd378577cc472a1e56fff062b7959e00268)
This commit is contained in:
parent
e56f59514f
commit
0f271c20b5
@ -15,6 +15,8 @@
|
|||||||
*/
|
*/
|
||||||
package androidx.media3.effect;
|
package androidx.media3.effect;
|
||||||
|
|
||||||
|
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
|
||||||
|
|
||||||
import android.graphics.SurfaceTexture;
|
import android.graphics.SurfaceTexture;
|
||||||
import androidx.annotation.Nullable;
|
import androidx.annotation.Nullable;
|
||||||
import androidx.annotation.WorkerThread;
|
import androidx.annotation.WorkerThread;
|
||||||
@ -41,15 +43,17 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||||||
private final float[] textureTransformMatrix;
|
private final float[] textureTransformMatrix;
|
||||||
private final Queue<FrameInfo> pendingFrames;
|
private final Queue<FrameInfo> pendingFrames;
|
||||||
|
|
||||||
// Incremented on any thread, decremented on the GL thread only.
|
// Incremented on any thread when a frame becomes available on the surfaceTexture, decremented on
|
||||||
|
// the GL thread only.
|
||||||
private final AtomicInteger availableFrameCount;
|
private final AtomicInteger availableFrameCount;
|
||||||
// Incremented on any thread, decremented on the GL thread only.
|
// Incremented on any thread, decremented on the GL thread only.
|
||||||
private final AtomicInteger externalTextureProcessorInputCapacity;
|
private final AtomicInteger externalTextureProcessorInputCapacity;
|
||||||
|
|
||||||
// Set to true on any thread. Read on the GL thread only.
|
// Set to true on any thread. Read on the GL thread only.
|
||||||
private volatile boolean inputStreamEnded;
|
private volatile boolean inputStreamEnded;
|
||||||
|
// The frame that is sent downstream and is not done processing yet.
|
||||||
// Set to null on any thread. Read and set to non-null on the GL thread only.
|
// Set to null on any thread. Read and set to non-null on the GL thread only.
|
||||||
@Nullable private volatile FrameInfo frame;
|
@Nullable private volatile FrameInfo currentFrame;
|
||||||
|
|
||||||
private long previousStreamOffsetUs;
|
private long previousStreamOffsetUs;
|
||||||
|
|
||||||
@ -84,12 +88,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||||||
surfaceTexture.setOnFrameAvailableListener(
|
surfaceTexture.setOnFrameAvailableListener(
|
||||||
unused -> {
|
unused -> {
|
||||||
availableFrameCount.getAndIncrement();
|
availableFrameCount.getAndIncrement();
|
||||||
frameProcessingTaskExecutor.submit(
|
frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor);
|
||||||
() -> {
|
|
||||||
if (maybeUpdateFrame()) {
|
|
||||||
maybeQueueFrameToExternalTextureProcessor();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
return surfaceTexture;
|
return surfaceTexture;
|
||||||
}
|
}
|
||||||
@ -102,20 +101,15 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onInputFrameProcessed(TextureInfo inputTexture) {
|
public void onInputFrameProcessed(TextureInfo inputTexture) {
|
||||||
frame = null;
|
currentFrame = null;
|
||||||
frameProcessingTaskExecutor.submit(
|
frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor);
|
||||||
() -> {
|
|
||||||
if (maybeUpdateFrame()) {
|
|
||||||
maybeQueueFrameToExternalTextureProcessor();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will
|
* Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will
|
||||||
* become available via the {@link SurfaceTexture} eventually.
|
* become available via the {@link SurfaceTexture} eventually.
|
||||||
*
|
*
|
||||||
* <p>Can be called on any thread, but the caller must ensure that frames are registered in the
|
* <p>Can be called on any thread. The caller must ensure that frames are registered in the
|
||||||
* correct order.
|
* correct order.
|
||||||
*/
|
*/
|
||||||
public void registerInputFrame(FrameInfo frame) {
|
public void registerInputFrame(FrameInfo frame) {
|
||||||
@ -140,7 +134,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||||||
@WorkerThread
|
@WorkerThread
|
||||||
public void signalEndOfInput() {
|
public void signalEndOfInput() {
|
||||||
inputStreamEnded = true;
|
inputStreamEnded = true;
|
||||||
if (pendingFrames.isEmpty() && frame == null) {
|
if (pendingFrames.isEmpty() && currentFrame == null) {
|
||||||
externalTextureProcessor.signalEndOfCurrentInputStream();
|
externalTextureProcessor.signalEndOfCurrentInputStream();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -150,29 +144,23 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||||||
}
|
}
|
||||||
|
|
||||||
@WorkerThread
|
@WorkerThread
|
||||||
private boolean maybeUpdateFrame() {
|
private void maybeQueueFrameToExternalTextureProcessor() {
|
||||||
if (frame != null || availableFrameCount.get() == 0) {
|
if (externalTextureProcessorInputCapacity.get() == 0
|
||||||
return false;
|
|| availableFrameCount.get() == 0
|
||||||
|
|| currentFrame != null) {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
availableFrameCount.getAndDecrement();
|
availableFrameCount.getAndDecrement();
|
||||||
surfaceTexture.updateTexImage();
|
surfaceTexture.updateTexImage();
|
||||||
frame = pendingFrames.remove();
|
this.currentFrame = pendingFrames.remove();
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@WorkerThread
|
FrameInfo currentFrame = checkNotNull(this.currentFrame);
|
||||||
private void maybeQueueFrameToExternalTextureProcessor() {
|
|
||||||
if (externalTextureProcessorInputCapacity.get() == 0 || frame == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
FrameInfo frame = this.frame;
|
|
||||||
externalTextureProcessorInputCapacity.getAndDecrement();
|
externalTextureProcessorInputCapacity.getAndDecrement();
|
||||||
surfaceTexture.getTransformMatrix(textureTransformMatrix);
|
surfaceTexture.getTransformMatrix(textureTransformMatrix);
|
||||||
externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);
|
externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);
|
||||||
long frameTimeNs = surfaceTexture.getTimestamp();
|
long frameTimeNs = surfaceTexture.getTimestamp();
|
||||||
long streamOffsetUs = frame.streamOffsetUs;
|
long streamOffsetUs = currentFrame.streamOffsetUs;
|
||||||
if (streamOffsetUs != previousStreamOffsetUs) {
|
if (streamOffsetUs != previousStreamOffsetUs) {
|
||||||
if (previousStreamOffsetUs != C.TIME_UNSET) {
|
if (previousStreamOffsetUs != C.TIME_UNSET) {
|
||||||
externalTextureProcessor.signalEndOfCurrentInputStream();
|
externalTextureProcessor.signalEndOfCurrentInputStream();
|
||||||
@ -182,7 +170,8 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||||||
// Correct for the stream offset so processors see original media presentation timestamps.
|
// Correct for the stream offset so processors see original media presentation timestamps.
|
||||||
long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs;
|
long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs;
|
||||||
externalTextureProcessor.queueInputFrame(
|
externalTextureProcessor.queueInputFrame(
|
||||||
new TextureInfo(externalTexId, /* fboId= */ C.INDEX_UNSET, frame.width, frame.height),
|
new TextureInfo(
|
||||||
|
externalTexId, /* fboId= */ C.INDEX_UNSET, currentFrame.width, currentFrame.height),
|
||||||
presentationTimeUs);
|
presentationTimeUs);
|
||||||
|
|
||||||
if (inputStreamEnded && pendingFrames.isEmpty()) {
|
if (inputStreamEnded && pendingFrames.isEmpty()) {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user