Fix ExternalTextureManager: repeated queueing input frame in preview

TL;DR: we should check if there are new frames available to queue to the
ExternalTextureProcessor before actually queueing a frame.

The overall flow on the external texture processor:

- `SurfaceTexture.onFrameAvailable` is called on `ExtTexMgr`, and
  - it calls `updateTexImage()`, and sets `frame`
  - it calls `maybeQueueFrameToExtTexProc()`
    - the frame is queued to `ExtTexProc` if `frame` is set

  - From `ExtTexProc.queueInputFrame()`:
    - notifies the `frameProcessorListener` of available frame
    - notifies the `inputListener` of `onReadyToAcceptInputFrame`
      - (`ExtTexMgr` is the listener), it calls `maybeQueueFrameToExtTexProc()`
       again

-- Parallelly --
- `ExtTexProc` calls `inputListener.onInputFrameProcessed`, when the frame is
released
  - (`ExtTexMgr` is the listener), sets `frame` to `null`

*Problem*

This logic relies on `frame` to be cleared at the right time.

In transformer, it's OK b/c `ExtTexProc` release the frame immediately in
`queueInputFrame()` and calls `onInputFrameProcessed` which also reset `frame`

But in previewing, the frame is not released for a while, up to 10 ms.
In this case, `frame` will not reset in this 10 ms, and
`maybeQueueFrameToExtTexProc()` is repeatedly queueing the same input frame.

PiperOrigin-RevId: 470211620
(cherry picked from commit a8c54dd378577cc472a1e56fff062b7959e00268)
This commit is contained in:
claincly 2022-08-26 11:03:54 +00:00 committed by microkatz
parent e56f59514f
commit 0f271c20b5

View File

@ -15,6 +15,8 @@
*/ */
package androidx.media3.effect; package androidx.media3.effect;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.WorkerThread; import androidx.annotation.WorkerThread;
@ -41,15 +43,17 @@ import java.util.concurrent.atomic.AtomicInteger;
private final float[] textureTransformMatrix; private final float[] textureTransformMatrix;
private final Queue<FrameInfo> pendingFrames; private final Queue<FrameInfo> pendingFrames;
// Incremented on any thread, decremented on the GL thread only. // Incremented on any thread when a frame becomes available on the surfaceTexture, decremented on
// the GL thread only.
private final AtomicInteger availableFrameCount; private final AtomicInteger availableFrameCount;
// Incremented on any thread, decremented on the GL thread only. // Incremented on any thread, decremented on the GL thread only.
private final AtomicInteger externalTextureProcessorInputCapacity; private final AtomicInteger externalTextureProcessorInputCapacity;
// Set to true on any thread. Read on the GL thread only. // Set to true on any thread. Read on the GL thread only.
private volatile boolean inputStreamEnded; private volatile boolean inputStreamEnded;
// The frame that is sent downstream and is not done processing yet.
// Set to null on any thread. Read and set to non-null on the GL thread only. // Set to null on any thread. Read and set to non-null on the GL thread only.
@Nullable private volatile FrameInfo frame; @Nullable private volatile FrameInfo currentFrame;
private long previousStreamOffsetUs; private long previousStreamOffsetUs;
@ -84,12 +88,7 @@ import java.util.concurrent.atomic.AtomicInteger;
surfaceTexture.setOnFrameAvailableListener( surfaceTexture.setOnFrameAvailableListener(
unused -> { unused -> {
availableFrameCount.getAndIncrement(); availableFrameCount.getAndIncrement();
frameProcessingTaskExecutor.submit( frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor);
() -> {
if (maybeUpdateFrame()) {
maybeQueueFrameToExternalTextureProcessor();
}
});
}); });
return surfaceTexture; return surfaceTexture;
} }
@ -102,20 +101,15 @@ import java.util.concurrent.atomic.AtomicInteger;
@Override @Override
public void onInputFrameProcessed(TextureInfo inputTexture) { public void onInputFrameProcessed(TextureInfo inputTexture) {
frame = null; currentFrame = null;
frameProcessingTaskExecutor.submit( frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor);
() -> {
if (maybeUpdateFrame()) {
maybeQueueFrameToExternalTextureProcessor();
}
});
} }
/** /**
* Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will * Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will
* become available via the {@link SurfaceTexture} eventually. * become available via the {@link SurfaceTexture} eventually.
* *
* <p>Can be called on any thread, but the caller must ensure that frames are registered in the * <p>Can be called on any thread. The caller must ensure that frames are registered in the
* correct order. * correct order.
*/ */
public void registerInputFrame(FrameInfo frame) { public void registerInputFrame(FrameInfo frame) {
@ -140,7 +134,7 @@ import java.util.concurrent.atomic.AtomicInteger;
@WorkerThread @WorkerThread
public void signalEndOfInput() { public void signalEndOfInput() {
inputStreamEnded = true; inputStreamEnded = true;
if (pendingFrames.isEmpty() && frame == null) { if (pendingFrames.isEmpty() && currentFrame == null) {
externalTextureProcessor.signalEndOfCurrentInputStream(); externalTextureProcessor.signalEndOfCurrentInputStream();
} }
} }
@ -150,29 +144,23 @@ import java.util.concurrent.atomic.AtomicInteger;
} }
@WorkerThread @WorkerThread
private boolean maybeUpdateFrame() { private void maybeQueueFrameToExternalTextureProcessor() {
if (frame != null || availableFrameCount.get() == 0) { if (externalTextureProcessorInputCapacity.get() == 0
return false; || availableFrameCount.get() == 0
|| currentFrame != null) {
return;
} }
availableFrameCount.getAndDecrement(); availableFrameCount.getAndDecrement();
surfaceTexture.updateTexImage(); surfaceTexture.updateTexImage();
frame = pendingFrames.remove(); this.currentFrame = pendingFrames.remove();
return true;
}
@WorkerThread FrameInfo currentFrame = checkNotNull(this.currentFrame);
private void maybeQueueFrameToExternalTextureProcessor() {
if (externalTextureProcessorInputCapacity.get() == 0 || frame == null) {
return;
}
FrameInfo frame = this.frame;
externalTextureProcessorInputCapacity.getAndDecrement(); externalTextureProcessorInputCapacity.getAndDecrement();
surfaceTexture.getTransformMatrix(textureTransformMatrix); surfaceTexture.getTransformMatrix(textureTransformMatrix);
externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix); externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);
long frameTimeNs = surfaceTexture.getTimestamp(); long frameTimeNs = surfaceTexture.getTimestamp();
long streamOffsetUs = frame.streamOffsetUs; long streamOffsetUs = currentFrame.streamOffsetUs;
if (streamOffsetUs != previousStreamOffsetUs) { if (streamOffsetUs != previousStreamOffsetUs) {
if (previousStreamOffsetUs != C.TIME_UNSET) { if (previousStreamOffsetUs != C.TIME_UNSET) {
externalTextureProcessor.signalEndOfCurrentInputStream(); externalTextureProcessor.signalEndOfCurrentInputStream();
@ -182,7 +170,8 @@ import java.util.concurrent.atomic.AtomicInteger;
// Correct for the stream offset so processors see original media presentation timestamps. // Correct for the stream offset so processors see original media presentation timestamps.
long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs; long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs;
externalTextureProcessor.queueInputFrame( externalTextureProcessor.queueInputFrame(
new TextureInfo(externalTexId, /* fboId= */ C.INDEX_UNSET, frame.width, frame.height), new TextureInfo(
externalTexId, /* fboId= */ C.INDEX_UNSET, currentFrame.width, currentFrame.height),
presentationTimeUs); presentationTimeUs);
if (inputStreamEnded && pendingFrames.isEmpty()) { if (inputStreamEnded && pendingFrames.isEmpty()) {