From 0f271c20b5d0b22b85e94453b1f2f8ef4863bbbb Mon Sep 17 00:00:00 2001 From: claincly Date: Fri, 26 Aug 2022 11:03:54 +0000 Subject: [PATCH] Fix ExternalTextureManager: repeated queueing input frame in preview TL;DR: we should check if there are new frames available to queue to the ExternalTextureProcessor before actually queueing a frame. The overall flow on the external texture processor: - `SurfaceTexture.onFrameAvailable` is called on `ExtTexMgr`, and - it calls `updateTexImage()`, and sets `frame` - it calls `maybeQueueFrameToExtTexProc()` - the frame is queued to `ExtTexProc` if `frame` is set - From `ExtTexProc.queueInputFrame()`: - notifies the `frameProcessorListener` of available frame - notifies the `inputListener` of `onReadyToAcceptInputFrame` - (`ExtTexMgr` is the listener), it calls `maybeQueueFrameToExtTexProc()` again -- Parallelly -- - `ExtTexProc` calls `inputListener.onInputFrameProcessed`, when the frame is released - (`ExtTexMgr` is the listener), sets `frame` to `null` *Problem* This logic relies on `frame` to be cleared at the right time. In transformer, it's OK b/c `ExtTexProc` release the frame immediately in `queueInputFrame()` and calls `onInputFrameProcessed` which also reset `frame` But in previewing, the frame is not released for a while, up to 10 ms. In this case, `frame` will not reset in this 10 ms, and `maybeQueueFrameToExtTexProc()` is repeatedly queueing the same input frame. PiperOrigin-RevId: 470211620 (cherry picked from commit a8c54dd378577cc472a1e56fff062b7959e00268) --- .../media3/effect/ExternalTextureManager.java | 53 ++++++++----------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index 69dbfb7d1f..1bb77ebacc 100644 --- a/google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -15,6 +15,8 @@ */ package androidx.media3.effect; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; + import android.graphics.SurfaceTexture; import androidx.annotation.Nullable; import androidx.annotation.WorkerThread; @@ -41,15 +43,17 @@ import java.util.concurrent.atomic.AtomicInteger; private final float[] textureTransformMatrix; private final Queue pendingFrames; - // Incremented on any thread, decremented on the GL thread only. + // Incremented on any thread when a frame becomes available on the surfaceTexture, decremented on + // the GL thread only. private final AtomicInteger availableFrameCount; // Incremented on any thread, decremented on the GL thread only. private final AtomicInteger externalTextureProcessorInputCapacity; // Set to true on any thread. Read on the GL thread only. private volatile boolean inputStreamEnded; + // The frame that is sent downstream and is not done processing yet. // Set to null on any thread. Read and set to non-null on the GL thread only. - @Nullable private volatile FrameInfo frame; + @Nullable private volatile FrameInfo currentFrame; private long previousStreamOffsetUs; @@ -84,12 +88,7 @@ import java.util.concurrent.atomic.AtomicInteger; surfaceTexture.setOnFrameAvailableListener( unused -> { availableFrameCount.getAndIncrement(); - frameProcessingTaskExecutor.submit( - () -> { - if (maybeUpdateFrame()) { - maybeQueueFrameToExternalTextureProcessor(); - } - }); + frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor); }); return surfaceTexture; } @@ -102,20 +101,15 @@ import java.util.concurrent.atomic.AtomicInteger; @Override public void onInputFrameProcessed(TextureInfo inputTexture) { - frame = null; - frameProcessingTaskExecutor.submit( - () -> { - if (maybeUpdateFrame()) { - maybeQueueFrameToExternalTextureProcessor(); - } - }); + currentFrame = null; + frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor); } /** * Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will * become available via the {@link SurfaceTexture} eventually. * - *

Can be called on any thread, but the caller must ensure that frames are registered in the + *

Can be called on any thread. The caller must ensure that frames are registered in the * correct order. */ public void registerInputFrame(FrameInfo frame) { @@ -140,7 +134,7 @@ import java.util.concurrent.atomic.AtomicInteger; @WorkerThread public void signalEndOfInput() { inputStreamEnded = true; - if (pendingFrames.isEmpty() && frame == null) { + if (pendingFrames.isEmpty() && currentFrame == null) { externalTextureProcessor.signalEndOfCurrentInputStream(); } } @@ -150,29 +144,23 @@ import java.util.concurrent.atomic.AtomicInteger; } @WorkerThread - private boolean maybeUpdateFrame() { - if (frame != null || availableFrameCount.get() == 0) { - return false; + private void maybeQueueFrameToExternalTextureProcessor() { + if (externalTextureProcessorInputCapacity.get() == 0 + || availableFrameCount.get() == 0 + || currentFrame != null) { + return; } availableFrameCount.getAndDecrement(); surfaceTexture.updateTexImage(); - frame = pendingFrames.remove(); - return true; - } + this.currentFrame = pendingFrames.remove(); - @WorkerThread - private void maybeQueueFrameToExternalTextureProcessor() { - if (externalTextureProcessorInputCapacity.get() == 0 || frame == null) { - return; - } - - FrameInfo frame = this.frame; + FrameInfo currentFrame = checkNotNull(this.currentFrame); externalTextureProcessorInputCapacity.getAndDecrement(); surfaceTexture.getTransformMatrix(textureTransformMatrix); externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix); long frameTimeNs = surfaceTexture.getTimestamp(); - long streamOffsetUs = frame.streamOffsetUs; + long streamOffsetUs = currentFrame.streamOffsetUs; if (streamOffsetUs != previousStreamOffsetUs) { if (previousStreamOffsetUs != C.TIME_UNSET) { externalTextureProcessor.signalEndOfCurrentInputStream(); @@ -182,7 +170,8 @@ import java.util.concurrent.atomic.AtomicInteger; // Correct for the stream offset so processors see original media presentation timestamps. long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs; externalTextureProcessor.queueInputFrame( - new TextureInfo(externalTexId, /* fboId= */ C.INDEX_UNSET, frame.width, frame.height), + new TextureInfo( + externalTexId, /* fboId= */ C.INDEX_UNSET, currentFrame.width, currentFrame.height), presentationTimeUs); if (inputStreamEnded && pendingFrames.isEmpty()) {