From abd96598d9edee52c0a7815c74e0c1347a7bd1f4 Mon Sep 17 00:00:00 2001 From: claincly Date: Tue, 13 Jun 2023 19:30:57 +0100 Subject: [PATCH] Add a timer to end a video stream prematurely in ExtTexMgr Partially roll forward of https://github.com/androidx/media/commit/5c29abbbf4d2fb7c7770bcc79a1027c532f7b96e, and adds some extra logic Changes to the original CL The original logic (https://github.com/androidx/media/commit/a66f08ba978c2bd146242eec86dd69d8a85b5408) fails in the following case: > This is only seem on emulators. - EOS is sent to ExtTexMgr - The timer starts - One frame arrives on SurfaceTexture, reset the timer - The frame is sent for processing, now `availablFrames == 0` - One frame arrives on Surface, reset the timer - The frame is kept on SurfaceTexture for the downstream shader doesn't have capacity, `availablFrames == 1` - Timer times out as the downstream processor doesn't report being able to take another frame. - Although there's a frame available on the SurfaceTexture This is solved by having the force EOS logic clear all the frames that the SurfaceTexture holds. This also ensures the first frame dequeued from the next stream isn't from the previous stream. PiperOrigin-RevId: 540023359 --- .../media3/effect/ExternalTextureManager.java | 87 ++++++++++++++++++- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index 2a2337023c..2e41d81713 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -17,6 +17,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import android.graphics.SurfaceTexture; import android.view.Surface; @@ -26,9 +27,13 @@ import androidx.media3.common.FrameInfo; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.Util; import androidx.media3.effect.GlShaderProgram.InputListener; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; /** @@ -37,6 +42,19 @@ import java.util.concurrent.atomic.AtomicInteger; */ /* package */ final class ExternalTextureManager implements TextureManager { + private static final String TAG = "ExtTexMgr"; + private static final String TIMER_THREAD_NAME = "ExtTexMgr:Timer"; + /** + * The time out in milliseconds after calling signalEndOfCurrentInputStream after which the input + * stream is considered to have ended, even if not all expected frames have been received from the + * decoder. This has been observed on some decoders. Some emulator decoders are slower, hence + * using a longer timeout. Also on some emulators, GL operation takes a long time to finish, the + * timeout could be a result of slow GL operation back pressured the decoder, and the decoder is + * not able to decode another frame. + */ + private static final long SURFACE_TEXTURE_TIMEOUT_MS = + Util.DEVICE.contains("emulator") ? 10_000 : 500; + private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; private final ExternalShaderProgram externalShaderProgram; private final int externalTexId; @@ -44,6 +62,7 @@ import java.util.concurrent.atomic.AtomicInteger; private final SurfaceTexture surfaceTexture; private final float[] textureTransformMatrix; private final Queue pendingFrames; + private final ScheduledExecutorService forceEndOfStreamExecutorService; // Incremented on any thread, decremented on the GL thread only. private final AtomicInteger externalShaderProgramInputCapacity; @@ -66,6 +85,10 @@ import java.util.concurrent.atomic.AtomicInteger; // TODO(b/238302341) Remove the use of after flush task, block the calling thread instead. @Nullable private volatile VideoFrameProcessingTask onFlushCompleteTask; + @Nullable private Future forceSignalEndOfStreamFuture; + + // Whether to reject frames from the SurfaceTexture. Accessed only on GL thread. + private boolean shouldRejectIncomingFrames; /** * Creates a new instance. @@ -91,6 +114,7 @@ import java.util.concurrent.atomic.AtomicInteger; surfaceTexture = new SurfaceTexture(externalTexId); textureTransformMatrix = new float[16]; pendingFrames = new ConcurrentLinkedQueue<>(); + forceEndOfStreamExecutorService = Util.newSingleThreadScheduledExecutor(TIMER_THREAD_NAME); externalShaderProgramInputCapacity = new AtomicInteger(); surfaceTexture.setOnFrameAvailableListener( unused -> @@ -101,7 +125,16 @@ import java.util.concurrent.atomic.AtomicInteger; numberOfFramesToDropOnBecomingAvailable--; surfaceTexture.updateTexImage(); maybeExecuteAfterFlushTask(); + } else if (shouldRejectIncomingFrames) { + surfaceTexture.updateTexImage(); + Log.w( + TAG, + "Dropping frame received on SurfaceTexture after forcing EOS: " + + surfaceTexture.getTimestamp() / 1000); } else { + if (currentInputStreamEnded) { + restartForceSignalEndOfStreamTimer(); + } availableFrameCount++; maybeQueueFrameToExternalShaderProgram(); } @@ -138,6 +171,7 @@ import java.util.concurrent.atomic.AtomicInteger; currentInputStreamEnded = false; externalShaderProgram.signalEndOfCurrentInputStream(); DebugTraceUtil.recordExternalInputManagerSignalEndOfCurrentInputStream(); + cancelForceSignalEndOfStreamTimer(); } else { maybeQueueFrameToExternalShaderProgram(); } @@ -165,6 +199,7 @@ import java.util.concurrent.atomic.AtomicInteger; public void registerInputFrame(FrameInfo frame) { checkState(!inputStreamEnded); pendingFrames.add(frame); + videoFrameProcessingTaskExecutor.submit(() -> shouldRejectIncomingFrames = false); } /** @@ -185,8 +220,10 @@ import java.util.concurrent.atomic.AtomicInteger; if (pendingFrames.isEmpty() && currentFrame == null) { externalShaderProgram.signalEndOfCurrentInputStream(); DebugTraceUtil.recordExternalInputManagerSignalEndOfCurrentInputStream(); + cancelForceSignalEndOfStreamTimer(); } else { currentInputStreamEnded = true; + restartForceSignalEndOfStreamTimer(); } }); } @@ -201,6 +238,7 @@ import java.util.concurrent.atomic.AtomicInteger; public void release() { surfaceTexture.release(); surface.release(); + forceEndOfStreamExecutorService.shutdownNow(); } private void maybeExecuteAfterFlushTask() { @@ -212,19 +250,60 @@ import java.util.concurrent.atomic.AtomicInteger; // Methods that must be called on the GL thread. + private void restartForceSignalEndOfStreamTimer() { + cancelForceSignalEndOfStreamTimer(); + forceSignalEndOfStreamFuture = + forceEndOfStreamExecutorService.schedule( + () -> videoFrameProcessingTaskExecutor.submit(this::forceSignalEndOfStream), + SURFACE_TEXTURE_TIMEOUT_MS, + MILLISECONDS); + } + + private void cancelForceSignalEndOfStreamTimer() { + if (forceSignalEndOfStreamFuture != null) { + forceSignalEndOfStreamFuture.cancel(/* mayInterruptIfRunning= */ false); + } + forceSignalEndOfStreamFuture = null; + } + + private void forceSignalEndOfStream() { + // Reset because there could be further input streams after the current one ends. + Log.w( + TAG, + Util.formatInvariant( + "Forcing EOS after missing %d frames for %d ms, with available frame count: %d", + pendingFrames.size(), SURFACE_TEXTURE_TIMEOUT_MS, availableFrameCount)); + // Reset because there could be further input streams after the current one ends. + currentInputStreamEnded = false; + currentFrame = null; + pendingFrames.clear(); + shouldRejectIncomingFrames = true; + + // Frames could be made available while waiting for OpenGL to finish processing. That is, + // time out is triggered while waiting for the downstream shader programs to process a frame, + // when there are frames available on the SurfaceTexture. This has only been observed on + // emulators. + removeAllSurfaceTextureFrames(); + signalEndOfCurrentInputStream(); + } + private void flush() { // A frame that is registered before flush may arrive after flush. numberOfFramesToDropOnBecomingAvailable = pendingFrames.size() - availableFrameCount; - while (availableFrameCount > 0) { - availableFrameCount--; - surfaceTexture.updateTexImage(); - } + removeAllSurfaceTextureFrames(); externalShaderProgramInputCapacity.set(0); currentFrame = null; pendingFrames.clear(); maybeExecuteAfterFlushTask(); } + private void removeAllSurfaceTextureFrames() { + while (availableFrameCount > 0) { + availableFrameCount--; + surfaceTexture.updateTexImage(); + } + } + private void maybeQueueFrameToExternalShaderProgram() { if (externalShaderProgramInputCapacity.get() == 0 || availableFrameCount == 0