loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a timer to end a video stream prematurely in ExtTexMgr

Browse Source 
Partially roll forward of 5c29abbbf4, and adds some extra logic

Changes to the original CL

The original logic (a66f08ba97) fails in the following case:

> This is only seem on emulators.

- EOS is sent to ExtTexMgr
- The timer starts
- One frame arrives on SurfaceTexture, reset the timer
  - The frame is sent for processing, now `availablFrames == 0`
- One frame arrives on Surface, reset the timer
  - The frame is kept on SurfaceTexture for the downstream shader doesn't have
    capacity, `availablFrames == 1`
- Timer times out as the downstream processor doesn't report being able to take
  another frame.
  - Although there's a frame available on the SurfaceTexture

This is solved by having the force EOS logic clear all the frames that the
SurfaceTexture holds. This also ensures the first frame dequeued from the next
stream isn't from the previous stream.

PiperOrigin-RevId: 540023359
This commit is contained in:
claincly 2023-06-13 19:30:57 +01:00 committed by Ian Baker
parent 4b5a457790
commit abd96598d9
1 changed files with 83 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java
View File

@ -17,6 +17,7 @@ package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import android.graphics.SurfaceTexture;
import android.view.Surface;
@ -26,9 +27,13 @@ import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.Util;
import androidx.media3.effect.GlShaderProgram.InputListener;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
/**
@ -37,6 +42,19 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
/* package */ final class ExternalTextureManager implements TextureManager {
private static final String TAG = "ExtTexMgr";
private static final String TIMER_THREAD_NAME = "ExtTexMgr:Timer";
/**
* The time out in milliseconds after calling signalEndOfCurrentInputStream after which the input
* stream is considered to have ended, even if not all expected frames have been received from the
* decoder. This has been observed on some decoders. Some emulator decoders are slower, hence
* using a longer timeout. Also on some emulators, GL operation takes a long time to finish, the
* timeout could be a result of slow GL operation back pressured the decoder, and the decoder is
* not able to decode another frame.
*/
private static final long SURFACE_TEXTURE_TIMEOUT_MS =
Util.DEVICE.contains("emulator") ? 10_000 : 500;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final ExternalShaderProgram externalShaderProgram;
private final int externalTexId;
@ -44,6 +62,7 @@ import java.util.concurrent.atomic.AtomicInteger;
private final SurfaceTexture surfaceTexture;
private final float[] textureTransformMatrix;
private final Queue<FrameInfo> pendingFrames;
private final ScheduledExecutorService forceEndOfStreamExecutorService;
// Incremented on any thread, decremented on the GL thread only.
private final AtomicInteger externalShaderProgramInputCapacity;
@ -66,6 +85,10 @@ import java.util.concurrent.atomic.AtomicInteger;
// TODO(b/238302341) Remove the use of after flush task, block the calling thread instead.
@Nullable private volatile VideoFrameProcessingTask onFlushCompleteTask;
@Nullable private Future<?> forceSignalEndOfStreamFuture;
// Whether to reject frames from the SurfaceTexture. Accessed only on GL thread.
private boolean shouldRejectIncomingFrames;
/**
* Creates a new instance.
@ -91,6 +114,7 @@ import java.util.concurrent.atomic.AtomicInteger;
surfaceTexture = new SurfaceTexture(externalTexId);
textureTransformMatrix = new float[16];
pendingFrames = new ConcurrentLinkedQueue<>();
forceEndOfStreamExecutorService = Util.newSingleThreadScheduledExecutor(TIMER_THREAD_NAME);
externalShaderProgramInputCapacity = new AtomicInteger();
surfaceTexture.setOnFrameAvailableListener(
unused ->
@ -101,7 +125,16 @@ import java.util.concurrent.atomic.AtomicInteger;
numberOfFramesToDropOnBecomingAvailable--;
surfaceTexture.updateTexImage();
maybeExecuteAfterFlushTask();
} else if (shouldRejectIncomingFrames) {
surfaceTexture.updateTexImage();
Log.w(
TAG,
"Dropping frame received on SurfaceTexture after forcing EOS: "
+ surfaceTexture.getTimestamp() / 1000);
} else {
if (currentInputStreamEnded) {
restartForceSignalEndOfStreamTimer();
}
availableFrameCount++;
maybeQueueFrameToExternalShaderProgram();
}
@ -138,6 +171,7 @@ import java.util.concurrent.atomic.AtomicInteger;
currentInputStreamEnded = false;
externalShaderProgram.signalEndOfCurrentInputStream();
DebugTraceUtil.recordExternalInputManagerSignalEndOfCurrentInputStream();
cancelForceSignalEndOfStreamTimer();
} else {
maybeQueueFrameToExternalShaderProgram();
}
@ -165,6 +199,7 @@ import java.util.concurrent.atomic.AtomicInteger;
public void registerInputFrame(FrameInfo frame) {
checkState(!inputStreamEnded);
pendingFrames.add(frame);
videoFrameProcessingTaskExecutor.submit(() -> shouldRejectIncomingFrames = false);
}
/**
@ -185,8 +220,10 @@ import java.util.concurrent.atomic.AtomicInteger;
if (pendingFrames.isEmpty() && currentFrame == null) {
externalShaderProgram.signalEndOfCurrentInputStream();
DebugTraceUtil.recordExternalInputManagerSignalEndOfCurrentInputStream();
cancelForceSignalEndOfStreamTimer();
} else {
currentInputStreamEnded = true;
restartForceSignalEndOfStreamTimer();
}
});
}
@ -201,6 +238,7 @@ import java.util.concurrent.atomic.AtomicInteger;
public void release() {
surfaceTexture.release();
surface.release();
forceEndOfStreamExecutorService.shutdownNow();
}
private void maybeExecuteAfterFlushTask() {
@ -212,19 +250,60 @@ import java.util.concurrent.atomic.AtomicInteger;
// Methods that must be called on the GL thread.
private void restartForceSignalEndOfStreamTimer() {
cancelForceSignalEndOfStreamTimer();
forceSignalEndOfStreamFuture =
forceEndOfStreamExecutorService.schedule(
() -> videoFrameProcessingTaskExecutor.submit(this::forceSignalEndOfStream),
SURFACE_TEXTURE_TIMEOUT_MS,
MILLISECONDS);
}
private void cancelForceSignalEndOfStreamTimer() {
if (forceSignalEndOfStreamFuture != null) {
forceSignalEndOfStreamFuture.cancel(/* mayInterruptIfRunning= */ false);
}
forceSignalEndOfStreamFuture = null;
}
private void forceSignalEndOfStream() {
// Reset because there could be further input streams after the current one ends.
Log.w(
TAG,
Util.formatInvariant(
"Forcing EOS after missing %d frames for %d ms, with available frame count: %d",
pendingFrames.size(), SURFACE_TEXTURE_TIMEOUT_MS, availableFrameCount));
// Reset because there could be further input streams after the current one ends.
currentInputStreamEnded = false;
currentFrame = null;
pendingFrames.clear();
shouldRejectIncomingFrames = true;
// Frames could be made available while waiting for OpenGL to finish processing. That is,
// time out is triggered while waiting for the downstream shader programs to process a frame,
// when there are frames available on the SurfaceTexture. This has only been observed on
// emulators.
removeAllSurfaceTextureFrames();
signalEndOfCurrentInputStream();
}
private void flush() {
// A frame that is registered before flush may arrive after flush.
numberOfFramesToDropOnBecomingAvailable = pendingFrames.size() - availableFrameCount;
while (availableFrameCount > 0) {
availableFrameCount--;
surfaceTexture.updateTexImage();
}
removeAllSurfaceTextureFrames();
externalShaderProgramInputCapacity.set(0);
currentFrame = null;
pendingFrames.clear();
maybeExecuteAfterFlushTask();
}
private void removeAllSurfaceTextureFrames() {
while (availableFrameCount > 0) {
availableFrameCount--;
surfaceTexture.updateTexImage();
}
}
private void maybeQueueFrameToExternalShaderProgram() {
if (externalShaderProgramInputCapacity.get() == 0
|| availableFrameCount == 0