loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Flush: Move TextureManager flush logic to abstract class.

Browse Source 
This also adds flush support to texture input.

PiperOrigin-RevId: 578536040
This commit is contained in:
huangdarwin 2023-11-01 09:05:43 -07:00 committed by Copybara-Service
parent 7b762642db
commit 44f317693b
4 changed files with 77 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java
View File

@ -22,7 +22,6 @@ import static androidx.media3.common.util.Assertions.checkState;
import android.graphics.Bitmap;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlObjectsProvider;
@ -43,7 +42,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>Public methods in this class can be called from any thread.
*/
@UnstableApi
/* package */ final class BitmapTextureManager implements TextureManager {
/* package */ final class BitmapTextureManager extends TextureManager {
private static final String UNSUPPORTED_IMAGE_CONFIGURATION =
"Unsupported Image Configuration: No more than 8 bits of precision should be used for each"
@ -51,7 +50,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final GlObjectsProvider glObjectsProvider;
private final GlShaderProgram shaderProgram;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
// The queue holds all bitmaps with one or more frames pending to be sent downstream.
private final Queue<BitmapFrameSequenceInfo> pendingBitmaps;
@ -61,8 +59,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private boolean currentInputStreamEnded;
private boolean isNextFrameInTexture;
@Nullable private volatile VideoFrameProcessingTaskExecutor.Task onFlushCompleteTask;
/**
* Creates a new instance.
*
@ -76,9 +72,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlObjectsProvider glObjectsProvider,
GlShaderProgram shaderProgram,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) {
super(videoFrameProcessingTaskExecutor);
this.glObjectsProvider = glObjectsProvider;
this.shaderProgram = shaderProgram;
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
pendingBitmaps = new LinkedBlockingQueue<>();
}
@ -91,11 +87,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
});
}
@Override
public void onFlush() {
videoFrameProcessingTaskExecutor.submit(this::flush);
}
@Override
public void queueInputBitmap(
Bitmap inputBitmap,
@ -129,11 +120,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
});
}
@Override
public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTaskExecutor.Task task) {
onFlushCompleteTask = task;
}
@Override
public void release() {
videoFrameProcessingTaskExecutor.submit(
@ -199,11 +185,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
private void flush() {
@Override
protected void flush() {
pendingBitmaps.clear();
if (onFlushCompleteTask != null) {
videoFrameProcessingTaskExecutor.submitWithHighPriority(onFlushCompleteTask);
}
super.flush();
}
/** Information needed to generate all the frames associated with a specific {@link Bitmap}. */

24
libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java
View File

@ -41,7 +41,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
* {@link ExternalShaderProgram} for consumption.
*/
/* package */ final class ExternalTextureManager implements TextureManager {
/* package */ final class ExternalTextureManager extends TextureManager {
private static final String TAG = "ExtTexMgr";
private static final String TIMER_THREAD_NAME = "ExtTexMgr:Timer";
@ -62,7 +62,6 @@ import java.util.concurrent.atomic.AtomicInteger;
: 500;
private final GlObjectsProvider glObjectsProvider;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final ExternalShaderProgram externalShaderProgram;
private final int externalTexId;
private final Surface surface;
@ -87,8 +86,6 @@ import java.util.concurrent.atomic.AtomicInteger;
// Set to null on any thread. Read and set to non-null on the GL thread only.
@Nullable private volatile FrameInfo currentFrame;
// TODO(b/238302341) Remove the use of after flush task, block the calling thread instead.
@Nullable private volatile VideoFrameProcessingTaskExecutor.Task onFlushCompleteTask;
@Nullable private Future<?> forceSignalEndOfStreamFuture;
// Whether to reject frames from the SurfaceTexture. Accessed only on GL thread.
@ -110,9 +107,9 @@ import java.util.concurrent.atomic.AtomicInteger;
ExternalShaderProgram externalShaderProgram,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor)
throws VideoFrameProcessingException {
super(videoFrameProcessingTaskExecutor);
this.glObjectsProvider = glObjectsProvider;
this.externalShaderProgram = externalShaderProgram;
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
try {
externalTexId = GlUtil.createExternalTexture();
} catch (GlUtil.GlException e) {
@ -187,16 +184,6 @@ import java.util.concurrent.atomic.AtomicInteger;
});
}
@Override
public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTaskExecutor.Task task) {
onFlushCompleteTask = task;
}
@Override
public void onFlush() {
videoFrameProcessingTaskExecutor.submit(this::flush);
}
/**
* Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will
* become available via the {@link SurfaceTexture} eventually.
@ -245,10 +232,10 @@ import java.util.concurrent.atomic.AtomicInteger;
}
private void maybeExecuteAfterFlushTask() {
if (onFlushCompleteTask == null || numberOfFramesToDropOnBecomingAvailable > 0) {
if (numberOfFramesToDropOnBecomingAvailable > 0) {
return;
}
videoFrameProcessingTaskExecutor.submitWithHighPriority(onFlushCompleteTask);
super.flush();
}
// Methods that must be called on the GL thread.
@ -290,7 +277,8 @@ import java.util.concurrent.atomic.AtomicInteger;
signalEndOfCurrentInputStream();
}
private void flush() {
@Override
protected void flush() {
// A frame that is registered before flush may arrive after flush.
numberOfFramesToDropOnBecomingAvailable = pendingFrames.size() - availableFrameCount;
removeAllSurfaceTextureFrames();

21
libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java
View File

@ -18,13 +18,11 @@ package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkNotNull;
import android.opengl.GLES10;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.OnInputFrameProcessedListener;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.GlUtil;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -34,8 +32,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*
* <p>Public methods in this class can be called from any thread.
*/
/* package */ final class TexIdTextureManager implements TextureManager {
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
/* package */ final class TexIdTextureManager extends TextureManager {
private final FrameConsumptionManager frameConsumptionManager;
private @MonotonicNonNull OnInputFrameProcessedListener frameProcessedListener;
@ -53,7 +50,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlObjectsProvider glObjectsProvider,
GlShaderProgram shaderProgram,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) {
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
super(videoFrameProcessingTaskExecutor);
frameConsumptionManager =
new FrameConsumptionManager(
glObjectsProvider, shaderProgram, videoFrameProcessingTaskExecutor);
@ -72,11 +69,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
.onInputFrameProcessed(inputTexture.texId, GlUtil.createGlSyncFence()));
}
@Override
public void onFlush() {
videoFrameProcessingTaskExecutor.submit(frameConsumptionManager::onFlush);
}
@Override
public void queueInputTexture(int inputTexId, long presentationTimeUs) {
FrameInfo frameInfo = checkNotNull(this.inputFrameInfo);
@ -124,12 +116,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTaskExecutor.Task task) {
public void release() {
// Do nothing.
}
// Methods that must be called on the GL thread.
@Override
public void release() throws VideoFrameProcessingException {
// Do nothing.
protected synchronized void flush() {
frameConsumptionManager.onFlush();
super.flush();
}
}

71
libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java
View File

@ -20,6 +20,7 @@ package androidx.media3.effect;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.annotation.GuardedBy;
import androidx.annotation.Nullable;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.OnInputFrameProcessedListener;
@ -27,15 +28,39 @@ import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.TimestampIterator;
/** Handles {@code DefaultVideoFrameProcessor}'s input. */
/* package */ interface TextureManager extends GlShaderProgram.InputListener {
/**
* Handles {@code DefaultVideoFrameProcessor}'s input.
*
* <p>All instance methods must be called from either the thread that owns {@code this} instance, or
* an internal GL thread.
*/
/* package */ abstract class TextureManager implements GlShaderProgram.InputListener {
protected final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final Object lock;
// TODO(b/238302341) Remove the use of onFlushCompleteTask, block the calling thread instead.
@GuardedBy("lock")
@Nullable
private VideoFrameProcessingTaskExecutor.Task onFlushCompleteTask;
/**
* Creates a new instance.
*
* @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor}.
*/
public TextureManager(VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) {
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
lock = new Object();
}
/**
* See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}.
*
* <p>Only works when the input is received on a {@link SurfaceTexture}.
*/
default void setDefaultBufferSize(int width, int height) {
public void setDefaultBufferSize(int width, int height) {
throw new UnsupportedOperationException();
}
@ -48,7 +73,7 @@ import androidx.media3.common.util.TimestampIterator;
* at. The timestamps should be monotonically increasing.
* @param useHdr Whether input and/or output colors are HDR.
*/
default void queueInputBitmap(
public void queueInputBitmap(
Bitmap inputBitmap,
FrameInfo frameInfo,
TimestampIterator inStreamOffsetsUs,
@ -61,7 +86,7 @@ import androidx.media3.common.util.TimestampIterator;
*
* @see VideoFrameProcessor#queueInputTexture
*/
default void queueInputTexture(int inputTexId, long presentationTimeUs) {
public void queueInputTexture(int inputTexId, long presentationTimeUs) {
throw new UnsupportedOperationException();
}
@ -70,7 +95,7 @@ import androidx.media3.common.util.TimestampIterator;
*
* @see VideoFrameProcessor#setOnInputFrameProcessedListener
*/
default void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener) {
public void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener) {
throw new UnsupportedOperationException();
}
@ -83,7 +108,7 @@ import androidx.media3.common.util.TimestampIterator;
* <p>Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output
* frames' pixels have a ratio of 1.
*/
default void setInputFrameInfo(FrameInfo inputFrameInfo) {
public void setInputFrameInfo(FrameInfo inputFrameInfo) {
// Do nothing.
}
@ -92,28 +117,48 @@ import androidx.media3.common.util.TimestampIterator;
*
* <p>Only works when the input is received on a {@link SurfaceTexture}.
*/
default Surface getInputSurface() {
public Surface getInputSurface() {
throw new UnsupportedOperationException();
}
/** Informs the {@code TextureManager} that a frame will be queued. */
default void registerInputFrame(FrameInfo frameInfo) {
public void registerInputFrame(FrameInfo frameInfo) {
throw new UnsupportedOperationException();
}
/** See {@link VideoFrameProcessor#getPendingInputFrameCount}. */
int getPendingFrameCount();
public abstract int getPendingFrameCount();
/** Signals the end of the current input stream. */
void signalEndOfCurrentInputStream();
public abstract void signalEndOfCurrentInputStream();
/** Sets the task to run on completing flushing, or {@code null} to clear any task. */
void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTaskExecutor.Task task);
public final void setOnFlushCompleteListener(
@Nullable VideoFrameProcessingTaskExecutor.Task task) {
synchronized (lock) {
onFlushCompleteTask = task;
}
}
@Override
public final void onFlush() {
videoFrameProcessingTaskExecutor.submit(this::flush);
}
/**
* Releases all resources.
*
* @see VideoFrameProcessor#release()
*/
void release() throws VideoFrameProcessingException;
public abstract void release() throws VideoFrameProcessingException;
// Methods that must be called on the GL thread.
protected void flush() {
synchronized (lock) {
if (onFlushCompleteTask != null) {
videoFrameProcessingTaskExecutor.submitWithHighPriority(onFlushCompleteTask);
}
}
}
}