loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Avoid spinning while queueing input to ExternalTextureProcessor.

Browse Source 
This change adds ExternalTextureManager which implements
InputListener to only queue input frames to the
ExternalTextureProcessor when it is ready to accept an input
frame. This replaces the old retry-logic in GlEffectsFrameProcessor.

Before this change, the retrying in GlEffectFrameProcessor wasted
CPU time if input becomes available faster than the
ExternalTextureProcessor can process it.

PiperOrigin-RevId: 467177659
This commit is contained in:
Googler 2022-08-12 11:10:47 +00:00 committed by Marc Baechinger
parent 5874327e5d
commit d7bf1ed2d7
5 changed files with 222 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java Normal file
View File

@ -0,0 +1,192 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.effect;
import android.graphics.SurfaceTexture;
import androidx.annotation.Nullable;
import androidx.annotation.WorkerThread;
import androidx.media3.common.C;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.FrameProcessor;
import androidx.media3.common.util.GlUtil;
import androidx.media3.effect.GlTextureProcessor.InputListener;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
* {@link ExternalTextureProcessor} for consumption.
*/
/* package */ class ExternalTextureManager implements InputListener {
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
private final ExternalTextureProcessor externalTextureProcessor;
private final int externalTexId;
private final SurfaceTexture surfaceTexture;
private final float[] textureTransformMatrix;
private final Queue<FrameInfo> pendingFrames;
// Incremented on any thread, decremented on the GL thread only.
private final AtomicInteger availableFrameCount;
// Incremented on any thread, decremented on the GL thread only.
private final AtomicInteger externalTextureProcessorInputCapacity;
// Set to true on any thread. Read on the GL thread only.
private volatile boolean inputStreamEnded;
// Set to null on any thread. Read and set to non-null on the GL thread only.
@Nullable private volatile FrameInfo frame;
private long previousStreamOffsetUs;
/**
* Creates a new instance.
*
* @param externalTextureProcessor The {@link ExternalTextureProcessor} for which this {@code
* ExternalTextureManager} will be set as the {@link InputListener}.
* @param frameProcessingTaskExecutor The {@link FrameProcessingTaskExecutor}.
* @throws FrameProcessingException If a problem occurs while creating the external texture.
*/
public ExternalTextureManager(
ExternalTextureProcessor externalTextureProcessor,
FrameProcessingTaskExecutor frameProcessingTaskExecutor)
throws FrameProcessingException {
this.externalTextureProcessor = externalTextureProcessor;
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
try {
externalTexId = GlUtil.createExternalTexture();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
surfaceTexture = new SurfaceTexture(externalTexId);
textureTransformMatrix = new float[16];
pendingFrames = new ConcurrentLinkedQueue<>();
availableFrameCount = new AtomicInteger();
externalTextureProcessorInputCapacity = new AtomicInteger();
previousStreamOffsetUs = C.TIME_UNSET;
}
public SurfaceTexture getSurfaceTexture() {
surfaceTexture.setOnFrameAvailableListener(
unused -> {
availableFrameCount.getAndIncrement();
frameProcessingTaskExecutor.submit(
() -> {
if (maybeUpdateFrame()) {
maybeQueueFrameToExternalTextureProcessor();
}
});
});
return surfaceTexture;
}
@Override
public void onReadyToAcceptInputFrame() {
externalTextureProcessorInputCapacity.getAndIncrement();
frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor);
}
@Override
public void onInputFrameProcessed(TextureInfo inputTexture) {
frame = null;
frameProcessingTaskExecutor.submit(
() -> {
if (maybeUpdateFrame()) {
maybeQueueFrameToExternalTextureProcessor();
}
});
}
/**
* Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will
* become available via the {@link SurfaceTexture} eventually.
*
* <p>Can be called on any thread, but the caller must ensure that frames are registered in the
* correct order.
*/
public void registerInputFrame(FrameInfo frame) {
pendingFrames.add(frame);
}
/**
* Returns the number of {@linkplain #registerInputFrame(FrameInfo) registered} frames that have
* not been rendered to the external texture yet.
*
* <p>Can be called on any thread.
*/
public int getPendingFrameCount() {
return pendingFrames.size();
}
/**
* Signals the end of the input.
*
* @see FrameProcessor#signalEndOfInput()
*/
@WorkerThread
public void signalEndOfInput() {
inputStreamEnded = true;
if (pendingFrames.isEmpty() && frame == null) {
externalTextureProcessor.signalEndOfCurrentInputStream();
}
}
public void release() {
surfaceTexture.release();
}
@WorkerThread
private boolean maybeUpdateFrame() {
if (frame != null || availableFrameCount.get() == 0) {
return false;
}
availableFrameCount.getAndDecrement();
surfaceTexture.updateTexImage();
frame = pendingFrames.remove();
return true;
}
@WorkerThread
private void maybeQueueFrameToExternalTextureProcessor() {
if (externalTextureProcessorInputCapacity.get() == 0 || frame == null) {
return;
}
FrameInfo frame = this.frame;
externalTextureProcessorInputCapacity.getAndDecrement();
surfaceTexture.getTransformMatrix(textureTransformMatrix);
externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);
long frameTimeNs = surfaceTexture.getTimestamp();
long streamOffsetUs = frame.streamOffsetUs;
if (streamOffsetUs != previousStreamOffsetUs) {
if (previousStreamOffsetUs != C.TIME_UNSET) {
externalTextureProcessor.signalEndOfCurrentInputStream();
}
previousStreamOffsetUs = streamOffsetUs;
}
// Correct for the stream offset so processors see original media presentation timestamps.
long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs;
externalTextureProcessor.queueInputFrame(
new TextureInfo(externalTexId, /* fboId= */ C.INDEX_UNSET, frame.width, frame.height),
presentationTimeUs);
if (inputStreamEnded && pendingFrames.isEmpty()) {
externalTextureProcessor.signalEndOfCurrentInputStream();
}
}
}

7
libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureProcessor.java
View File

@ -31,11 +31,4 @@ package androidx.media3.effect;
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
*/
void setTextureTransformMatrix(float[] textureTransformMatrix);
/**
* Returns whether another input frame can be {@linkplain #queueInputFrame(TextureInfo, long)
* queued}.
*/
// TODO(b/227625423): Remove this method and use the input listener instead.
boolean acceptsInputFrame();
}

17
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
View File

@ -42,8 +42,8 @@ import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import java.util.ArrayDeque;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -112,7 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
textureTransformMatrix = new float[16];
Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
streamOffsetUsQueue = new ArrayDeque<>();
streamOffsetUsQueue = new ConcurrentLinkedQueue<>();
inputListener = new InputListener() {};
}
@ -134,11 +134,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
throw new UnsupportedOperationException();
}
@Override
public boolean acceptsInputFrame() {
return true;
}
@Override
public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
checkState(!streamOffsetUsQueue.isEmpty(), "No input stream specified.");
@ -329,12 +324,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* Signals that there will be another input stream after all previously appended input streams
* have {@linkplain #signalEndOfCurrentInputStream() ended}.
*
* <p>This method does not need to be called on the GL thread, but the caller must ensure that
* stream offsets are appended in the correct order.
*
* @param streamOffsetUs The presentation timestamp offset, in microseconds.
*/
public void appendStream(long streamOffsetUs) {
streamOffsetUsQueue.add(streamOffsetUs);
}
/**
* Sets the output {@link SurfaceInfo}.
*
* @see FrameProcessor#setOutputSurfaceInfo(SurfaceInfo)
*/
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
if (outputSurfaceInfo != null

133
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
View File

@ -21,7 +21,6 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static com.google.common.collect.Iterables.getLast;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
@ -41,7 +40,6 @@ import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@ -143,11 +141,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener);
return new GlEffectsFrameProcessor(
eglDisplay,
eglContext,
frameProcessingTaskExecutor,
/* inputExternalTextureId= */ GlUtil.createExternalTexture(),
textureProcessors);
eglDisplay, eglContext, frameProcessingTaskExecutor, textureProcessors);
}
/**
@ -234,30 +228,18 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
}
}
private static final String THREAD_NAME = "Transformer:GlEffectsFrameProcessor";
private static final String THREAD_NAME = "Effect:GlThread";
private static final long RELEASE_WAIT_TIME_MS = 100;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
/** Associated with an OpenGL external texture. */
private final SurfaceTexture inputSurfaceTexture;
/** Wraps the {@link #inputSurfaceTexture}. */
private final ExternalTextureManager inputExternalTextureManager;
private final Surface inputSurface;
private final float[] inputSurfaceTextureTransformMatrix;
private final int inputExternalTextureId;
private final ExternalTextureProcessor inputExternalTextureProcessor;
private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper;
private final ImmutableList<GlTextureProcessor> allTextureProcessors;
private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames;
// Fields accessed on the thread used by the GlEffectsFrameProcessor's caller.
private @MonotonicNonNull FrameInfo nextInputFrameInfo;
// Fields accessed on the frameProcessingTaskExecutor's thread.
private boolean inputTextureInUse;
private boolean inputStreamEnded;
/**
* Offset compared to original media presentation time that has been added to incoming frame
@ -269,39 +251,41 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
EGLDisplay eglDisplay,
EGLContext eglContext,
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
int inputExternalTextureId,
ImmutableList<GlTextureProcessor> textureProcessors) {
ImmutableList<GlTextureProcessor> textureProcessors)
throws FrameProcessingException {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
this.inputExternalTextureId = inputExternalTextureId;
checkState(!textureProcessors.isEmpty());
checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor);
checkState(getLast(textureProcessors) instanceof FinalMatrixTransformationProcessorWrapper);
inputExternalTextureProcessor = (ExternalTextureProcessor) textureProcessors.get(0);
ExternalTextureProcessor inputExternalTextureProcessor =
(ExternalTextureProcessor) textureProcessors.get(0);
inputExternalTextureManager =
new ExternalTextureManager(inputExternalTextureProcessor, frameProcessingTaskExecutor);
inputExternalTextureProcessor.setInputListener(inputExternalTextureManager);
inputSurface = new Surface(inputExternalTextureManager.getSurfaceTexture());
finalTextureProcessorWrapper =
(FinalMatrixTransformationProcessorWrapper) getLast(textureProcessors);
allTextureProcessors = textureProcessors;
inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId);
inputSurface = new Surface(inputSurfaceTexture);
inputSurfaceTextureTransformMatrix = new float[16];
pendingInputFrames = new ConcurrentLinkedQueue<>();
previousStreamOffsetUs = C.TIME_UNSET;
}
@Override
public Surface getInputSurface() {
inputSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> frameProcessingTaskExecutor.submit(this::processInputFrame));
return inputSurface;
}
@Override
public void setInputFrameInfo(FrameInfo inputFrameInfo) {
nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo);
if (nextInputFrameInfo.streamOffsetUs != previousStreamOffsetUs) {
finalTextureProcessorWrapper.appendStream(nextInputFrameInfo.streamOffsetUs);
previousStreamOffsetUs = nextInputFrameInfo.streamOffsetUs;
}
}
@Override
@ -310,12 +294,12 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
checkStateNotNull(
nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
pendingInputFrames.add(nextInputFrameInfo);
inputExternalTextureManager.registerInputFrame(nextInputFrameInfo);
}
@Override
public int getPendingInputFrameCount() {
return pendingInputFrames.size();
return inputExternalTextureManager.getPendingFrameCount();
}
@Override
@ -327,7 +311,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
public void signalEndOfInput() {
checkState(!inputStreamEnded);
inputStreamEnded = true;
frameProcessingTaskExecutor.submit(this::processEndOfInputStream);
frameProcessingTaskExecutor.submit(inputExternalTextureManager::signalEndOfInput);
}
@Override
@ -340,71 +324,10 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
Thread.currentThread().interrupt();
throw new IllegalStateException(unexpected);
}
inputSurfaceTexture.release();
inputExternalTextureManager.release();
inputSurface.release();
}
/**
* Processes an input frame from the {@link #inputSurfaceTexture}.
*
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
*/
@WorkerThread
private void processInputFrame() {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
if (inputTextureInUse) {
frameProcessingTaskExecutor.submit(this::processInputFrame); // Try again later.
return;
}
inputTextureInUse = true;
inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix);
inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix);
long inputFrameTimeNs = inputSurfaceTexture.getTimestamp();
long streamOffsetUs = checkStateNotNull(pendingInputFrames.peek()).streamOffsetUs;
if (streamOffsetUs != previousStreamOffsetUs) {
if (previousStreamOffsetUs != C.TIME_UNSET) {
inputExternalTextureProcessor.signalEndOfCurrentInputStream();
}
finalTextureProcessorWrapper.appendStream(streamOffsetUs);
previousStreamOffsetUs = streamOffsetUs;
}
// Correct for the stream offset so processors see original media presentation timestamps.
long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs;
queueInputFrameToTextureProcessors(presentationTimeUs);
}
/**
* Queues the input frame to the first texture processor until it is accepted.
*
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
*/
@WorkerThread
private void queueInputFrameToTextureProcessors(long presentationTimeUs) {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
checkState(inputTextureInUse);
FrameInfo inputFrameInfo = checkStateNotNull(pendingInputFrames.peek());
if (inputExternalTextureProcessor.acceptsInputFrame()) {
inputExternalTextureProcessor.queueInputFrame(
new TextureInfo(
inputExternalTextureId,
/* fboId= */ C.INDEX_UNSET,
inputFrameInfo.width,
inputFrameInfo.height),
presentationTimeUs);
inputTextureInUse = false;
pendingInputFrames.remove();
// After the externalTextureProcessor has produced an output frame, it is processed
// asynchronously by the texture processors chained after it.
} else {
// Try again later.
frameProcessingTaskExecutor.submit(
() -> queueInputFrameToTextureProcessors(presentationTimeUs));
}
}
/**
* Expands or shrinks the frame based on the {@link FrameInfo#pixelWidthHeightRatio} and returns a
* new {@link FrameInfo} instance with scaled dimensions and {@link
@ -428,22 +351,6 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
}
}
/**
* Propagates the end-of-stream signal through the texture processors once no more input frames
* are pending.
*
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
*/
@WorkerThread
private void processEndOfInputStream() {
if (getPendingInputFrameCount() == 0) {
// Propagates the end of stream signal through the chained texture processors.
inputExternalTextureProcessor.signalEndOfCurrentInputStream();
} else {
frameProcessingTaskExecutor.submit(this::processEndOfInputStream);
}
}
/**
* Releases the {@link GlTextureProcessor} instances and destroys the OpenGL context.
*

4
libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlTextureProcessor.java
View File

@ -108,10 +108,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
this.errorListener = errorListener;
}
public final boolean acceptsInputFrame() {
return !outputTextureInUse;
}
@Override
public final void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
checkState(