diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java new file mode 100644 index 0000000000..6f01fb082b --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -0,0 +1,192 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.graphics.SurfaceTexture; +import androidx.annotation.Nullable; +import androidx.annotation.WorkerThread; +import androidx.media3.common.C; +import androidx.media3.common.FrameInfo; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.FrameProcessor; +import androidx.media3.common.util.GlUtil; +import androidx.media3.effect.GlTextureProcessor.InputListener; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Forwards externally produced frames that become available via a {@link SurfaceTexture} to an + * {@link ExternalTextureProcessor} for consumption. + */ +/* package */ class ExternalTextureManager implements InputListener { + + private final FrameProcessingTaskExecutor frameProcessingTaskExecutor; + private final ExternalTextureProcessor externalTextureProcessor; + private final int externalTexId; + private final SurfaceTexture surfaceTexture; + private final float[] textureTransformMatrix; + private final Queue pendingFrames; + + // Incremented on any thread, decremented on the GL thread only. + private final AtomicInteger availableFrameCount; + // Incremented on any thread, decremented on the GL thread only. + private final AtomicInteger externalTextureProcessorInputCapacity; + + // Set to true on any thread. Read on the GL thread only. + private volatile boolean inputStreamEnded; + // Set to null on any thread. Read and set to non-null on the GL thread only. + @Nullable private volatile FrameInfo frame; + + private long previousStreamOffsetUs; + + /** + * Creates a new instance. + * + * @param externalTextureProcessor The {@link ExternalTextureProcessor} for which this {@code + * ExternalTextureManager} will be set as the {@link InputListener}. + * @param frameProcessingTaskExecutor The {@link FrameProcessingTaskExecutor}. + * @throws FrameProcessingException If a problem occurs while creating the external texture. + */ + public ExternalTextureManager( + ExternalTextureProcessor externalTextureProcessor, + FrameProcessingTaskExecutor frameProcessingTaskExecutor) + throws FrameProcessingException { + this.externalTextureProcessor = externalTextureProcessor; + this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; + try { + externalTexId = GlUtil.createExternalTexture(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + surfaceTexture = new SurfaceTexture(externalTexId); + textureTransformMatrix = new float[16]; + pendingFrames = new ConcurrentLinkedQueue<>(); + availableFrameCount = new AtomicInteger(); + externalTextureProcessorInputCapacity = new AtomicInteger(); + previousStreamOffsetUs = C.TIME_UNSET; + } + + public SurfaceTexture getSurfaceTexture() { + surfaceTexture.setOnFrameAvailableListener( + unused -> { + availableFrameCount.getAndIncrement(); + frameProcessingTaskExecutor.submit( + () -> { + if (maybeUpdateFrame()) { + maybeQueueFrameToExternalTextureProcessor(); + } + }); + }); + return surfaceTexture; + } + + @Override + public void onReadyToAcceptInputFrame() { + externalTextureProcessorInputCapacity.getAndIncrement(); + frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor); + } + + @Override + public void onInputFrameProcessed(TextureInfo inputTexture) { + frame = null; + frameProcessingTaskExecutor.submit( + () -> { + if (maybeUpdateFrame()) { + maybeQueueFrameToExternalTextureProcessor(); + } + }); + } + + /** + * Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will + * become available via the {@link SurfaceTexture} eventually. + * + *

Can be called on any thread, but the caller must ensure that frames are registered in the + * correct order. + */ + public void registerInputFrame(FrameInfo frame) { + pendingFrames.add(frame); + } + + /** + * Returns the number of {@linkplain #registerInputFrame(FrameInfo) registered} frames that have + * not been rendered to the external texture yet. + * + *

Can be called on any thread. + */ + public int getPendingFrameCount() { + return pendingFrames.size(); + } + + /** + * Signals the end of the input. + * + * @see FrameProcessor#signalEndOfInput() + */ + @WorkerThread + public void signalEndOfInput() { + inputStreamEnded = true; + if (pendingFrames.isEmpty() && frame == null) { + externalTextureProcessor.signalEndOfCurrentInputStream(); + } + } + + public void release() { + surfaceTexture.release(); + } + + @WorkerThread + private boolean maybeUpdateFrame() { + if (frame != null || availableFrameCount.get() == 0) { + return false; + } + + availableFrameCount.getAndDecrement(); + surfaceTexture.updateTexImage(); + frame = pendingFrames.remove(); + return true; + } + + @WorkerThread + private void maybeQueueFrameToExternalTextureProcessor() { + if (externalTextureProcessorInputCapacity.get() == 0 || frame == null) { + return; + } + + FrameInfo frame = this.frame; + externalTextureProcessorInputCapacity.getAndDecrement(); + surfaceTexture.getTransformMatrix(textureTransformMatrix); + externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix); + long frameTimeNs = surfaceTexture.getTimestamp(); + long streamOffsetUs = frame.streamOffsetUs; + if (streamOffsetUs != previousStreamOffsetUs) { + if (previousStreamOffsetUs != C.TIME_UNSET) { + externalTextureProcessor.signalEndOfCurrentInputStream(); + } + previousStreamOffsetUs = streamOffsetUs; + } + // Correct for the stream offset so processors see original media presentation timestamps. + long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs; + externalTextureProcessor.queueInputFrame( + new TextureInfo(externalTexId, /* fboId= */ C.INDEX_UNSET, frame.width, frame.height), + presentationTimeUs); + + if (inputStreamEnded && pendingFrames.isEmpty()) { + externalTextureProcessor.signalEndOfCurrentInputStream(); + } + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureProcessor.java index 19b5cbd008..16194da5f3 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureProcessor.java @@ -31,11 +31,4 @@ package androidx.media3.effect; * android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}. */ void setTextureTransformMatrix(float[] textureTransformMatrix); - - /** - * Returns whether another input frame can be {@linkplain #queueInputFrame(TextureInfo, long) - * queued}. - */ - // TODO(b/227625423): Remove this method and use the input listener instead. - boolean acceptsInputFrame(); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java index 23a5fa0972..e0fd531987 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java @@ -42,8 +42,8 @@ import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; -import java.util.ArrayDeque; import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -112,7 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; textureTransformMatrix = new float[16]; Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0); - streamOffsetUsQueue = new ArrayDeque<>(); + streamOffsetUsQueue = new ConcurrentLinkedQueue<>(); inputListener = new InputListener() {}; } @@ -134,11 +134,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new UnsupportedOperationException(); } - @Override - public boolean acceptsInputFrame() { - return true; - } - @Override public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { checkState(!streamOffsetUsQueue.isEmpty(), "No input stream specified."); @@ -329,12 +324,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * Signals that there will be another input stream after all previously appended input streams * have {@linkplain #signalEndOfCurrentInputStream() ended}. * + *

This method does not need to be called on the GL thread, but the caller must ensure that + * stream offsets are appended in the correct order. + * * @param streamOffsetUs The presentation timestamp offset, in microseconds. */ public void appendStream(long streamOffsetUs) { streamOffsetUsQueue.add(streamOffsetUs); } + /** + * Sets the output {@link SurfaceInfo}. + * + * @see FrameProcessor#setOutputSurfaceInfo(SurfaceInfo) + */ public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) { if (outputSurfaceInfo != null diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java index 87ccbacfb8..a74033d1c8 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java @@ -21,7 +21,6 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import static com.google.common.collect.Iterables.getLast; import android.content.Context; -import android.graphics.SurfaceTexture; import android.opengl.EGL14; import android.opengl.EGLContext; import android.opengl.EGLDisplay; @@ -41,7 +40,6 @@ import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import java.util.List; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -143,11 +141,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener); return new GlEffectsFrameProcessor( - eglDisplay, - eglContext, - frameProcessingTaskExecutor, - /* inputExternalTextureId= */ GlUtil.createExternalTexture(), - textureProcessors); + eglDisplay, eglContext, frameProcessingTaskExecutor, textureProcessors); } /** @@ -234,30 +228,18 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { } } - private static final String THREAD_NAME = "Transformer:GlEffectsFrameProcessor"; + private static final String THREAD_NAME = "Effect:GlThread"; private static final long RELEASE_WAIT_TIME_MS = 100; private final EGLDisplay eglDisplay; private final EGLContext eglContext; private final FrameProcessingTaskExecutor frameProcessingTaskExecutor; - - /** Associated with an OpenGL external texture. */ - private final SurfaceTexture inputSurfaceTexture; - /** Wraps the {@link #inputSurfaceTexture}. */ + private final ExternalTextureManager inputExternalTextureManager; private final Surface inputSurface; - - private final float[] inputSurfaceTextureTransformMatrix; - private final int inputExternalTextureId; - private final ExternalTextureProcessor inputExternalTextureProcessor; private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper; private final ImmutableList allTextureProcessors; - private final ConcurrentLinkedQueue pendingInputFrames; - // Fields accessed on the thread used by the GlEffectsFrameProcessor's caller. private @MonotonicNonNull FrameInfo nextInputFrameInfo; - - // Fields accessed on the frameProcessingTaskExecutor's thread. - private boolean inputTextureInUse; private boolean inputStreamEnded; /** * Offset compared to original media presentation time that has been added to incoming frame @@ -269,39 +251,41 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { EGLDisplay eglDisplay, EGLContext eglContext, FrameProcessingTaskExecutor frameProcessingTaskExecutor, - int inputExternalTextureId, - ImmutableList textureProcessors) { + ImmutableList textureProcessors) + throws FrameProcessingException { this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; - this.inputExternalTextureId = inputExternalTextureId; checkState(!textureProcessors.isEmpty()); checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor); checkState(getLast(textureProcessors) instanceof FinalMatrixTransformationProcessorWrapper); - inputExternalTextureProcessor = (ExternalTextureProcessor) textureProcessors.get(0); + ExternalTextureProcessor inputExternalTextureProcessor = + (ExternalTextureProcessor) textureProcessors.get(0); + inputExternalTextureManager = + new ExternalTextureManager(inputExternalTextureProcessor, frameProcessingTaskExecutor); + inputExternalTextureProcessor.setInputListener(inputExternalTextureManager); + inputSurface = new Surface(inputExternalTextureManager.getSurfaceTexture()); finalTextureProcessorWrapper = (FinalMatrixTransformationProcessorWrapper) getLast(textureProcessors); allTextureProcessors = textureProcessors; - - inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId); - inputSurface = new Surface(inputSurfaceTexture); - inputSurfaceTextureTransformMatrix = new float[16]; - pendingInputFrames = new ConcurrentLinkedQueue<>(); previousStreamOffsetUs = C.TIME_UNSET; } @Override public Surface getInputSurface() { - inputSurfaceTexture.setOnFrameAvailableListener( - surfaceTexture -> frameProcessingTaskExecutor.submit(this::processInputFrame)); return inputSurface; } @Override public void setInputFrameInfo(FrameInfo inputFrameInfo) { nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo); + + if (nextInputFrameInfo.streamOffsetUs != previousStreamOffsetUs) { + finalTextureProcessorWrapper.appendStream(nextInputFrameInfo.streamOffsetUs); + previousStreamOffsetUs = nextInputFrameInfo.streamOffsetUs; + } } @Override @@ -310,12 +294,12 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { checkStateNotNull( nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames"); - pendingInputFrames.add(nextInputFrameInfo); + inputExternalTextureManager.registerInputFrame(nextInputFrameInfo); } @Override public int getPendingInputFrameCount() { - return pendingInputFrames.size(); + return inputExternalTextureManager.getPendingFrameCount(); } @Override @@ -327,7 +311,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { public void signalEndOfInput() { checkState(!inputStreamEnded); inputStreamEnded = true; - frameProcessingTaskExecutor.submit(this::processEndOfInputStream); + frameProcessingTaskExecutor.submit(inputExternalTextureManager::signalEndOfInput); } @Override @@ -340,71 +324,10 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { Thread.currentThread().interrupt(); throw new IllegalStateException(unexpected); } - inputSurfaceTexture.release(); + inputExternalTextureManager.release(); inputSurface.release(); } - /** - * Processes an input frame from the {@link #inputSurfaceTexture}. - * - *

This method must be called on the {@linkplain #THREAD_NAME background thread}. - */ - @WorkerThread - private void processInputFrame() { - checkState(Thread.currentThread().getName().equals(THREAD_NAME)); - if (inputTextureInUse) { - frameProcessingTaskExecutor.submit(this::processInputFrame); // Try again later. - return; - } - - inputTextureInUse = true; - inputSurfaceTexture.updateTexImage(); - inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix); - inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix); - long inputFrameTimeNs = inputSurfaceTexture.getTimestamp(); - long streamOffsetUs = checkStateNotNull(pendingInputFrames.peek()).streamOffsetUs; - if (streamOffsetUs != previousStreamOffsetUs) { - if (previousStreamOffsetUs != C.TIME_UNSET) { - inputExternalTextureProcessor.signalEndOfCurrentInputStream(); - } - finalTextureProcessorWrapper.appendStream(streamOffsetUs); - previousStreamOffsetUs = streamOffsetUs; - } - // Correct for the stream offset so processors see original media presentation timestamps. - long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs; - queueInputFrameToTextureProcessors(presentationTimeUs); - } - - /** - * Queues the input frame to the first texture processor until it is accepted. - * - *

This method must be called on the {@linkplain #THREAD_NAME background thread}. - */ - @WorkerThread - private void queueInputFrameToTextureProcessors(long presentationTimeUs) { - checkState(Thread.currentThread().getName().equals(THREAD_NAME)); - checkState(inputTextureInUse); - - FrameInfo inputFrameInfo = checkStateNotNull(pendingInputFrames.peek()); - if (inputExternalTextureProcessor.acceptsInputFrame()) { - inputExternalTextureProcessor.queueInputFrame( - new TextureInfo( - inputExternalTextureId, - /* fboId= */ C.INDEX_UNSET, - inputFrameInfo.width, - inputFrameInfo.height), - presentationTimeUs); - inputTextureInUse = false; - pendingInputFrames.remove(); - // After the externalTextureProcessor has produced an output frame, it is processed - // asynchronously by the texture processors chained after it. - } else { - // Try again later. - frameProcessingTaskExecutor.submit( - () -> queueInputFrameToTextureProcessors(presentationTimeUs)); - } - } - /** * Expands or shrinks the frame based on the {@link FrameInfo#pixelWidthHeightRatio} and returns a * new {@link FrameInfo} instance with scaled dimensions and {@link @@ -428,22 +351,6 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { } } - /** - * Propagates the end-of-stream signal through the texture processors once no more input frames - * are pending. - * - *

This method must be called on the {@linkplain #THREAD_NAME background thread}. - */ - @WorkerThread - private void processEndOfInputStream() { - if (getPendingInputFrameCount() == 0) { - // Propagates the end of stream signal through the chained texture processors. - inputExternalTextureProcessor.signalEndOfCurrentInputStream(); - } else { - frameProcessingTaskExecutor.submit(this::processEndOfInputStream); - } - } - /** * Releases the {@link GlTextureProcessor} instances and destroys the OpenGL context. * diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlTextureProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlTextureProcessor.java index 593fe345e2..fee83ad9c1 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlTextureProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlTextureProcessor.java @@ -108,10 +108,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso this.errorListener = errorListener; } - public final boolean acceptsInputFrame() { - return !outputTextureInUse; - } - @Override public final void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { checkState(