diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index d8a5d1136e..86fec35cae 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -208,11 +208,8 @@ public interface VideoFrameProcessor { void registerInputFrame(); /** - * Returns the number of input frames that have been {@linkplain #registerInputFrame() registered} - * but not processed off the {@linkplain #getInputSurface() input surface} yet. - * - *

This method must only be called when the {@link VideoFrameProcessor} is {@linkplain - * Factory#create created} with {@link #INPUT_TYPE_SURFACE}. + * returns the number of input frames that have been made available to the {@code + * VideoFrameProcessor} but have not been processed yet. * *

Can be called on any thread. */ diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java index 9a239a735c..720ade53be 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java @@ -21,10 +21,12 @@ import static java.lang.Math.round; import android.graphics.Bitmap; import android.opengl.GLES20; import android.opengl.GLUtils; +import android.view.Surface; +import androidx.annotation.Nullable; import androidx.media3.common.C; +import androidx.media3.common.FrameInfo; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; import java.util.Queue; @@ -38,7 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; *

Public methods in this class can be called from any thread. */ @UnstableApi -/* package */ final class BitmapTextureManager implements GlShaderProgram.InputListener { +/* package */ final class BitmapTextureManager implements InputHandler { private final GlShaderProgram shaderProgram; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; // The queue holds all bitmaps with one or more frames pending to be sent downstream. @@ -77,22 +79,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; }); } - /** - * Provides an input {@link Bitmap} to put into the video frames. - * - * @see VideoFrameProcessor#queueInputBitmap - */ + @Override + public void setDefaultBufferSize(int width, int height) { + throw new UnsupportedOperationException(); + } + + @Override public void queueInputBitmap( Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) { videoFrameProcessingTaskExecutor.submit( () -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr)); } - /** - * Signals the end of the input. - * - * @see VideoFrameProcessor#signalEndOfInput() - */ + @Override + public Surface getInputSurface() { + throw new UnsupportedOperationException(); + } + + @Override + public void registerInputFrame(FrameInfo frameInfo) { + // Do nothing. + } + + @Override + public int getPendingFrameCount() { + // Always treat all queued bitmaps as immediately processed. + return 0; + } + + @Override public void signalEndOfInput() { videoFrameProcessingTaskExecutor.submit( () -> { @@ -101,11 +116,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; }); } - /** - * Releases all resources. - * - * @see VideoFrameProcessor#release() - */ + @Override + public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) { + // Do nothing. + } + + @Override public void release() { videoFrameProcessingTaskExecutor.submit( () -> { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index eb783fe19d..5dbec3d81b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -17,7 +17,6 @@ package androidx.media3.effect; import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE; import static androidx.media3.common.util.Assertions.checkArgument; -import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static com.google.common.collect.Iterables.getLast; @@ -247,8 +246,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final EGLDisplay eglDisplay; private final EGLContext eglContext; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; - private @MonotonicNonNull BitmapTextureManager inputBitmapTextureManager; - private @MonotonicNonNull ExternalTextureManager inputExternalTextureManager; + private final InputHandler inputHandler; private final boolean releaseFramesAutomatically; private final FinalShaderProgramWrapper finalShaderProgramWrapper; private final ImmutableList allShaderPrograms; @@ -278,20 +276,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { switch (inputType) { case VideoFrameProcessor.INPUT_TYPE_SURFACE: checkState(inputShaderProgram instanceof ExternalShaderProgram); - inputExternalTextureManager = + inputHandler = new ExternalTextureManager( (ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor); - inputShaderProgram.setInputListener(inputExternalTextureManager); break; case VideoFrameProcessor.INPUT_TYPE_BITMAP: - inputBitmapTextureManager = + inputHandler = new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor); - inputShaderProgram.setInputListener(inputBitmapTextureManager); break; case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through default: throw new VideoFrameProcessingException("Input type not supported yet"); } + inputShaderProgram.setInputListener(inputHandler); finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(shaderPrograms); allShaderPrograms = shaderPrograms; @@ -319,18 +316,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * @param height The default height for input buffers, in pixels. */ public void setInputDefaultBufferSize(int width, int height) { - checkNotNull(inputExternalTextureManager).setDefaultBufferSize(width, height); + inputHandler.setDefaultBufferSize(width, height); } @Override public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { - checkNotNull(inputBitmapTextureManager) - .queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false); + inputHandler.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false); } @Override public Surface getInputSurface() { - return checkNotNull(inputExternalTextureManager).getInputSurface(); + return inputHandler.getInputSurface(); } @Override @@ -344,12 +340,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { checkStateNotNull( nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames"); - checkNotNull(inputExternalTextureManager).registerInputFrame(nextInputFrameInfo); + inputHandler.registerInputFrame(nextInputFrameInfo); } @Override public int getPendingInputFrameCount() { - return checkNotNull(inputExternalTextureManager).getPendingFrameCount(); + return inputHandler.getPendingFrameCount(); } @Override @@ -370,12 +366,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { public void signalEndOfInput() { checkState(!inputStreamEnded); inputStreamEnded = true; - if (inputBitmapTextureManager != null) { - videoFrameProcessingTaskExecutor.submit(inputBitmapTextureManager::signalEndOfInput); - } - if (inputExternalTextureManager != null) { - videoFrameProcessingTaskExecutor.submit(inputExternalTextureManager::signalEndOfInput); - } + videoFrameProcessingTaskExecutor.submit(inputHandler::signalEndOfInput); } @Override @@ -383,10 +374,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { try { videoFrameProcessingTaskExecutor.flush(); CountDownLatch latch = new CountDownLatch(1); - checkNotNull(inputExternalTextureManager).setOnFlushCompleteListener(latch::countDown); + inputHandler.setOnFlushCompleteListener(latch::countDown); videoFrameProcessingTaskExecutor.submit(finalShaderProgramWrapper::flush); latch.await(); - inputExternalTextureManager.setOnFlushCompleteListener(null); + inputHandler.setOnFlushCompleteListener(null); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } @@ -401,12 +392,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { Thread.currentThread().interrupt(); throw new IllegalStateException(unexpected); } - if (inputExternalTextureManager != null) { - inputExternalTextureManager.release(); - } - if (inputBitmapTextureManager != null) { - inputBitmapTextureManager.release(); - } + inputHandler.release(); } /** diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index b99eb2ea20..bc3702ad53 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -17,6 +17,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkStateNotNull; +import android.graphics.Bitmap; import android.graphics.SurfaceTexture; import android.view.Surface; import androidx.annotation.Nullable; @@ -24,7 +25,6 @@ import androidx.media3.common.C; import androidx.media3.common.FrameInfo; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.GlUtil; import androidx.media3.effect.GlShaderProgram.InputListener; import java.util.Queue; @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; * Forwards externally produced frames that become available via a {@link SurfaceTexture} to an * {@link ExternalShaderProgram} for consumption. */ -/* package */ final class ExternalTextureManager implements InputListener { +/* package */ final class ExternalTextureManager implements InputHandler { private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; private final ExternalShaderProgram externalShaderProgram; @@ -105,12 +105,18 @@ import java.util.concurrent.atomic.AtomicInteger; surface = new Surface(surfaceTexture); } - /** See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. */ + @Override public void setDefaultBufferSize(int width, int height) { surfaceTexture.setDefaultBufferSize(width, height); } - /** Returns the {@linkplain Surface input surface} that wraps the external texture. */ + @Override + public void queueInputBitmap( + Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) { + throw new UnsupportedOperationException(); + } + + @Override public Surface getInputSurface() { return surface; } @@ -137,7 +143,7 @@ import java.util.concurrent.atomic.AtomicInteger; }); } - /** Sets the task to run on completing flushing, or {@code null} to clear any task. */ + @Override public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) { onFlushCompleteTask = task; } @@ -154,6 +160,7 @@ import java.util.concurrent.atomic.AtomicInteger; *

Can be called on any thread. The caller must ensure that frames are registered in the * correct order. */ + @Override public void registerInputFrame(FrameInfo frame) { pendingFrames.add(frame); } @@ -164,15 +171,12 @@ import java.util.concurrent.atomic.AtomicInteger; * *

Can be called on any thread. */ + @Override public int getPendingFrameCount() { return pendingFrames.size(); } - /** - * Signals the end of the input. - * - * @see VideoFrameProcessor#signalEndOfInput() - */ + @Override public void signalEndOfInput() { videoFrameProcessingTaskExecutor.submit( () -> { @@ -183,11 +187,7 @@ import java.util.concurrent.atomic.AtomicInteger; }); } - /** - * Releases all resources. - * - * @see VideoFrameProcessor#release() - */ + @Override public void release() { surfaceTexture.release(); surface.release(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java b/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java new file mode 100644 index 0000000000..ed9a7d4e97 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java @@ -0,0 +1,73 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package androidx.media3.effect; + +import android.graphics.Bitmap; +import android.graphics.SurfaceTexture; +import android.view.Surface; +import androidx.annotation.Nullable; +import androidx.media3.common.FrameInfo; +import androidx.media3.common.VideoFrameProcessor; + +/** A component that handles {@code DefaultVideoFrameProcessor}'s input. */ +/* package */ interface InputHandler extends GlShaderProgram.InputListener { + + /** + * See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. + * + *

Only works when the input is received on a {@link SurfaceTexture}. + */ + void setDefaultBufferSize(int width, int height); + + /** + * Provides an input {@link Bitmap} to put into the video frames. + * + * @see VideoFrameProcessor#queueInputBitmap + */ + void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr); + + /** + * See {@link VideoFrameProcessor#getInputSurface}. + * + *

Only works when the input is received on a {@link SurfaceTexture}. + */ + Surface getInputSurface(); + + /** Informs the {@code InputHandler} that a frame will be queued. */ + void registerInputFrame(FrameInfo frameInfo); + + /** See {@link VideoFrameProcessor#getPendingInputFrameCount}. */ + int getPendingFrameCount(); + + /** + * Signals the end of the input. + * + * @see VideoFrameProcessor#signalEndOfInput() + */ + void signalEndOfInput(); + + /** Sets the task to run on completing flushing, or {@code null} to clear any task. */ + void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task); + + /** + * Releases all resources. + * + * @see VideoFrameProcessor#release() + */ + void release(); +}