The input source must be able to have at least two {@linkplain #queueInputTexture queued - * textures} before one texture is {@linkplain + *
The input source must be able to have at least two {@linkplain + * VideoCompositor#queueInputTexture queued textures} before one texture is {@linkplain * DefaultVideoFrameProcessor.ReleaseOutputTextureCallback released}. * *
When composited, textures are drawn in the reverse order of their registration order, so @@ -173,14 +173,14 @@ public final class DefaultVideoCompositor implements VideoCompositor { @Override public synchronized void queueInputTexture( int inputId, + GlTextureProducer textureProducer, GlTextureInfo inputTexture, - long presentationTimeUs, - DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseTextureCallback) { + long presentationTimeUs) { InputSource inputSource = inputSources.get(inputId); checkState(!inputSource.isInputEnded); InputFrameInfo inputFrameInfo = - new InputFrameInfo(inputTexture, presentationTimeUs, releaseTextureCallback); + new InputFrameInfo(textureProducer, inputTexture, presentationTimeUs); inputSource.frameInfos.add(inputFrameInfo); if (inputId == PRIMARY_INPUT_ID) { @@ -202,6 +202,11 @@ public final class DefaultVideoCompositor implements VideoCompositor { } } + @Override + public void releaseOutputTexture(long presentationTimeUs) { + videoFrameProcessingTaskExecutor.submit(() -> releaseOutputTextureInternal(presentationTimeUs)); + } + private synchronized void releaseExcessFramesInAllSecondaryStreams() { for (int i = 0; i < inputSources.size(); i++) { if (i == PRIMARY_INPUT_ID) { @@ -248,7 +253,8 @@ public final class DefaultVideoCompositor implements VideoCompositor { private synchronized void releaseFrames(InputSource inputSource, int numberOfFramesToRelease) { for (int i = 0; i < numberOfFramesToRelease; i++) { InputFrameInfo frameInfoToRelease = inputSource.frameInfos.remove(); - frameInfoToRelease.releaseCallback.release(frameInfoToRelease.presentationTimeUs); + frameInfoToRelease.textureProducer.releaseOutputTexture( + frameInfoToRelease.presentationTimeUs); } } @@ -290,10 +296,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { long syncObject = GlUtil.createGlSyncFence(); syncObjects.add(syncObject); textureOutputListener.onTextureRendered( - outputTexture, - /* presentationTimeUs= */ outputPresentationTimestampUs, - this::releaseOutputFrame, - syncObject); + /* textureProducer= */ this, outputTexture, outputPresentationTimestampUs, syncObject); InputSource primaryInputSource = inputSources.get(PRIMARY_INPUT_ID); releaseFrames(primaryInputSource, /* numberOfFramesToRelease= */ 1); @@ -368,11 +371,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { return framesToCompositeList; } - private void releaseOutputFrame(long presentationTimeUs) { - videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs)); - } - - private synchronized void releaseOutputFrameInternal(long presentationTimeUs) + private synchronized void releaseOutputTextureInternal(long presentationTimeUs) throws VideoFrameProcessingException, GlUtil.GlException { while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity() && outputTextureTimestamps.element() <= presentationTimeUs) { @@ -478,17 +477,15 @@ public final class DefaultVideoCompositor implements VideoCompositor { /** Holds information on a frame and how to release it. */ private static final class InputFrameInfo { + public final GlTextureProducer textureProducer; public final GlTextureInfo texture; public final long presentationTimeUs; - public final DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseCallback; public InputFrameInfo( - GlTextureInfo texture, - long presentationTimeUs, - DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseCallback) { + GlTextureProducer textureProducer, GlTextureInfo texture, long presentationTimeUs) { + this.textureProducer = textureProducer; this.texture = texture; this.presentationTimeUs = presentationTimeUs; - this.releaseCallback = releaseCallback; } } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 152342c067..0e5762a956 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -44,7 +44,6 @@ import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.FrameInfo; import androidx.media3.common.GlObjectsProvider; -import androidx.media3.common.GlTextureInfo; import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; @@ -74,28 +73,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @UnstableApi public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { - /** Listener interface for texture output. */ - @VisibleForTesting(otherwise = PACKAGE_PRIVATE) - public interface TextureOutputListener { - /** - * Called when a texture has been rendered to. - * - * @param outputTexture The texture that has been rendered. - * @param presentationTimeUs The presentation time of the texture. - * @param releaseOutputTextureCallback A {@link ReleaseOutputTextureCallback} that must be - * called to release the {@link GlTextureInfo}. - * @param syncObject A GL sync object that has been inserted into the GL command stream after - * the last write of the {@code outputTexture}. Value is 0 if and only if the {@link - * GLES30#glFenceSync} failed. - */ - void onTextureRendered( - GlTextureInfo outputTexture, - long presentationTimeUs, - ReleaseOutputTextureCallback releaseOutputTextureCallback, - long syncObject) - throws VideoFrameProcessingException, GlUtil.GlException; - } - /** * Releases the output information stored for textures before and at {@code presentationTimeUs}. */ @@ -112,7 +89,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private boolean enableColorTransfers; private @MonotonicNonNull GlObjectsProvider glObjectsProvider; private @MonotonicNonNull ExecutorService executorService; - private @MonotonicNonNull TextureOutputListener textureOutputListener; + private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener; private int textureOutputCapacity; /** Creates an instance. */ @@ -165,8 +142,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * Sets texture output settings. * *
If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via - * {@link TextureOutputListener#onTextureRendered}. Textures will stop being outputted when - * the number of output textures available reaches the {@code textureOutputCapacity}. To + * {@link GlTextureProducer.Listener#onTextureRendered}. Textures will stop being outputted + * when the number of output textures available reaches the {@code textureOutputCapacity}. To * regain capacity, output textures must be released using {@link * ReleaseOutputTextureCallback}. * @@ -175,14 +152,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * *
If not set, there will be no texture output.
*
- * @param textureOutputListener The {@link TextureOutputListener}.
+ * @param textureOutputListener The {@link GlTextureProducer.Listener}.
* @param textureOutputCapacity The amount of output textures that may be allocated at a time
* before texture output blocks. Must be greater than or equal to 1.
*/
@VisibleForTesting
@CanIgnoreReturnValue
public Builder setTextureOutput(
- TextureOutputListener textureOutputListener,
+ GlTextureProducer.Listener textureOutputListener,
@IntRange(from = 1) int textureOutputCapacity) {
this.textureOutputListener = textureOutputListener;
checkArgument(textureOutputCapacity >= 1);
@@ -204,14 +181,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private final boolean enableColorTransfers;
private final GlObjectsProvider glObjectsProvider;
@Nullable private final ExecutorService executorService;
- @Nullable private final TextureOutputListener textureOutputListener;
+ @Nullable private final GlTextureProducer.Listener textureOutputListener;
private final int textureOutputCapacity;
private Factory(
boolean enableColorTransfers,
GlObjectsProvider glObjectsProvider,
@Nullable ExecutorService executorService,
- @Nullable TextureOutputListener textureOutputListener,
+ @Nullable GlTextureProducer.Listener textureOutputListener,
int textureOutputCapacity) {
this.enableColorTransfers = enableColorTransfers;
this.glObjectsProvider = glObjectsProvider;
@@ -262,7 +239,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo outputColorInfo,
boolean renderFramesAutomatically,
Executor listenerExecutor,
- Listener listener)
+ VideoFrameProcessor.Listener listener)
throws VideoFrameProcessingException {
// TODO(b/261188041) Add tests to verify the Listener is invoked on the given Executor.
@@ -363,7 +340,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
EGLContext eglContext,
InputSwitcher inputSwitcher,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
- Listener listener,
+ VideoFrameProcessor.Listener listener,
Executor listenerExecutor,
FinalShaderProgramWrapper finalShaderProgramWrapper,
boolean renderFramesAutomatically,
@@ -626,9 +603,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
boolean renderFramesAutomatically,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor videoFrameProcessorListenerExecutor,
- Listener listener,
+ VideoFrameProcessor.Listener listener,
GlObjectsProvider glObjectsProvider,
- @Nullable TextureOutputListener textureOutputListener,
+ @Nullable GlTextureProducer.Listener textureOutputListener,
int textureOutputCapacity)
throws GlUtil.GlException, VideoFrameProcessingException {
EGLDisplay eglDisplay = GlUtil.getDefaultEglDisplay();
@@ -780,7 +757,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
List This wrapper is used for the final {@link DefaultShaderProgram} instance in the chain of
* {@link DefaultShaderProgram} instances used by {@link VideoFrameProcessor}.
*/
-/* package */ final class FinalShaderProgramWrapper implements GlShaderProgram {
+/* package */ final class FinalShaderProgramWrapper implements GlShaderProgram, GlTextureProducer {
interface OnInputStreamProcessedListener {
void onInputStreamProcessed();
@@ -90,7 +90,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final TexturePool outputTexturePool;
private final LongArrayQueue outputTextureTimestamps; // Synchronized with outputTexturePool.
private final LongArrayQueue syncObjects;
- @Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
+ @Nullable private final GlTextureProducer.Listener textureOutputListener;
private int inputWidth;
private int inputHeight;
@@ -127,7 +127,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor videoFrameProcessorListenerExecutor,
VideoFrameProcessor.Listener videoFrameProcessorListener,
- @Nullable DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener,
+ @Nullable GlTextureProducer.Listener textureOutputListener,
int textureOutputCapacity) {
this.context = context;
this.matrixTransformations = new ArrayList<>();
@@ -220,11 +220,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
throw new UnsupportedOperationException();
}
- private void releaseOutputFrame(long presentationTimeUs) {
- videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs));
+ @Override
+ public void releaseOutputTexture(long presentationTimeUs) {
+ videoFrameProcessingTaskExecutor.submit(() -> releaseOutputTextureInternal(presentationTimeUs));
}
- private void releaseOutputFrameInternal(long presentationTimeUs) throws GlUtil.GlException {
+ private void releaseOutputTextureInternal(long presentationTimeUs) throws GlUtil.GlException {
checkState(textureOutputListener != null);
while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
&& outputTextureTimestamps.element() <= presentationTimeUs) {
@@ -390,7 +391,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long syncObject = GlUtil.createGlSyncFence();
syncObjects.add(syncObject);
checkNotNull(textureOutputListener)
- .onTextureRendered(outputTexture, presentationTimeUs, this::releaseOutputFrame, syncObject);
+ .onTextureRendered(
+ /* textureProducer= */ this, outputTexture, presentationTimeUs, syncObject);
}
/**
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlTextureProducer.java b/libraries/effect/src/main/java/androidx/media3/effect/GlTextureProducer.java
new file mode 100644
index 0000000000..cccb71f88e
--- /dev/null
+++ b/libraries/effect/src/main/java/androidx/media3/effect/GlTextureProducer.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.effect;
+
+import android.opengl.GLES30;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.UnstableApi;
+
+/** A component that outputs {@linkplain GlTextureInfo OpenGL textures}. */
+@UnstableApi
+public interface GlTextureProducer {
+
+ /** Listener for texture output. */
+ interface Listener {
+ /**
+ * Called when a texture has been rendered to.
+ *
+ * @param textureProducer The {@link GlTextureProducer} that has rendered the texture.
+ * @param outputTexture The texture that has been rendered.
+ * @param presentationTimeUs The presentation time of the texture.
+ * @param syncObject A GL sync object that has been inserted into the GL command stream after
+ * the last write of the {@code outputTexture}. Value is 0 if and only if the {@link
+ * GLES30#glFenceSync} failed.
+ */
+ void onTextureRendered(
+ GlTextureProducer textureProducer,
+ GlTextureInfo outputTexture,
+ long presentationTimeUs,
+ long syncObject)
+ throws VideoFrameProcessingException, GlUtil.GlException;
+ }
+
+ /** Releases the output texture at the given {@code presentationTimeUs}. */
+ void releaseOutputTexture(long presentationTimeUs);
+}
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java
index 4ef3a44ba0..c108b6fc62 100644
--- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java
@@ -26,7 +26,7 @@ import androidx.media3.common.util.UnstableApi;
* Input and output are provided via OpenGL textures.
*/
@UnstableApi
-public interface VideoCompositor {
+public interface VideoCompositor extends GlTextureProducer {
/** Listener for errors. */
interface Listener {
@@ -48,8 +48,7 @@ public interface VideoCompositor {
int registerInputSource();
/**
- * Signals that no more frames will come from the upstream {@link
- * DefaultVideoFrameProcessor.TextureOutputListener}.
+ * Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}.
*
* Each input source must have a unique {@code inputId} returned from {@link
* #registerInputSource}.
@@ -58,16 +57,16 @@ public interface VideoCompositor {
/**
* Queues an input texture to be composited, for example from an upstream {@link
- * DefaultVideoFrameProcessor.TextureOutputListener}.
+ * GlTextureProducer.Listener}.
*
* Each input source must have a unique {@code inputId} returned from {@link
* #registerInputSource}.
*/
void queueInputTexture(
int inputId,
+ GlTextureProducer textureProducer,
GlTextureInfo inputTexture,
- long presentationTimeUs,
- DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseTextureCallback);
+ long presentationTimeUs);
/** Releases all resources. */
void release();
diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java
index d91d54db1a..4dff162042 100644
--- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java
@@ -647,9 +647,9 @@ public final class DefaultVideoCompositorPixelTest {
compositorEnded.countDown();
}
},
- /* textureOutputListener= */ (outputTexture,
+ /* textureOutputListener= */ (outputTextureProducer,
+ outputTexture,
presentationTimeUs,
- releaseOutputTextureCallback,
syncObject) -> {
if (!useSharedExecutor) {
GlUtil.awaitSyncObject(syncObject);
@@ -658,7 +658,7 @@ public final class DefaultVideoCompositorPixelTest {
presentationTimeUs,
BitmapPixelTestUtil.createUnpremultipliedArgb8888BitmapFromFocusedGlFramebuffer(
outputTexture.width, outputTexture.height));
- releaseOutputTextureCallback.release(presentationTimeUs);
+ outputTextureProducer.releaseOutputTexture(presentationTimeUs);
},
/* textureOutputCapacity= */ 1);
inputBitmapReaders = new ArrayList<>();
@@ -791,15 +791,15 @@ public final class DefaultVideoCompositorPixelTest {
new DefaultVideoFrameProcessor.Factory.Builder()
.setGlObjectsProvider(glObjectsProvider)
.setTextureOutput(
- /* textureOutputListener= */ (outputTexture,
+ /* textureOutputListener= */ (outputTextureProducer,
+ outputTexture,
presentationTimeUs,
- releaseOutputTextureCallback,
syncObject) -> {
GlUtil.awaitSyncObject(syncObject);
textureBitmapReader.readBitmapUnpremultipliedAlpha(
outputTexture, presentationTimeUs);
videoCompositor.queueInputTexture(
- inputId, outputTexture, presentationTimeUs, releaseOutputTextureCallback);
+ inputId, outputTextureProducer, outputTexture, presentationTimeUs);
},
/* textureOutputCapacity= */ 2);
if (executorService != null) {
diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java
index cd1c57fbca..df803901ba 100644
--- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java
@@ -138,12 +138,9 @@ public class DefaultVideoFrameProcessorMultipleTextureOutputPixelTest {
VideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setTextureOutput(
- (outputTexture,
- presentationTimeUs,
- releaseOutputTextureCallback,
- unusedSyncObject) -> {
+ (outputTextureProducer, outputTexture, presentationTimeUs, unusedSyncObject) -> {
checkNotNull(textureBitmapReader).readBitmap(outputTexture, presentationTimeUs);
- releaseOutputTextureCallback.release(presentationTimeUs);
+ outputTextureProducer.releaseOutputTexture(presentationTimeUs);
},
/* textureOutputCapacity= */ 1)
.build();
diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
index e8da0f9179..55c65bd962 100644
--- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
@@ -42,6 +42,7 @@ import androidx.media3.common.util.GlUtil;
import androidx.media3.effect.BitmapOverlay;
import androidx.media3.effect.DefaultGlObjectsProvider;
import androidx.media3.effect.DefaultVideoFrameProcessor;
+import androidx.media3.effect.GlTextureProducer;
import androidx.media3.effect.OverlayEffect;
import androidx.media3.test.utils.BitmapPixelTestUtil;
import androidx.media3.test.utils.TextureBitmapReader;
@@ -496,15 +497,15 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setTextureOutput(
- (outputTexture, presentationTimeUs, releaseOutputTextureCallback, syncObject) ->
+ (outputTextureProducer, outputTexture, presentationTimeUs, syncObject) ->
inputTextureIntoVideoFrameProcessor(
testId,
consumersBitmapReader,
colorInfo,
effects,
outputTexture,
+ outputTextureProducer,
presentationTimeUs,
- releaseOutputTextureCallback,
syncObject),
/* textureOutputCapacity= */ 1)
.build();
@@ -524,8 +525,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
ColorInfo colorInfo,
List