diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index a712a2a0b0..4425df68fd 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -401,7 +401,11 @@ public final class GlUtil { return syncObject; } - /** Releases the underlying native object. */ + /** + * Deletes the underlying native object. + * + *

The {@code syncObject} must not be used after deletion. + */ public static void deleteSyncObject(long syncObject) throws GlException { GLES30.glDeleteSync(syncObject); checkGlError(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 87fc82453d..49498cdd0d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -213,6 +213,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new UnsupportedOperationException(); } + private void releaseOutputFrame(long presentationTimeUs) { + videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs)); + } + + private void releaseOutputFrameInternal(long presentationTimeUs) throws GlUtil.GlException { + while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity() + && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) { + outputTexturePool.freeTexture(); + outputTextureTimestamps.remove(); + GlUtil.deleteSyncObject(syncObjects.remove()); + maybeOnReadyToAcceptInputFrame(); + } + } + /** * Sets the list of {@link GlMatrixTransformation GlMatrixTransformations} and list of {@link * RgbMatrix RgbMatrices} to apply to the next {@linkplain #queueInputFrame queued} frame. @@ -229,20 +243,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; matrixTransformationsChanged = true; } - public void releaseOutputFrame(long presentationTimeUs) { - videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs)); - } - - private void releaseOutputFrameInternal(long presentationTimeUs) throws GlUtil.GlException { - while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity() - && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) { - outputTexturePool.freeTexture(); - outputTextureTimestamps.remove(); - GlUtil.deleteSyncObject(syncObjects.remove()); - maybeOnReadyToAcceptInputFrame(); - } - } - @Override public void flush() { // Drops all frames that aren't rendered yet. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 545ad4a439..230e62c319 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; import android.opengl.EGLContext; import android.opengl.EGLDisplay; +import android.opengl.EGLSurface; import android.opengl.GLES20; import androidx.annotation.GuardedBy; import androidx.annotation.IntRange; @@ -52,6 +53,7 @@ public final class VideoCompositor { // * Handle mismatched timestamps // * Before allowing customization of this class, add an interface, and rename this class to // DefaultCompositor. + // * Use a lock to synchronize inputFrameInfos more narrowly, to reduce blocking. /** Listener for errors. */ public interface ErrorListener { @@ -79,11 +81,13 @@ public final class VideoCompositor { private final List> inputFrameInfos; private final TexturePool outputTexturePool; + private final Queue outputTextureTimestamps; // Synchronized with outputTexturePool. + private final Queue syncObjects; // Synchronized with outputTexturePool. // Only used on the GL Thread. private @MonotonicNonNull EGLContext eglContext; private @MonotonicNonNull EGLDisplay eglDisplay; private @MonotonicNonNull GlProgram glProgram; - private long syncObject; + private @MonotonicNonNull EGLSurface placeholderEglSurface; /** * Creates an instance. @@ -105,6 +109,8 @@ public final class VideoCompositor { inputFrameInfos = new ArrayList<>(); outputTexturePool = new TexturePool(/* useHighPrecisionColorComponents= */ false, textureOutputCapacity); + outputTextureTimestamps = new ArrayDeque<>(textureOutputCapacity); + syncObjects = new ArrayDeque<>(textureOutputCapacity); boolean ownsExecutor = executorService == null; ExecutorService instanceExecutorService = @@ -142,13 +148,7 @@ public final class VideoCompositor { InputFrameInfo inputFrameInfo = new InputFrameInfo(inputTexture, presentationTimeUs, releaseTextureCallback); checkNotNull(inputFrameInfos.get(inputId)).add(inputFrameInfo); - - videoFrameProcessingTaskExecutor.submit( - () -> { - if (isReadyToComposite()) { - compositeToOutputTexture(); - } - }); + videoFrameProcessingTaskExecutor.submit(this::maybeComposite); } public void release() { @@ -162,25 +162,19 @@ public final class VideoCompositor { // Below methods must be called on the GL thread. private void setupGlObjects() throws GlUtil.GlException { - EGLDisplay eglDisplay = GlUtil.getDefaultEglDisplay(); - EGLContext eglContext = + eglDisplay = GlUtil.getDefaultEglDisplay(); + eglContext = glObjectsProvider.createEglContext( eglDisplay, /* openGlVersion= */ 2, GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888); - glObjectsProvider.createFocusedPlaceholderEglSurface(eglContext, eglDisplay); + placeholderEglSurface = + glObjectsProvider.createFocusedPlaceholderEglSurface(eglContext, eglDisplay); } - private synchronized boolean isReadyToComposite() { - // TODO: b/262694346 - Use timestamps to determine when to composite instead of number of - // frames. - for (int inputId = 0; inputId < inputFrameInfos.size(); inputId++) { - if (checkNotNull(inputFrameInfos.get(inputId)).isEmpty()) { - return false; - } + private synchronized void maybeComposite() throws VideoFrameProcessingException { + if (!isReadyToComposite()) { + return; } - return true; - } - private synchronized void compositeToOutputTexture() throws VideoFrameProcessingException { List framesToComposite = new ArrayList<>(); for (int inputId = 0; inputId < inputFrameInfos.size(); inputId++) { framesToComposite.add(checkNotNull(inputFrameInfos.get(inputId)).remove()); @@ -199,27 +193,55 @@ public final class VideoCompositor { outputTexturePool.ensureConfigured( glObjectsProvider, inputFrame1.texture.width, inputFrame1.texture.height); GlTextureInfo outputTexture = outputTexturePool.useTexture(); + long outputPresentationTimestampUs = framesToComposite.get(0).presentationTimeUs; + outputTextureTimestamps.add(outputPresentationTimestampUs); drawFrame(inputFrame1.texture, inputFrame2.texture, outputTexture); - syncObject = GlUtil.createGlSyncFence(); - + long syncObject = GlUtil.createGlSyncFence(); + syncObjects.add(syncObject); + textureOutputListener.onTextureRendered( + outputTexture, + /* presentationTimeUs= */ framesToComposite.get(0).presentationTimeUs, + this::releaseOutputFrame, + syncObject); for (int i = 0; i < framesToComposite.size(); i++) { InputFrameInfo inputFrameInfo = framesToComposite.get(i); inputFrameInfo.releaseCallback.release(inputFrameInfo.presentationTimeUs); } - - // TODO: b/262694346 - Use presentationTimeUs here for freeing textures. - textureOutputListener.onTextureRendered( - checkNotNull(outputTexture), - /* presentationTimeUs= */ framesToComposite.get(0).presentationTimeUs, - (presentationTimeUs) -> - videoFrameProcessingTaskExecutor.submit(outputTexturePool::freeTexture), - syncObject); } catch (GlUtil.GlException e) { throw VideoFrameProcessingException.from(e); } } + private synchronized boolean isReadyToComposite() { + if (outputTexturePool.freeTextureCount() == 0) { + return false; + } + // TODO: b/262694346 - Use timestamps to determine when to composite instead of number of + // frames. + for (int inputId = 0; inputId < inputFrameInfos.size(); inputId++) { + if (checkNotNull(inputFrameInfos.get(inputId)).isEmpty()) { + return false; + } + } + return true; + } + + private void releaseOutputFrame(long presentationTimeUs) { + videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs)); + } + + private synchronized void releaseOutputFrameInternal(long presentationTimeUs) + throws VideoFrameProcessingException, GlUtil.GlException { + while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity() + && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) { + outputTexturePool.freeTexture(); + outputTextureTimestamps.remove(); + GlUtil.deleteSyncObject(syncObjects.remove()); + } + maybeComposite(); + } + private void ensureGlProgramConfigured() throws VideoFrameProcessingException { if (glProgram != null) { return; @@ -262,10 +284,11 @@ public final class VideoCompositor { private void releaseGlObjects() { try { outputTexturePool.deleteAllTextures(); + GlUtil.destroyEglSurface(eglDisplay, placeholderEglSurface); if (glProgram != null) { glProgram.delete(); } - } catch (Exception e) { + } catch (GlUtil.GlException e) { Log.e(TAG, "Error releasing GL resources", e); } finally { try { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java index dd803eec64..195c155fe6 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java @@ -96,9 +96,7 @@ public final class VideoCompositorPixelTest { testId, (outputTexture, presentationTimeUs, releaseOutputTextureCallback, syncObject) -> { try { - if (useSharedExecutor) { - GlUtil.deleteSyncObject(syncObject); - } else { + if (!useSharedExecutor) { GlUtil.awaitSyncObject(syncObject); } compositedOutputBitmap.set( @@ -146,9 +144,7 @@ public final class VideoCompositorPixelTest { testId, (outputTexture, presentationTimeUs, releaseOutputTextureCallback, syncObject) -> { try { - if (useSharedExecutor) { - GlUtil.deleteSyncObject(syncObject); - } else { + if (!useSharedExecutor) { GlUtil.awaitSyncObject(syncObject); } if (compositedFirstOutputBitmap.get() == null) {