diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java new file mode 100644 index 0000000000..af776fa9cc --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -0,0 +1,342 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; + +import android.content.Context; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLSurface; +import android.opengl.GLES20; +import androidx.annotation.GuardedBy; +import androidx.annotation.IntRange; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ExecutorService; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * A basic {@link VideoCompositor} implementation that takes in frames from exactly 2 input sources' + * streams and combines them into one output stream. + * + *

The first {@linkplain #registerInputSource registered source} will be the primary stream, + * which is used to determine the output frames' timestamps and dimensions. + */ +@UnstableApi +public final class DefaultVideoCompositor implements VideoCompositor { + // TODO: b/262694346 - Flesh out this implementation by doing the following: + // * Handle mismatched timestamps + // * Use a lock to synchronize inputFrameInfos more narrowly, to reduce blocking. + // * If the primary stream ends, consider setting the secondary stream as the new primary stream, + // so that secondary stream frames aren't dropped. + + private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread"; + private static final String TAG = "DefaultVideoCompositor"; + private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_compositor_es2.glsl"; + private static final int PRIMARY_INPUT_ID = 0; + + private final Context context; + private final Listener listener; + private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener; + private final GlObjectsProvider glObjectsProvider; + private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; + + @GuardedBy("this") + private final List inputSources; + + private boolean allInputsEnded; // Whether all inputSources have signaled end of input. + + private final TexturePool outputTexturePool; + private final Queue outputTextureTimestamps; // Synchronized with outputTexturePool. + private final Queue syncObjects; // Synchronized with outputTexturePool. + + // Only used on the GL Thread. + private @MonotonicNonNull EGLContext eglContext; + private @MonotonicNonNull EGLDisplay eglDisplay; + private @MonotonicNonNull GlProgram glProgram; + private @MonotonicNonNull EGLSurface placeholderEglSurface; + + /** + * Creates an instance. + * + *

If a non-null {@code executorService} is set, the {@link ExecutorService} must be + * {@linkplain ExecutorService#shutdown shut down} by the caller. + */ + public DefaultVideoCompositor( + Context context, + GlObjectsProvider glObjectsProvider, + @Nullable ExecutorService executorService, + Listener listener, + DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener, + @IntRange(from = 1) int textureOutputCapacity) { + this.context = context; + this.listener = listener; + this.textureOutputListener = textureOutputListener; + this.glObjectsProvider = glObjectsProvider; + + inputSources = new ArrayList<>(); + outputTexturePool = + new TexturePool(/* useHighPrecisionColorComponents= */ false, textureOutputCapacity); + outputTextureTimestamps = new ArrayDeque<>(textureOutputCapacity); + syncObjects = new ArrayDeque<>(textureOutputCapacity); + + boolean ownsExecutor = executorService == null; + ExecutorService instanceExecutorService = + ownsExecutor ? Util.newSingleThreadExecutor(THREAD_NAME) : checkNotNull(executorService); + videoFrameProcessingTaskExecutor = + new VideoFrameProcessingTaskExecutor( + instanceExecutorService, + /* shouldShutdownExecutorService= */ ownsExecutor, + listener::onError); + videoFrameProcessingTaskExecutor.submit(this::setupGlObjects); + } + + @Override + public synchronized int registerInputSource() { + inputSources.add(new InputSource()); + return inputSources.size() - 1; + } + + @Override + public synchronized void signalEndOfInputSource(int inputId) { + inputSources.get(inputId).isInputEnded = true; + for (int i = 0; i < inputSources.size(); i++) { + if (!inputSources.get(i).isInputEnded) { + return; + } + } + allInputsEnded = true; + if (inputSources.get(PRIMARY_INPUT_ID).frameInfos.isEmpty()) { + listener.onEnded(); + } + } + + @Override + public synchronized void queueInputTexture( + int inputId, + GlTextureInfo inputTexture, + long presentationTimeUs, + DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseTextureCallback) { + checkState(!inputSources.get(inputId).isInputEnded); + InputFrameInfo inputFrameInfo = + new InputFrameInfo(inputTexture, presentationTimeUs, releaseTextureCallback); + inputSources.get(inputId).frameInfos.add(inputFrameInfo); + videoFrameProcessingTaskExecutor.submit(this::maybeComposite); + } + + @Override + public void release() { + try { + videoFrameProcessingTaskExecutor.release(/* releaseTask= */ this::releaseGlObjects); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + // Below methods must be called on the GL thread. + private void setupGlObjects() throws GlUtil.GlException { + eglDisplay = GlUtil.getDefaultEglDisplay(); + eglContext = + glObjectsProvider.createEglContext( + eglDisplay, /* openGlVersion= */ 2, GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888); + placeholderEglSurface = + glObjectsProvider.createFocusedPlaceholderEglSurface(eglContext, eglDisplay); + } + + private synchronized void maybeComposite() + throws VideoFrameProcessingException, GlUtil.GlException { + if (!isReadyToComposite()) { + return; + } + + List framesToComposite = new ArrayList<>(); + for (int inputId = 0; inputId < inputSources.size(); inputId++) { + framesToComposite.add(inputSources.get(inputId).frameInfos.remove()); + } + + ensureGlProgramConfigured(); + + // TODO: b/262694346 - + // * Support an arbitrary number of inputs. + // * Allow different frame dimensions. + InputFrameInfo inputFrame1 = framesToComposite.get(0); + InputFrameInfo inputFrame2 = framesToComposite.get(1); + checkState(inputFrame1.texture.width == inputFrame2.texture.width); + checkState(inputFrame1.texture.height == inputFrame2.texture.height); + outputTexturePool.ensureConfigured( + glObjectsProvider, inputFrame1.texture.width, inputFrame1.texture.height); + GlTextureInfo outputTexture = outputTexturePool.useTexture(); + long outputPresentationTimestampUs = framesToComposite.get(PRIMARY_INPUT_ID).presentationTimeUs; + outputTextureTimestamps.add(outputPresentationTimestampUs); + + drawFrame(inputFrame1.texture, inputFrame2.texture, outputTexture); + long syncObject = GlUtil.createGlSyncFence(); + syncObjects.add(syncObject); + textureOutputListener.onTextureRendered( + outputTexture, + /* presentationTimeUs= */ framesToComposite.get(0).presentationTimeUs, + this::releaseOutputFrame, + syncObject); + for (int i = 0; i < framesToComposite.size(); i++) { + InputFrameInfo inputFrameInfo = framesToComposite.get(i); + inputFrameInfo.releaseCallback.release(inputFrameInfo.presentationTimeUs); + } + if (allInputsEnded && inputSources.get(PRIMARY_INPUT_ID).frameInfos.isEmpty()) { + listener.onEnded(); + } + } + + private synchronized boolean isReadyToComposite() { + if (outputTexturePool.freeTextureCount() == 0) { + return false; + } + long compositeTimestampUs = C.TIME_UNSET; + for (int inputId = 0; inputId < inputSources.size(); inputId++) { + Queue inputFrameInfos = inputSources.get(inputId).frameInfos; + if (inputFrameInfos.isEmpty()) { + return false; + } + + long inputTimestampUs = checkNotNull(inputFrameInfos.peek()).presentationTimeUs; + if (inputId == PRIMARY_INPUT_ID) { + compositeTimestampUs = inputTimestampUs; + } + // TODO: b/262694346 - Allow for different frame-rates to be composited, by potentially + // dropping some frames in non-primary streams. + if (inputTimestampUs != compositeTimestampUs) { + throw new IllegalStateException("Non-matched timestamps not yet supported."); + } + } + return true; + } + + private void releaseOutputFrame(long presentationTimeUs) { + videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs)); + } + + private synchronized void releaseOutputFrameInternal(long presentationTimeUs) + throws VideoFrameProcessingException, GlUtil.GlException { + while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity() + && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) { + outputTexturePool.freeTexture(); + outputTextureTimestamps.remove(); + GlUtil.deleteSyncObject(syncObjects.remove()); + } + maybeComposite(); + } + + private void ensureGlProgramConfigured() + throws VideoFrameProcessingException, GlUtil.GlException { + if (glProgram != null) { + return; + } + try { + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + } catch (IOException e) { + throw new VideoFrameProcessingException(e); + } + } + + private void drawFrame( + GlTextureInfo inputTexture1, GlTextureInfo inputTexture2, GlTextureInfo outputTexture) + throws GlUtil.GlException { + GlUtil.focusFramebufferUsingCurrentContext( + outputTexture.fboId, outputTexture.width, outputTexture.height); + GlUtil.clearFocusedBuffers(); + + GlProgram glProgram = checkNotNull(this.glProgram); + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler1", inputTexture1.texId, /* texUnitIndex= */ 0); + glProgram.setSamplerTexIdUniform("uTexSampler2", inputTexture2.texId, /* texUnitIndex= */ 1); + + glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix()); + glProgram.setFloatsUniform("uTransformationMatrix", GlUtil.create4x4IdentityMatrix()); + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); + } + + private void releaseGlObjects() { + try { + checkState(allInputsEnded); + outputTexturePool.deleteAllTextures(); + GlUtil.destroyEglSurface(eglDisplay, placeholderEglSurface); + if (glProgram != null) { + glProgram.delete(); + } + } catch (GlUtil.GlException e) { + Log.e(TAG, "Error releasing GL resources", e); + } finally { + try { + GlUtil.destroyEglContext(eglDisplay, eglContext); + } catch (GlUtil.GlException e) { + Log.e(TAG, "Error releasing GL context", e); + } + } + } + + /** Holds information on an input source. */ + private static final class InputSource { + public final Queue frameInfos; + public boolean isInputEnded; + + public InputSource() { + frameInfos = new ArrayDeque<>(); + } + } + + /** Holds information on a frame and how to release it. */ + private static final class InputFrameInfo { + public final GlTextureInfo texture; + public final long presentationTimeUs; + public final DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseCallback; + + public InputFrameInfo( + GlTextureInfo texture, + long presentationTimeUs, + DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseCallback) { + this.texture = texture; + this.presentationTimeUs = presentationTimeUs; + this.releaseCallback = releaseCallback; + } + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index 019f3c1774..4ef3a44ba0 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -15,51 +15,21 @@ */ package androidx.media3.effect; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; - -import android.content.Context; -import android.opengl.EGLContext; -import android.opengl.EGLDisplay; -import android.opengl.EGLSurface; -import android.opengl.GLES20; -import androidx.annotation.GuardedBy; -import androidx.annotation.IntRange; -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.util.GlProgram; -import androidx.media3.common.util.GlUtil; -import androidx.media3.common.util.Log; import androidx.media3.common.util.UnstableApi; -import androidx.media3.common.util.Util; -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.ExecutorService; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** - * A basic VideoCompositor that takes in frames from exactly 2 input sources and combines it to one - * output. + * Interface for a video compositor that combines frames from mutliple input sources to produce + * output frames. * - *

The first {@linkplain #registerInputSource registered source} will be the primary stream, - * which is used to determine the output frames' timestamps and dimensions. + *

Input and output are provided via OpenGL textures. */ @UnstableApi -public final class VideoCompositor { - // TODO: b/262694346 - Flesh out this implementation by doing the following: - // * Handle mismatched timestamps - // * Before allowing customization of this class, add an interface, and rename this class to - // DefaultCompositor. - // * Use a lock to synchronize inputFrameInfos more narrowly, to reduce blocking. +public interface VideoCompositor { /** Listener for errors. */ - public interface Listener { + interface Listener { /** * Called when an exception occurs during asynchronous frame compositing. * @@ -71,75 +41,11 @@ public final class VideoCompositor { void onEnded(); } - private static final String THREAD_NAME = "Effect:VideoCompositor:GlThread"; - private static final String TAG = "VideoCompositor"; - private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; - private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_compositor_es2.glsl"; - private static final int PRIMARY_INPUT_ID = 0; - - private final Context context; - private final Listener listener; - private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener; - private final GlObjectsProvider glObjectsProvider; - private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; - - @GuardedBy("this") - private final List inputSources; - - private boolean allInputsEnded; // Whether all inputSources have signaled end of input. - - private final TexturePool outputTexturePool; - private final Queue outputTextureTimestamps; // Synchronized with outputTexturePool. - private final Queue syncObjects; // Synchronized with outputTexturePool. - // Only used on the GL Thread. - private @MonotonicNonNull EGLContext eglContext; - private @MonotonicNonNull EGLDisplay eglDisplay; - private @MonotonicNonNull GlProgram glProgram; - private @MonotonicNonNull EGLSurface placeholderEglSurface; - - /** - * Creates an instance. - * - *

If a non-null {@code executorService} is set, the {@link ExecutorService} must be - * {@linkplain ExecutorService#shutdown shut down} by the caller. - */ - public VideoCompositor( - Context context, - GlObjectsProvider glObjectsProvider, - @Nullable ExecutorService executorService, - Listener listener, - DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener, - @IntRange(from = 1) int textureOutputCapacity) { - this.context = context; - this.listener = listener; - this.textureOutputListener = textureOutputListener; - this.glObjectsProvider = glObjectsProvider; - - inputSources = new ArrayList<>(); - outputTexturePool = - new TexturePool(/* useHighPrecisionColorComponents= */ false, textureOutputCapacity); - outputTextureTimestamps = new ArrayDeque<>(textureOutputCapacity); - syncObjects = new ArrayDeque<>(textureOutputCapacity); - - boolean ownsExecutor = executorService == null; - ExecutorService instanceExecutorService = - ownsExecutor ? Util.newSingleThreadExecutor(THREAD_NAME) : checkNotNull(executorService); - videoFrameProcessingTaskExecutor = - new VideoFrameProcessingTaskExecutor( - instanceExecutorService, - /* shouldShutdownExecutorService= */ ownsExecutor, - listener::onError); - videoFrameProcessingTaskExecutor.submit(this::setupGlObjects); - } - /** * Registers a new input source, and returns a unique {@code inputId} corresponding to this * source, to be used in {@link #queueInputTexture}. */ - public synchronized int registerInputSource() { - inputSources.add(new InputSource()); - return inputSources.size() - 1; - } + int registerInputSource(); /** * Signals that no more frames will come from the upstream {@link @@ -148,18 +54,7 @@ public final class VideoCompositor { *

Each input source must have a unique {@code inputId} returned from {@link * #registerInputSource}. */ - public synchronized void signalEndOfInputSource(int inputId) { - inputSources.get(inputId).isInputEnded = true; - for (int i = 0; i < inputSources.size(); i++) { - if (!inputSources.get(i).isInputEnded) { - return; - } - } - allInputsEnded = true; - if (inputSources.get(PRIMARY_INPUT_ID).frameInfos.isEmpty()) { - listener.onEnded(); - } - } + void signalEndOfInputSource(int inputId); /** * Queues an input texture to be composited, for example from an upstream {@link @@ -168,202 +63,12 @@ public final class VideoCompositor { *

Each input source must have a unique {@code inputId} returned from {@link * #registerInputSource}. */ - public synchronized void queueInputTexture( + void queueInputTexture( int inputId, GlTextureInfo inputTexture, long presentationTimeUs, - DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseTextureCallback) - throws VideoFrameProcessingException { - checkState(!inputSources.get(inputId).isInputEnded); - InputFrameInfo inputFrameInfo = - new InputFrameInfo(inputTexture, presentationTimeUs, releaseTextureCallback); - inputSources.get(inputId).frameInfos.add(inputFrameInfo); - videoFrameProcessingTaskExecutor.submit(this::maybeComposite); - } + DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseTextureCallback); - public void release() { - try { - videoFrameProcessingTaskExecutor.release(/* releaseTask= */ this::releaseGlObjects); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException(e); - } - } - - // Below methods must be called on the GL thread. - private void setupGlObjects() throws GlUtil.GlException { - eglDisplay = GlUtil.getDefaultEglDisplay(); - eglContext = - glObjectsProvider.createEglContext( - eglDisplay, /* openGlVersion= */ 2, GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888); - placeholderEglSurface = - glObjectsProvider.createFocusedPlaceholderEglSurface(eglContext, eglDisplay); - } - - private synchronized void maybeComposite() - throws VideoFrameProcessingException, GlUtil.GlException { - if (!isReadyToComposite()) { - return; - } - - List framesToComposite = new ArrayList<>(); - for (int inputId = 0; inputId < inputSources.size(); inputId++) { - framesToComposite.add(inputSources.get(inputId).frameInfos.remove()); - } - - ensureGlProgramConfigured(); - - // TODO: b/262694346 - - // * Support an arbitrary number of inputs. - // * Allow different frame dimensions. - InputFrameInfo inputFrame1 = framesToComposite.get(0); - InputFrameInfo inputFrame2 = framesToComposite.get(1); - checkState(inputFrame1.texture.width == inputFrame2.texture.width); - checkState(inputFrame1.texture.height == inputFrame2.texture.height); - outputTexturePool.ensureConfigured( - glObjectsProvider, inputFrame1.texture.width, inputFrame1.texture.height); - GlTextureInfo outputTexture = outputTexturePool.useTexture(); - long outputPresentationTimestampUs = framesToComposite.get(PRIMARY_INPUT_ID).presentationTimeUs; - outputTextureTimestamps.add(outputPresentationTimestampUs); - - drawFrame(inputFrame1.texture, inputFrame2.texture, outputTexture); - long syncObject = GlUtil.createGlSyncFence(); - syncObjects.add(syncObject); - textureOutputListener.onTextureRendered( - outputTexture, - /* presentationTimeUs= */ framesToComposite.get(0).presentationTimeUs, - this::releaseOutputFrame, - syncObject); - for (int i = 0; i < framesToComposite.size(); i++) { - InputFrameInfo inputFrameInfo = framesToComposite.get(i); - inputFrameInfo.releaseCallback.release(inputFrameInfo.presentationTimeUs); - } - if (allInputsEnded && inputSources.get(PRIMARY_INPUT_ID).frameInfos.isEmpty()) { - listener.onEnded(); - } - } - - private synchronized boolean isReadyToComposite() { - if (outputTexturePool.freeTextureCount() == 0) { - return false; - } - long compositeTimestampUs = C.TIME_UNSET; - for (int inputId = 0; inputId < inputSources.size(); inputId++) { - Queue inputFrameInfos = inputSources.get(inputId).frameInfos; - if (inputFrameInfos.isEmpty()) { - return false; - } - - long inputTimestampUs = checkNotNull(inputFrameInfos.peek()).presentationTimeUs; - if (inputId == PRIMARY_INPUT_ID) { - compositeTimestampUs = inputTimestampUs; - } - // TODO: b/262694346 - Allow for different frame-rates to be composited, by potentially - // dropping some frames in non-primary streams. - if (inputTimestampUs != compositeTimestampUs) { - throw new IllegalStateException("Non-matched timestamps not yet supported."); - } - } - return true; - } - - private void releaseOutputFrame(long presentationTimeUs) { - videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs)); - } - - private synchronized void releaseOutputFrameInternal(long presentationTimeUs) - throws VideoFrameProcessingException, GlUtil.GlException { - while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity() - && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) { - outputTexturePool.freeTexture(); - outputTextureTimestamps.remove(); - GlUtil.deleteSyncObject(syncObjects.remove()); - } - maybeComposite(); - } - - private void ensureGlProgramConfigured() - throws VideoFrameProcessingException, GlUtil.GlException { - if (glProgram != null) { - return; - } - try { - glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); - glProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - } catch (IOException e) { - throw new VideoFrameProcessingException(e); - } - } - - private void drawFrame( - GlTextureInfo inputTexture1, GlTextureInfo inputTexture2, GlTextureInfo outputTexture) - throws GlUtil.GlException { - GlUtil.focusFramebufferUsingCurrentContext( - outputTexture.fboId, outputTexture.width, outputTexture.height); - GlUtil.clearFocusedBuffers(); - - GlProgram glProgram = checkNotNull(this.glProgram); - glProgram.use(); - glProgram.setSamplerTexIdUniform("uTexSampler1", inputTexture1.texId, /* texUnitIndex= */ 0); - glProgram.setSamplerTexIdUniform("uTexSampler2", inputTexture2.texId, /* texUnitIndex= */ 1); - - glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix()); - glProgram.setFloatsUniform("uTransformationMatrix", GlUtil.create4x4IdentityMatrix()); - glProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - GlUtil.checkGlError(); - } - - private void releaseGlObjects() { - try { - checkState(allInputsEnded); - outputTexturePool.deleteAllTextures(); - GlUtil.destroyEglSurface(eglDisplay, placeholderEglSurface); - if (glProgram != null) { - glProgram.delete(); - } - } catch (GlUtil.GlException e) { - Log.e(TAG, "Error releasing GL resources", e); - } finally { - try { - GlUtil.destroyEglContext(eglDisplay, eglContext); - } catch (GlUtil.GlException e) { - Log.e(TAG, "Error releasing GL context", e); - } - } - } - - /** Holds information on an input source. */ - private static final class InputSource { - public final Queue frameInfos; - public boolean isInputEnded; - - public InputSource() { - frameInfos = new ArrayDeque<>(); - } - } - - /** Holds information on a frame and how to release it. */ - private static final class InputFrameInfo { - public final GlTextureInfo texture; - public final long presentationTimeUs; - public final DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseCallback; - - public InputFrameInfo( - GlTextureInfo texture, - long presentationTimeUs, - DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseCallback) { - this.texture = texture; - this.presentationTimeUs = presentationTimeUs; - this.releaseCallback = releaseCallback; - } - } + /** Releases all resources. */ + void release(); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java similarity index 88% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java index 68f1d4441b..80b07eefb5 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java @@ -33,6 +33,7 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Util; import androidx.media3.effect.DefaultGlObjectsProvider; +import androidx.media3.effect.DefaultVideoCompositor; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.RgbFilter; import androidx.media3.effect.ScaleAndRotateTransformation; @@ -56,9 +57,9 @@ import org.junit.rules.TestName; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -/** Pixel test for {@link VideoCompositor} compositing 2 input frames into 1 output frame. */ +/** Pixel test for {@link DefaultVideoCompositor} compositing 2 input frames into 1 output frame. */ @RunWith(Parameterized.class) -public final class VideoCompositorPixelTest { +public final class DefaultVideoCompositorPixelTest { private static final String ORIGINAL_PNG_ASSET_PATH = "media/bitmap/input_images/media3test.png"; private static final String GRAYSCALE_PNG_ASSET_PATH = @@ -76,33 +77,33 @@ public final class VideoCompositorPixelTest { @Parameterized.Parameter public boolean useSharedExecutor; @Rule public TestName testName = new TestName(); - private @MonotonicNonNull VideoCompositorTestRunner videoCompositorTestRunner; + private @MonotonicNonNull VideoCompositorTestRunner compositorTestRunner; @After public void tearDown() { - if (videoCompositorTestRunner != null) { - videoCompositorTestRunner.release(); + if (compositorTestRunner != null) { + compositorTestRunner.release(); } } @Test public void compositeTwoInputs_withOneFrameFromEach_matchesExpectedBitmap() throws Exception { String testId = testName.getMethodName(); - videoCompositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor); + compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor); - videoCompositorTestRunner.queueBitmapsToBothInputs(/* count= */ 1); + compositorTestRunner.queueBitmapsToBothInputs(/* count= */ 1); saveAndAssertBitmapMatchesExpected( testId, - videoCompositorTestRunner.inputBitmapReader1.getBitmap(), + compositorTestRunner.inputBitmapReader1.getBitmap(), /* actualBitmapLabel= */ "actualCompositorInputBitmap1", GRAYSCALE_PNG_ASSET_PATH); saveAndAssertBitmapMatchesExpected( testId, - videoCompositorTestRunner.inputBitmapReader2.getBitmap(), + compositorTestRunner.inputBitmapReader2.getBitmap(), /* actualBitmapLabel= */ "actualCompositorInputBitmap2", ROTATE180_PNG_ASSET_PATH); - videoCompositorTestRunner.saveAndAssertFirstCompositedBitmapMatchesExpected( + compositorTestRunner.saveAndAssertFirstCompositedBitmapMatchesExpected( GRAYSCALE_AND_ROTATE180_COMPOSITE_PNG_ASSET_PATH); } @@ -110,9 +111,9 @@ public final class VideoCompositorPixelTest { public void compositeTwoInputs_withFiveFramesFromEach_matchesExpectedTimestamps() throws Exception { String testId = testName.getMethodName(); - videoCompositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor); + compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor); - videoCompositorTestRunner.queueBitmapsToBothInputs(/* count= */ 5); + compositorTestRunner.queueBitmapsToBothInputs(/* count= */ 5); ImmutableList expectedTimestamps = ImmutableList.of( @@ -121,16 +122,16 @@ public final class VideoCompositorPixelTest { 2 * C.MICROS_PER_SECOND, 3 * C.MICROS_PER_SECOND, 4 * C.MICROS_PER_SECOND); - assertThat(videoCompositorTestRunner.inputBitmapReader1.getOutputTimestamps()) + assertThat(compositorTestRunner.inputBitmapReader1.getOutputTimestamps()) .containsExactlyElementsIn(expectedTimestamps) .inOrder(); - assertThat(videoCompositorTestRunner.inputBitmapReader2.getOutputTimestamps()) + assertThat(compositorTestRunner.inputBitmapReader2.getOutputTimestamps()) .containsExactlyElementsIn(expectedTimestamps) .inOrder(); - assertThat(videoCompositorTestRunner.compositedTimestamps) + assertThat(compositorTestRunner.compositedTimestamps) .containsExactlyElementsIn(expectedTimestamps) .inOrder(); - videoCompositorTestRunner.saveAndAssertFirstCompositedBitmapMatchesExpected( + compositorTestRunner.saveAndAssertFirstCompositedBitmapMatchesExpected( GRAYSCALE_AND_ROTATE180_COMPOSITE_PNG_ASSET_PATH); } @@ -138,22 +139,22 @@ public final class VideoCompositorPixelTest { public void compositeTwoInputs_withTenFramesFromEach_matchesExpectedFrameCount() throws Exception { String testId = testName.getMethodName(); - videoCompositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor); + compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor); int numberOfFramesToQueue = 10; - videoCompositorTestRunner.queueBitmapsToBothInputs(numberOfFramesToQueue); + compositorTestRunner.queueBitmapsToBothInputs(numberOfFramesToQueue); - assertThat(videoCompositorTestRunner.inputBitmapReader1.getOutputTimestamps()) + assertThat(compositorTestRunner.inputBitmapReader1.getOutputTimestamps()) .hasSize(numberOfFramesToQueue); - assertThat(videoCompositorTestRunner.inputBitmapReader2.getOutputTimestamps()) + assertThat(compositorTestRunner.inputBitmapReader2.getOutputTimestamps()) .hasSize(numberOfFramesToQueue); - assertThat(videoCompositorTestRunner.compositedTimestamps).hasSize(numberOfFramesToQueue); - videoCompositorTestRunner.saveAndAssertFirstCompositedBitmapMatchesExpected( + assertThat(compositorTestRunner.compositedTimestamps).hasSize(numberOfFramesToQueue); + compositorTestRunner.saveAndAssertFirstCompositedBitmapMatchesExpected( GRAYSCALE_AND_ROTATE180_COMPOSITE_PNG_ASSET_PATH); } /** - * A test runner for {@link VideoCompositor tests} tests. + * A test runner for {@link DefaultVideoCompositor} tests. * *

Composites input bitmaps from two input sources. */ @@ -190,7 +191,7 @@ public final class VideoCompositorPixelTest { compositedTimestamps = new CopyOnWriteArrayList<>(); compositorEnded = new CountDownLatch(1); videoCompositor = - new VideoCompositor( + new DefaultVideoCompositor( getApplicationContext(), glObjectsProvider, sharedExecutorService,