diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeShaderProgram.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeShaderProgram.java index 8853562088..b2cdc988f8 100644 --- a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeShaderProgram.java +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeShaderProgram.java @@ -24,6 +24,7 @@ import android.content.Context; import android.opengl.EGL14; import androidx.annotation.Nullable; import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.LibraryLoader; @@ -159,6 +160,9 @@ import java.util.concurrent.Future; () -> errorListener.onError(new VideoFrameProcessingException(error)))); } + @Override + public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {} + @Override public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { AppTextureFrame appTextureFrame = diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 5bbe72e2ff..a13c89d45c 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -42,6 +42,15 @@ public interface VideoFrameProcessor { /** A factory for {@link VideoFrameProcessor} instances. */ interface Factory { + + /** + * Sets the {@link GlObjectsProvider}. + * + *

Must be called before {@link #create}. + */ + Factory setGlObjectsProvider(GlObjectsProvider glObjectsProvider); + + // TODO(271433904): Turn parameters with default values into setters. /** * Creates a new {@link VideoFrameProcessor} instance. * diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java index 0627f214f2..efeeca6bd4 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java @@ -28,6 +28,7 @@ import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.FrameInfo; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; @@ -410,6 +411,9 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { @Override public void setErrorListener(Executor executor, ErrorListener errorListener) {} + @Override + public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {} + @Override public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { // No input is queued in these tests. The BlankFrameProducer is used to produce frames. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 214b2fb1dd..8cf3577965 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -36,6 +36,7 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.FrameInfo; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; @@ -62,6 +63,20 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { /** A factory for {@link DefaultVideoFrameProcessor} instances. */ public static final class Factory implements VideoFrameProcessor.Factory { + private GlObjectsProvider glObjectsProvider = GlObjectsProvider.DEFAULT; + + /** + * {@inheritDoc} + * + *

The default value is {@link GlObjectsProvider#DEFAULT}. + */ + @Override + public DefaultVideoFrameProcessor.Factory setGlObjectsProvider( + GlObjectsProvider glObjectsProvider) { + this.glObjectsProvider = glObjectsProvider; + return this; + } + /** * {@inheritDoc} * @@ -139,7 +154,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { releaseFramesAutomatically, singleThreadExecutorService, listenerExecutor, - listener)); + listener, + glObjectsProvider)); try { return defaultVideoFrameProcessorFuture.get(); @@ -367,7 +383,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { boolean releaseFramesAutomatically, ExecutorService singleThreadExecutorService, Executor executor, - Listener listener) + Listener listener, + GlObjectsProvider glObjectsProvider) throws GlUtil.GlException, VideoFrameProcessingException { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); @@ -380,7 +397,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888; int openGlVersion = ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo) ? 3 : 2; - EGLContext eglContext = GlUtil.createEglContext(eglDisplay, openGlVersion, configAttributes); + EGLContext eglContext = + glObjectsProvider.createEglContext(eglDisplay, openGlVersion, configAttributes); GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay, configAttributes); // Not releaseFramesAutomatically means outputting to a display surface. HDR display surfaces @@ -409,6 +427,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { releaseFramesAutomatically, executor, listener); + setGlObjectProviderOnShaderPrograms(shaderPrograms, glObjectsProvider); VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor = new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener); chainShaderProgramsWithListeners( @@ -522,6 +541,15 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { return shaderProgramListBuilder.build(); } + /** Sets the {@link GlObjectsProvider} on all of the {@linkplain GlShaderProgram}s provided. */ + private static void setGlObjectProviderOnShaderPrograms( + ImmutableList shaderPrograms, GlObjectsProvider glObjectsProvider) { + for (int i = 0; i < shaderPrograms.size() - 1; i++) { + GlShaderProgram shaderProgram = shaderPrograms.get(i); + shaderProgram.setGlObjectsProvider(glObjectsProvider); + } + } + /** * Chains the given {@link GlShaderProgram} instances using {@link * ChainingGlShaderProgramListener} instances. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 5c64c37579..ce71a90e64 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -35,6 +35,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; @@ -86,9 +87,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private int inputHeight; @Nullable private DefaultShaderProgram defaultShaderProgram; @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper; + private GlObjectsProvider glObjectsProvider; private InputListener inputListener; private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation; @Nullable private SurfaceView debugSurfaceView; + private boolean frameProcessingStarted; private volatile boolean outputSizeOrRotationChanged; @@ -130,10 +133,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; textureTransformMatrix = GlUtil.create4x4IdentityMatrix(); streamOffsetUsQueue = new ConcurrentLinkedQueue<>(); + glObjectsProvider = GlObjectsProvider.DEFAULT; inputListener = new InputListener() {}; availableFrames = new ConcurrentLinkedQueue<>(); } + @Override + public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) { + checkState( + !frameProcessingStarted, + "The GlObjectsProvider cannot be set after frame processing has started."); + this.glObjectsProvider = glObjectsProvider; + } + @Override public void setInputListener(InputListener inputListener) { this.inputListener = inputListener; @@ -154,6 +166,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void signalEndOfCurrentInputStream() { + frameProcessingStarted = true; checkState(!streamOffsetUsQueue.isEmpty(), "No input stream to end."); streamOffsetUsQueue.remove(); if (streamOffsetUsQueue.isEmpty()) { @@ -178,6 +191,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { + frameProcessingStarted = true; long streamOffsetUs = checkStateNotNull(streamOffsetUsQueue.peek(), "No input stream specified."); long offsetPresentationTimeUs = presentationTimeUs + streamOffsetUs; @@ -199,6 +213,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } public void releaseOutputFrame(long releaseTimeNs) { + frameProcessingStarted = true; checkState(!releaseFramesAutomatically); Pair oldestAvailableFrame = availableFrames.remove(); renderFrameToSurfaces( @@ -209,6 +224,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void flush() { + frameProcessingStarted = true; // Drops all frames that aren't released yet. availableFrames.clear(); if (defaultShaderProgram != null) { @@ -304,7 +320,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputEglSurface, outputSurfaceInfo.width, outputSurfaceInfo.height); - GlUtil.clearOutputFrame(); + glObjectsProvider.clearOutputFrame(); defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs); EGLExt.eglPresentationTimeANDROID( @@ -446,7 +462,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; try { debugSurfaceViewWrapper.maybeRenderToSurfaceView( () -> { - GlUtil.clearOutputFrame(); + glObjectsProvider.clearOutputFrame(); @C.ColorTransfer int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer(); defaultShaderProgram.setOutputColorTransfer( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FrameCacheGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/FrameCacheGlShaderProgram.java index bd89b81e42..fa4571c422 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FrameCacheGlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FrameCacheGlShaderProgram.java @@ -19,7 +19,7 @@ import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; import android.opengl.GLES20; -import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlProgram; @@ -49,10 +49,12 @@ import java.util.concurrent.Executor; private final int capacity; private final boolean useHdr; + private GlObjectsProvider glObjectsProvider; private InputListener inputListener; private OutputListener outputListener; private ErrorListener errorListener; private Executor errorListenerExecutor; + private boolean frameProcessingStarted; /** Creates a new instance. */ public FrameCacheGlShaderProgram(Context context, int capacity, boolean useHdr) @@ -80,12 +82,21 @@ import java.util.concurrent.Executor; GlUtil.getNormalizedCoordinateBounds(), GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + glObjectsProvider = GlObjectsProvider.DEFAULT; inputListener = new InputListener() {}; outputListener = new OutputListener() {}; errorListener = videoFrameProcessingException -> {}; errorListenerExecutor = MoreExecutors.directExecutor(); } + @Override + public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) { + checkState( + !frameProcessingStarted, + "The GlObjectsProvider cannot be set after frame processing has started."); + this.glObjectsProvider = glObjectsProvider; + } + @Override public void setInputListener(InputListener inputListener) { this.inputListener = inputListener; @@ -115,6 +126,7 @@ import java.util.concurrent.Executor; @Override public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { + frameProcessingStarted = true; try { configureAllOutputTextures(inputTexture.width, inputTexture.height); @@ -125,7 +137,7 @@ import java.util.concurrent.Executor; // Copy frame to fbo. GlUtil.focusFramebufferUsingCurrentContext( outputTexture.fboId, outputTexture.width, outputTexture.height); - GlUtil.clearOutputFrame(); + glObjectsProvider.clearOutputFrame(); drawFrame(inputTexture.texId); inputListener.onInputFrameProcessed(inputTexture); outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs); @@ -147,6 +159,7 @@ import java.util.concurrent.Executor; @Override public void releaseOutputFrame(GlTextureInfo outputTexture) { + frameProcessingStarted = true; checkState(inUseOutputTextures.contains(outputTexture)); inUseOutputTextures.remove(outputTexture); freeOutputTextures.add(outputTexture); @@ -155,11 +168,13 @@ import java.util.concurrent.Executor; @Override public void signalEndOfCurrentInputStream() { + frameProcessingStarted = true; outputListener.onCurrentOutputStreamEnded(); } @Override public void flush() { + frameProcessingStarted = true; freeOutputTextures.addAll(inUseOutputTextures); inUseOutputTextures.clear(); inputListener.onFlush(); @@ -170,6 +185,7 @@ import java.util.concurrent.Executor; @Override public void release() throws VideoFrameProcessingException { + frameProcessingStarted = true; try { deleteAllOutputTextures(); } catch (GlUtil.GlException e) { @@ -196,9 +212,8 @@ import java.util.concurrent.Executor; checkState(inUseOutputTextures.isEmpty()); for (int i = 0; i < capacity; i++) { int outputTexId = GlUtil.createTexture(width, height, useHdr); - int outputFboId = GlUtil.createFboForTexture(outputTexId); GlTextureInfo outputTexture = - new GlTextureInfo(outputTexId, outputFboId, /* rboId= */ C.INDEX_UNSET, width, height); + glObjectsProvider.createBuffersForTexture(outputTexId, width, height); freeOutputTextures.add(outputTexture); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/GlShaderProgram.java index 0449e388cf..5cd37948de 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlShaderProgram.java @@ -15,6 +15,7 @@ */ package androidx.media3.effect; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.UnstableApi; @@ -148,6 +149,13 @@ public interface GlShaderProgram { */ void setErrorListener(Executor executor, ErrorListener errorListener); + /** + * Sets the {@link GlObjectsProvider}. + * + *

This method should not be called after any of the frame processing methods. + */ + void setGlObjectsProvider(GlObjectsProvider glObjectsProvider); + /** * Processes an input frame if possible. * diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlShaderProgram.java index 0b12226fa2..e42f1e1268 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlShaderProgram.java @@ -18,7 +18,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkState; import androidx.annotation.CallSuper; -import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; @@ -44,6 +44,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { private final boolean useHdr; + private GlObjectsProvider glObjectsProvider; private InputListener inputListener; private OutputListener outputListener; private ErrorListener errorListener; @@ -52,6 +53,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { private int inputHeight; private @MonotonicNonNull GlTextureInfo outputTexture; private boolean outputTextureInUse; + private boolean frameProcessingStarted; /** * Creates a {@code SingleFrameGlShaderProgram} instance. @@ -61,6 +63,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { */ public SingleFrameGlShaderProgram(boolean useHdr) { this.useHdr = useHdr; + glObjectsProvider = GlObjectsProvider.DEFAULT; inputListener = new InputListener() {}; outputListener = new OutputListener() {}; errorListener = (videoFrameProcessingException) -> {}; @@ -98,6 +101,14 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { public abstract void drawFrame(int inputTexId, long presentationTimeUs) throws VideoFrameProcessingException; + @Override + public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) { + checkState( + !frameProcessingStarted, + "The GlObjectsProvider cannot be set after frame processing has started."); + this.glObjectsProvider = glObjectsProvider; + } + @Override public final void setInputListener(InputListener inputListener) { this.inputListener = inputListener; @@ -123,7 +134,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { !outputTextureInUse, "The shader program does not currently accept input frames. Release prior output frames" + " first."); - + frameProcessingStarted = true; try { if (outputTexture == null || inputTexture.width != inputWidth @@ -133,7 +144,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { outputTextureInUse = true; GlUtil.focusFramebufferUsingCurrentContext( outputTexture.fboId, outputTexture.width, outputTexture.height); - GlUtil.clearOutputFrame(); + glObjectsProvider.clearOutputFrame(); drawFrame(inputTexture.texId, presentationTimeUs); inputListener.onInputFrameProcessed(inputTexture); outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs); @@ -161,25 +172,22 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { GlUtil.deleteFbo(outputTexture.fboId); } int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr); - int outputFboId = GlUtil.createFboForTexture(outputTexId); outputTexture = - new GlTextureInfo( - outputTexId, - outputFboId, - /* rboId= */ C.INDEX_UNSET, - outputSize.getWidth(), - outputSize.getHeight()); + glObjectsProvider.createBuffersForTexture( + outputTexId, outputSize.getWidth(), outputSize.getHeight()); } } @Override public final void releaseOutputFrame(GlTextureInfo outputTexture) { outputTextureInUse = false; + frameProcessingStarted = true; inputListener.onReadyToAcceptInputFrame(); } @Override public final void signalEndOfCurrentInputStream() { + frameProcessingStarted = true; outputListener.onCurrentOutputStreamEnded(); } @@ -187,6 +195,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { @CallSuper public void flush() { outputTextureInUse = false; + frameProcessingStarted = true; inputListener.onFlush(); inputListener.onReadyToAcceptInputFrame(); } @@ -194,6 +203,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { @Override @CallSuper public void release() throws VideoFrameProcessingException { + frameProcessingStarted = true; if (outputTexture != null) { try { GlUtil.deleteTexture(outputTexture.texId); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java index a11d7f70f3..badb88d5a2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; import androidx.media3.common.Effect; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.MediaItem; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.audio.AudioProcessor; @@ -49,15 +50,25 @@ public final class Effects { * applying the {@code videoEffects} to the video frames. */ public final VideoFrameProcessor.Factory videoFrameProcessorFactory; + /** + * The {@link GlObjectsProvider} used to create and maintain certain GL Objects in the {@link + * VideoFrameProcessor}. + */ + public final GlObjectsProvider glObjectsProvider; /** * Creates an instance using a {@link DefaultVideoFrameProcessor.Factory}. * *

This is equivalent to calling {@link Effects#Effects(List, List, - * VideoFrameProcessor.Factory)} with a {@link DefaultVideoFrameProcessor.Factory}. + * VideoFrameProcessor.Factory, GlObjectsProvider)} with a {@link + * DefaultVideoFrameProcessor.Factory} and {@link GlObjectsProvider#DEFAULT}. */ public Effects(List audioProcessors, List videoEffects) { - this(audioProcessors, videoEffects, new DefaultVideoFrameProcessor.Factory()); + this( + audioProcessors, + videoEffects, + new DefaultVideoFrameProcessor.Factory(), + GlObjectsProvider.DEFAULT); } /** @@ -66,13 +77,16 @@ public final class Effects { * @param audioProcessors The {@link #audioProcessors}. * @param videoEffects The {@link #videoEffects}. * @param videoFrameProcessorFactory The {@link #videoFrameProcessorFactory}. + * @param glObjectsProvider The {@link GlObjectsProvider}. */ public Effects( List audioProcessors, List videoEffects, - VideoFrameProcessor.Factory videoFrameProcessorFactory) { + VideoFrameProcessor.Factory videoFrameProcessorFactory, + GlObjectsProvider glObjectsProvider) { this.audioProcessors = ImmutableList.copyOf(audioProcessors); this.videoEffects = ImmutableList.copyOf(videoEffects); this.videoFrameProcessorFactory = videoFrameProcessorFactory; + this.glObjectsProvider = glObjectsProvider; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index b7e8baee75..9fc1527926 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -28,6 +28,7 @@ import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.MediaItem; import androidx.media3.common.MediaLibraryInfo; import androidx.media3.common.MimeTypes; @@ -88,6 +89,7 @@ public final class Transformer { private ListenerSet listeners; private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory; private VideoFrameProcessor.Factory videoFrameProcessorFactory; + private GlObjectsProvider glObjectsProvider; private Codec.EncoderFactory encoderFactory; private Muxer.Factory muxerFactory; private Looper looper; @@ -105,6 +107,7 @@ public final class Transformer { audioProcessors = ImmutableList.of(); videoEffects = ImmutableList.of(); videoFrameProcessorFactory = new DefaultVideoFrameProcessor.Factory(); + glObjectsProvider = GlObjectsProvider.DEFAULT; encoderFactory = new DefaultEncoderFactory.Builder(this.context).build(); muxerFactory = new DefaultMuxer.Factory(); looper = Util.getCurrentOrMainLooper(); @@ -124,6 +127,7 @@ public final class Transformer { this.listeners = transformer.listeners; this.assetLoaderFactory = transformer.assetLoaderFactory; this.videoFrameProcessorFactory = transformer.videoFrameProcessorFactory; + this.glObjectsProvider = transformer.glObjectsProvider; this.encoderFactory = transformer.encoderFactory; this.muxerFactory = transformer.muxerFactory; this.looper = transformer.looper; @@ -397,6 +401,7 @@ public final class Transformer { listeners, assetLoaderFactory, videoFrameProcessorFactory, + glObjectsProvider, encoderFactory, muxerFactory, looper, @@ -556,6 +561,7 @@ public final class Transformer { private final ListenerSet listeners; private final AssetLoader.Factory assetLoaderFactory; private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + private final GlObjectsProvider glObjectsProvider; private final Codec.EncoderFactory encoderFactory; private final Muxer.Factory muxerFactory; private final Looper looper; @@ -575,6 +581,7 @@ public final class Transformer { ListenerSet listeners, AssetLoader.Factory assetLoaderFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory, + GlObjectsProvider glObjectsProvider, Codec.EncoderFactory encoderFactory, Muxer.Factory muxerFactory, Looper looper, @@ -591,6 +598,7 @@ public final class Transformer { this.listeners = listeners; this.assetLoaderFactory = assetLoaderFactory; this.videoFrameProcessorFactory = videoFrameProcessorFactory; + this.glObjectsProvider = glObjectsProvider; this.encoderFactory = encoderFactory; this.muxerFactory = muxerFactory; this.looper = looper; @@ -794,7 +802,9 @@ public final class Transformer { .setRemoveAudio(removeAudio) .setRemoveVideo(removeVideo) .setFlattenForSlowMotion(flattenForSlowMotion) - .setEffects(new Effects(audioProcessors, videoEffects, videoFrameProcessorFactory)) + .setEffects( + new Effects( + audioProcessors, videoEffects, videoFrameProcessorFactory, glObjectsProvider)) .build(); start(editedMediaItem, path); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 74607b46f2..9bc5fbc6ea 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -533,6 +533,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; firstEditedMediaItem.effects.videoEffects, compositionPresentation, firstEditedMediaItem.effects.videoFrameProcessorFactory, + firstEditedMediaItem.effects.glObjectsProvider, encoderFactory, muxerWrapper, /* errorConsumer= */ this::onError, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index c6c08cff98..8b585ab589 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -35,6 +35,7 @@ import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.FrameInfo; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.MimeTypes; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; @@ -81,6 +82,7 @@ import org.checkerframework.dataflow.qual.Pure; ImmutableList effects, @Nullable Presentation presentation, VideoFrameProcessor.Factory videoFrameProcessorFactory, + GlObjectsProvider glObjectsProvider, Codec.EncoderFactory encoderFactory, MuxerWrapper muxerWrapper, Consumer errorConsumer, @@ -129,52 +131,54 @@ import org.checkerframework.dataflow.qual.Pure; } try { videoFrameProcessor = - videoFrameProcessorFactory.create( - context, - effectsWithPresentation, - debugViewProvider, - videoFrameProcessorInputColor, - videoFrameProcessorOutputColor, - MimeTypes.isVideo(firstInputFormat.sampleMimeType), - /* releaseFramesAutomatically= */ true, - MoreExecutors.directExecutor(), - new VideoFrameProcessor.Listener() { - private long lastProcessedFramePresentationTimeUs; + videoFrameProcessorFactory + .setGlObjectsProvider(glObjectsProvider) + .create( + context, + effectsWithPresentation, + debugViewProvider, + videoFrameProcessorInputColor, + videoFrameProcessorOutputColor, + MimeTypes.isVideo(firstInputFormat.sampleMimeType), + /* releaseFramesAutomatically= */ true, + MoreExecutors.directExecutor(), + new VideoFrameProcessor.Listener() { + private long lastProcessedFramePresentationTimeUs; - @Override - public void onOutputSizeChanged(int width, int height) { - try { - checkNotNull(videoFrameProcessor) - .setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height)); - } catch (ExportException exception) { - errorConsumer.accept(exception); - } - } + @Override + public void onOutputSizeChanged(int width, int height) { + try { + checkNotNull(videoFrameProcessor) + .setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height)); + } catch (ExportException exception) { + errorConsumer.accept(exception); + } + } - @Override - public void onOutputFrameAvailable(long presentationTimeUs) { - // Frames are released automatically. - lastProcessedFramePresentationTimeUs = presentationTimeUs; - } + @Override + public void onOutputFrameAvailable(long presentationTimeUs) { + // Frames are released automatically. + lastProcessedFramePresentationTimeUs = presentationTimeUs; + } - @Override - public void onError(VideoFrameProcessingException exception) { - errorConsumer.accept( - ExportException.createForVideoFrameProcessingException( - exception, ExportException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); - } + @Override + public void onError(VideoFrameProcessingException exception) { + errorConsumer.accept( + ExportException.createForVideoFrameProcessingException( + exception, ExportException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); + } - @Override - public void onEnded() { - VideoSamplePipeline.this.finalFramePresentationTimeUs = - lastProcessedFramePresentationTimeUs; - try { - encoderWrapper.signalEndOfInputStream(); - } catch (ExportException exception) { - errorConsumer.accept(exception); - } - } - }); + @Override + public void onEnded() { + VideoSamplePipeline.this.finalFramePresentationTimeUs = + lastProcessedFramePresentationTimeUs; + try { + encoderWrapper.signalEndOfInputStream(); + } catch (ExportException exception) { + errorConsumer.accept(exception); + } + } + }); } catch (VideoFrameProcessingException e) { throw ExportException.createForVideoFrameProcessingException( e, ExportException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED);