loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update effect to take in and use a GlObjectsProvider

Browse Source 
PiperOrigin-RevId: 514744747
This commit is contained in:
tofunmi 2023-03-07 16:46:46 +00:00 committed by tonihei
parent 20669fca27
commit e478d81b52
12 changed files with 188 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeShaderProgram.java
View File

@ -24,6 +24,7 @@ import android.content.Context;
import android.opengl.EGL14;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.LibraryLoader;
@ -159,6 +160,9 @@ import java.util.concurrent.Future;
() -> errorListener.onError(new VideoFrameProcessingException(error))));
}
@Override
public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {}
@Override
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
AppTextureFrame appTextureFrame =

9
libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java
View File

@ -42,6 +42,15 @@ public interface VideoFrameProcessor {
/** A factory for {@link VideoFrameProcessor} instances. */
interface Factory {
/**
* Sets the {@link GlObjectsProvider}.
*
* <p>Must be called before {@link #create}.
*/
Factory setGlObjectsProvider(GlObjectsProvider glObjectsProvider);
// TODO(271433904): Turn parameters with default values into setters.
/**
* Creates a new {@link VideoFrameProcessor} instance.
*

4
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java
View File

@ -28,6 +28,7 @@ import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException;
@ -410,6 +411,9 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest {
@Override
public void setErrorListener(Executor executor, ErrorListener errorListener) {}
@Override
public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {}
@Override
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
// No input is queued in these tests. The BlankFrameProducer is used to produce frames.

34
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -36,6 +36,7 @@ import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
@ -62,6 +63,20 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
/** A factory for {@link DefaultVideoFrameProcessor} instances. */
public static final class Factory implements VideoFrameProcessor.Factory {
private GlObjectsProvider glObjectsProvider = GlObjectsProvider.DEFAULT;
/**
* {@inheritDoc}
*
* <p>The default value is {@link GlObjectsProvider#DEFAULT}.
*/
@Override
public DefaultVideoFrameProcessor.Factory setGlObjectsProvider(
GlObjectsProvider glObjectsProvider) {
this.glObjectsProvider = glObjectsProvider;
return this;
}
/**
* {@inheritDoc}
*
@ -139,7 +154,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
releaseFramesAutomatically,
singleThreadExecutorService,
listenerExecutor,
listener));
listener,
glObjectsProvider));
try {
return defaultVideoFrameProcessorFuture.get();
@ -367,7 +383,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
boolean releaseFramesAutomatically,
ExecutorService singleThreadExecutorService,
Executor executor,
Listener listener)
Listener listener,
GlObjectsProvider glObjectsProvider)
throws GlUtil.GlException, VideoFrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
@ -380,7 +397,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
: GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888;
int openGlVersion =
ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo) ? 3 : 2;
EGLContext eglContext = GlUtil.createEglContext(eglDisplay, openGlVersion, configAttributes);
EGLContext eglContext =
glObjectsProvider.createEglContext(eglDisplay, openGlVersion, configAttributes);
GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay, configAttributes);
// Not releaseFramesAutomatically means outputting to a display surface. HDR display surfaces
@ -409,6 +427,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
releaseFramesAutomatically,
executor,
listener);
setGlObjectProviderOnShaderPrograms(shaderPrograms, glObjectsProvider);
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener);
chainShaderProgramsWithListeners(
@ -522,6 +541,15 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return shaderProgramListBuilder.build();
}
/** Sets the {@link GlObjectsProvider} on all of the {@linkplain GlShaderProgram}s provided. */
private static void setGlObjectProviderOnShaderPrograms(
ImmutableList<GlShaderProgram> shaderPrograms, GlObjectsProvider glObjectsProvider) {
for (int i = 0; i < shaderPrograms.size() - 1; i++) {
GlShaderProgram shaderProgram = shaderPrograms.get(i);
shaderProgram.setGlObjectsProvider(glObjectsProvider);
}
}
/**
* Chains the given {@link GlShaderProgram} instances using {@link
* ChainingGlShaderProgramListener} instances.

20
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -35,6 +35,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException;
@ -86,9 +87,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private int inputHeight;
@Nullable private DefaultShaderProgram defaultShaderProgram;
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
private GlObjectsProvider glObjectsProvider;
private InputListener inputListener;
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
@Nullable private SurfaceView debugSurfaceView;
private boolean frameProcessingStarted;
private volatile boolean outputSizeOrRotationChanged;
@ -130,10 +133,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
textureTransformMatrix = GlUtil.create4x4IdentityMatrix();
streamOffsetUsQueue = new ConcurrentLinkedQueue<>();
glObjectsProvider = GlObjectsProvider.DEFAULT;
inputListener = new InputListener() {};
availableFrames = new ConcurrentLinkedQueue<>();
}
@Override
public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
checkState(
!frameProcessingStarted,
"The GlObjectsProvider cannot be set after frame processing has started.");
this.glObjectsProvider = glObjectsProvider;
}
@Override
public void setInputListener(InputListener inputListener) {
this.inputListener = inputListener;
@ -154,6 +166,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void signalEndOfCurrentInputStream() {
frameProcessingStarted = true;
checkState(!streamOffsetUsQueue.isEmpty(), "No input stream to end.");
streamOffsetUsQueue.remove();
if (streamOffsetUsQueue.isEmpty()) {
@ -178,6 +191,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
frameProcessingStarted = true;
long streamOffsetUs =
checkStateNotNull(streamOffsetUsQueue.peek(), "No input stream specified.");
long offsetPresentationTimeUs = presentationTimeUs + streamOffsetUs;
@ -199,6 +213,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
public void releaseOutputFrame(long releaseTimeNs) {
frameProcessingStarted = true;
checkState(!releaseFramesAutomatically);
Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
renderFrameToSurfaces(
@ -209,6 +224,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void flush() {
frameProcessingStarted = true;
// Drops all frames that aren't released yet.
availableFrames.clear();
if (defaultShaderProgram != null) {
@ -304,7 +320,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputEglSurface,
outputSurfaceInfo.width,
outputSurfaceInfo.height);
GlUtil.clearOutputFrame();
glObjectsProvider.clearOutputFrame();
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(
@ -446,7 +462,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
try {
debugSurfaceViewWrapper.maybeRenderToSurfaceView(
() -> {
GlUtil.clearOutputFrame();
glObjectsProvider.clearOutputFrame();
@C.ColorTransfer
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
defaultShaderProgram.setOutputColorTransfer(

23
libraries/effect/src/main/java/androidx/media3/effect/FrameCacheGlShaderProgram.java
View File

@ -19,7 +19,7 @@ import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import android.opengl.GLES20;
import androidx.media3.common.C;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.GlProgram;
@ -49,10 +49,12 @@ import java.util.concurrent.Executor;
private final int capacity;
private final boolean useHdr;
private GlObjectsProvider glObjectsProvider;
private InputListener inputListener;
private OutputListener outputListener;
private ErrorListener errorListener;
private Executor errorListenerExecutor;
private boolean frameProcessingStarted;
/** Creates a new instance. */
public FrameCacheGlShaderProgram(Context context, int capacity, boolean useHdr)
@ -80,12 +82,21 @@ import java.util.concurrent.Executor;
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glObjectsProvider = GlObjectsProvider.DEFAULT;
inputListener = new InputListener() {};
outputListener = new OutputListener() {};
errorListener = videoFrameProcessingException -> {};
errorListenerExecutor = MoreExecutors.directExecutor();
}
@Override
public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
checkState(
!frameProcessingStarted,
"The GlObjectsProvider cannot be set after frame processing has started.");
this.glObjectsProvider = glObjectsProvider;
}
@Override
public void setInputListener(InputListener inputListener) {
this.inputListener = inputListener;
@ -115,6 +126,7 @@ import java.util.concurrent.Executor;
@Override
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
frameProcessingStarted = true;
try {
configureAllOutputTextures(inputTexture.width, inputTexture.height);
@ -125,7 +137,7 @@ import java.util.concurrent.Executor;
// Copy frame to fbo.
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame();
glObjectsProvider.clearOutputFrame();
drawFrame(inputTexture.texId);
inputListener.onInputFrameProcessed(inputTexture);
outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
@ -147,6 +159,7 @@ import java.util.concurrent.Executor;
@Override
public void releaseOutputFrame(GlTextureInfo outputTexture) {
frameProcessingStarted = true;
checkState(inUseOutputTextures.contains(outputTexture));
inUseOutputTextures.remove(outputTexture);
freeOutputTextures.add(outputTexture);
@ -155,11 +168,13 @@ import java.util.concurrent.Executor;
@Override
public void signalEndOfCurrentInputStream() {
frameProcessingStarted = true;
outputListener.onCurrentOutputStreamEnded();
}
@Override
public void flush() {
frameProcessingStarted = true;
freeOutputTextures.addAll(inUseOutputTextures);
inUseOutputTextures.clear();
inputListener.onFlush();
@ -170,6 +185,7 @@ import java.util.concurrent.Executor;
@Override
public void release() throws VideoFrameProcessingException {
frameProcessingStarted = true;
try {
deleteAllOutputTextures();
} catch (GlUtil.GlException e) {
@ -196,9 +212,8 @@ import java.util.concurrent.Executor;
checkState(inUseOutputTextures.isEmpty());
for (int i = 0; i < capacity; i++) {
int outputTexId = GlUtil.createTexture(width, height, useHdr);
int outputFboId = GlUtil.createFboForTexture(outputTexId);
GlTextureInfo outputTexture =
new GlTextureInfo(outputTexId, outputFboId, /* rboId= */ C.INDEX_UNSET, width, height);
glObjectsProvider.createBuffersForTexture(outputTexId, width, height);
freeOutputTextures.add(outputTexture);
}
}

8
libraries/effect/src/main/java/androidx/media3/effect/GlShaderProgram.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.UnstableApi;
@ -148,6 +149,13 @@ public interface GlShaderProgram {
*/
void setErrorListener(Executor executor, ErrorListener errorListener);
/**
* Sets the {@link GlObjectsProvider}.
*
* <p>This method should not be called after any of the frame processing methods.
*/
void setGlObjectsProvider(GlObjectsProvider glObjectsProvider);
/**
* Processes an input frame if possible.
*

30
libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlShaderProgram.java
View File

@ -18,7 +18,7 @@ package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkState;
import androidx.annotation.CallSuper;
import androidx.media3.common.C;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.GlUtil;
@ -44,6 +44,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
private final boolean useHdr;
private GlObjectsProvider glObjectsProvider;
private InputListener inputListener;
private OutputListener outputListener;
private ErrorListener errorListener;
@ -52,6 +53,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
private int inputHeight;
private @MonotonicNonNull GlTextureInfo outputTexture;
private boolean outputTextureInUse;
private boolean frameProcessingStarted;
/**
* Creates a {@code SingleFrameGlShaderProgram} instance.
@ -61,6 +63,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
*/
public SingleFrameGlShaderProgram(boolean useHdr) {
this.useHdr = useHdr;
glObjectsProvider = GlObjectsProvider.DEFAULT;
inputListener = new InputListener() {};
outputListener = new OutputListener() {};
errorListener = (videoFrameProcessingException) -> {};
@ -98,6 +101,14 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
public abstract void drawFrame(int inputTexId, long presentationTimeUs)
throws VideoFrameProcessingException;
@Override
public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
checkState(
!frameProcessingStarted,
"The GlObjectsProvider cannot be set after frame processing has started.");
this.glObjectsProvider = glObjectsProvider;
}
@Override
public final void setInputListener(InputListener inputListener) {
this.inputListener = inputListener;
@ -123,7 +134,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
!outputTextureInUse,
"The shader program does not currently accept input frames. Release prior output frames"
+ " first.");
frameProcessingStarted = true;
try {
if (outputTexture == null
|| inputTexture.width != inputWidth
@ -133,7 +144,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
outputTextureInUse = true;
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame();
glObjectsProvider.clearOutputFrame();
drawFrame(inputTexture.texId, presentationTimeUs);
inputListener.onInputFrameProcessed(inputTexture);
outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
@ -161,25 +172,22 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
GlUtil.deleteFbo(outputTexture.fboId);
}
int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr);
int outputFboId = GlUtil.createFboForTexture(outputTexId);
outputTexture =
new GlTextureInfo(
outputTexId,
outputFboId,
/* rboId= */ C.INDEX_UNSET,
outputSize.getWidth(),
outputSize.getHeight());
glObjectsProvider.createBuffersForTexture(
outputTexId, outputSize.getWidth(), outputSize.getHeight());
}
}
@Override
public final void releaseOutputFrame(GlTextureInfo outputTexture) {
outputTextureInUse = false;
frameProcessingStarted = true;
inputListener.onReadyToAcceptInputFrame();
}
@Override
public final void signalEndOfCurrentInputStream() {
frameProcessingStarted = true;
outputListener.onCurrentOutputStreamEnded();
}
@ -187,6 +195,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
@CallSuper
public void flush() {
outputTextureInUse = false;
frameProcessingStarted = true;
inputListener.onFlush();
inputListener.onReadyToAcceptInputFrame();
}
@ -194,6 +203,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
@Override
@CallSuper
public void release() throws VideoFrameProcessingException {
frameProcessingStarted = true;
if (outputTexture != null) {
try {
GlUtil.deleteTexture(outputTexture.texId);

20
libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import androidx.media3.common.Effect;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.MediaItem;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.audio.AudioProcessor;
@ -49,15 +50,25 @@ public final class Effects {
* applying the {@code videoEffects} to the video frames.
*/
public final VideoFrameProcessor.Factory videoFrameProcessorFactory;
/**
* The {@link GlObjectsProvider} used to create and maintain certain GL Objects in the {@link
* VideoFrameProcessor}.
*/
public final GlObjectsProvider glObjectsProvider;
/**
* Creates an instance using a {@link DefaultVideoFrameProcessor.Factory}.
*
* <p>This is equivalent to calling {@link Effects#Effects(List, List,
* VideoFrameProcessor.Factory)} with a {@link DefaultVideoFrameProcessor.Factory}.
* VideoFrameProcessor.Factory, GlObjectsProvider)} with a {@link
* DefaultVideoFrameProcessor.Factory} and {@link GlObjectsProvider#DEFAULT}.
*/
public Effects(List<AudioProcessor> audioProcessors, List<Effect> videoEffects) {
this(audioProcessors, videoEffects, new DefaultVideoFrameProcessor.Factory());
this(
audioProcessors,
videoEffects,
new DefaultVideoFrameProcessor.Factory(),
GlObjectsProvider.DEFAULT);
}
/**
@ -66,13 +77,16 @@ public final class Effects {
* @param audioProcessors The {@link #audioProcessors}.
* @param videoEffects The {@link #videoEffects}.
* @param videoFrameProcessorFactory The {@link #videoFrameProcessorFactory}.
* @param glObjectsProvider The {@link GlObjectsProvider}.
*/
public Effects(
List<AudioProcessor> audioProcessors,
List<Effect> videoEffects,
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
VideoFrameProcessor.Factory videoFrameProcessorFactory,
GlObjectsProvider glObjectsProvider) {
this.audioProcessors = ImmutableList.copyOf(audioProcessors);
this.videoEffects = ImmutableList.copyOf(videoEffects);
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
this.glObjectsProvider = glObjectsProvider;
}
}

12
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -28,6 +28,7 @@ import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MediaLibraryInfo;
import androidx.media3.common.MimeTypes;
@ -88,6 +89,7 @@ public final class Transformer {
private ListenerSet<Transformer.Listener> listeners;
private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory;
private VideoFrameProcessor.Factory videoFrameProcessorFactory;
private GlObjectsProvider glObjectsProvider;
private Codec.EncoderFactory encoderFactory;
private Muxer.Factory muxerFactory;
private Looper looper;
@ -105,6 +107,7 @@ public final class Transformer {
audioProcessors = ImmutableList.of();
videoEffects = ImmutableList.of();
videoFrameProcessorFactory = new DefaultVideoFrameProcessor.Factory();
glObjectsProvider = GlObjectsProvider.DEFAULT;
encoderFactory = new DefaultEncoderFactory.Builder(this.context).build();
muxerFactory = new DefaultMuxer.Factory();
looper = Util.getCurrentOrMainLooper();
@ -124,6 +127,7 @@ public final class Transformer {
this.listeners = transformer.listeners;
this.assetLoaderFactory = transformer.assetLoaderFactory;
this.videoFrameProcessorFactory = transformer.videoFrameProcessorFactory;
this.glObjectsProvider = transformer.glObjectsProvider;
this.encoderFactory = transformer.encoderFactory;
this.muxerFactory = transformer.muxerFactory;
this.looper = transformer.looper;
@ -397,6 +401,7 @@ public final class Transformer {
listeners,
assetLoaderFactory,
videoFrameProcessorFactory,
glObjectsProvider,
encoderFactory,
muxerFactory,
looper,
@ -556,6 +561,7 @@ public final class Transformer {
private final ListenerSet<Transformer.Listener> listeners;
private final AssetLoader.Factory assetLoaderFactory;
private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
private final GlObjectsProvider glObjectsProvider;
private final Codec.EncoderFactory encoderFactory;
private final Muxer.Factory muxerFactory;
private final Looper looper;
@ -575,6 +581,7 @@ public final class Transformer {
ListenerSet<Listener> listeners,
AssetLoader.Factory assetLoaderFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
GlObjectsProvider glObjectsProvider,
Codec.EncoderFactory encoderFactory,
Muxer.Factory muxerFactory,
Looper looper,
@ -591,6 +598,7 @@ public final class Transformer {
this.listeners = listeners;
this.assetLoaderFactory = assetLoaderFactory;
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
this.glObjectsProvider = glObjectsProvider;
this.encoderFactory = encoderFactory;
this.muxerFactory = muxerFactory;
this.looper = looper;
@ -794,7 +802,9 @@ public final class Transformer {
.setRemoveAudio(removeAudio)
.setRemoveVideo(removeVideo)
.setFlattenForSlowMotion(flattenForSlowMotion)
.setEffects(new Effects(audioProcessors, videoEffects, videoFrameProcessorFactory))
.setEffects(
new Effects(
audioProcessors, videoEffects, videoFrameProcessorFactory, glObjectsProvider))
.build();
start(editedMediaItem, path);
}

1
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -533,6 +533,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
firstEditedMediaItem.effects.videoEffects,
compositionPresentation,
firstEditedMediaItem.effects.videoFrameProcessorFactory,
firstEditedMediaItem.effects.glObjectsProvider,
encoderFactory,
muxerWrapper,
/* errorConsumer= */ this::onError,

88
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
View File

@ -35,6 +35,7 @@ import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException;
@ -81,6 +82,7 @@ import org.checkerframework.dataflow.qual.Pure;
ImmutableList<Effect> effects,
@Nullable Presentation presentation,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
GlObjectsProvider glObjectsProvider,
Codec.EncoderFactory encoderFactory,
MuxerWrapper muxerWrapper,
Consumer<ExportException> errorConsumer,
@ -129,52 +131,54 @@ import org.checkerframework.dataflow.qual.Pure;
}
try {
videoFrameProcessor =
videoFrameProcessorFactory.create(
context,
effectsWithPresentation,
debugViewProvider,
videoFrameProcessorInputColor,
videoFrameProcessorOutputColor,
MimeTypes.isVideo(firstInputFormat.sampleMimeType),
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {
private long lastProcessedFramePresentationTimeUs;
videoFrameProcessorFactory
.setGlObjectsProvider(glObjectsProvider)
.create(
context,
effectsWithPresentation,
debugViewProvider,
videoFrameProcessorInputColor,
videoFrameProcessorOutputColor,
MimeTypes.isVideo(firstInputFormat.sampleMimeType),
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {
private long lastProcessedFramePresentationTimeUs;
@Override
public void onOutputSizeChanged(int width, int height) {
try {
checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height));
} catch (ExportException exception) {
errorConsumer.accept(exception);
}
}
@Override
public void onOutputSizeChanged(int width, int height) {
try {
checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height));
} catch (ExportException exception) {
errorConsumer.accept(exception);
}
}
@Override
public void onOutputFrameAvailable(long presentationTimeUs) {
// Frames are released automatically.
lastProcessedFramePresentationTimeUs = presentationTimeUs;
}
@Override
public void onOutputFrameAvailable(long presentationTimeUs) {
// Frames are released automatically.
lastProcessedFramePresentationTimeUs = presentationTimeUs;
}
@Override
public void onError(VideoFrameProcessingException exception) {
errorConsumer.accept(
ExportException.createForVideoFrameProcessingException(
exception, ExportException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED));
}
@Override
public void onError(VideoFrameProcessingException exception) {
errorConsumer.accept(
ExportException.createForVideoFrameProcessingException(
exception, ExportException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED));
}
@Override
public void onEnded() {
VideoSamplePipeline.this.finalFramePresentationTimeUs =
lastProcessedFramePresentationTimeUs;
try {
encoderWrapper.signalEndOfInputStream();
} catch (ExportException exception) {
errorConsumer.accept(exception);
}
}
});
@Override
public void onEnded() {
VideoSamplePipeline.this.finalFramePresentationTimeUs =
lastProcessedFramePresentationTimeUs;
try {
encoderWrapper.signalEndOfInputStream();
} catch (ExportException exception) {
errorConsumer.accept(exception);
}
}
});
} catch (VideoFrameProcessingException e) {
throw ExportException.createForVideoFrameProcessingException(
e, ExportException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED);