loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow FrameProcessor.Factory to be set on Transformer.Builder.

Browse Source 
Extract a FrameProcessor.Factory interface from GlEffectsFrameProcessor
and allow it to  be customized using a setter on Transformer.Builder.

PiperOrigin-RevId: 463433438
This commit is contained in:
hschlueter 2022-07-26 21:52:30 +00:00 committed by tonihei
parent 2bde3f1e31
commit 22822d8e19
6 changed files with 142 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
View File

@ -356,32 +356,33 @@ public final class GlEffectsFrameProcessorPixelTest {
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
glEffectsFrameProcessor =
checkNotNull(
GlEffectsFrameProcessor.create(
context,
new FrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
outputImageReader =
ImageReader.newInstance(
width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
checkNotNull(glEffectsFrameProcessor)
.setOutputSurfaceInfo(
new SurfaceInfo(outputImageReader.getSurface(), width, height));
}
new GlEffectsFrameProcessor.Factory()
.create(
context,
new FrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
outputImageReader =
ImageReader.newInstance(
width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
checkNotNull(glEffectsFrameProcessor)
.setOutputSurfaceInfo(
new SurfaceInfo(outputImageReader.getSurface(), width, height));
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
frameProcessingException.set(exception);
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
frameProcessingException.set(exception);
}
@Override
public void onFrameProcessingEnded() {
frameProcessingEnded = true;
}
},
effects,
DebugViewProvider.NONE,
/* useHdr= */ false));
@Override
public void onFrameProcessingEnded() {
frameProcessingEnded = true;
}
},
effects,
DebugViewProvider.NONE,
/* useHdr= */ false));
glEffectsFrameProcessor.setInputFrameInfo(
new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0));
glEffectsFrameProcessor.registerInputFrame();

44
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java
View File

@ -15,11 +15,51 @@
*/
package androidx.media3.transformer;
import android.content.Context;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.util.UnstableApi;
import java.util.List;
/**
* Interface for a frame processor that applies changes to individual video frames.
*
* <p>The changes are specified by {@link GlEffect} instances passed to the {@link Factory}.
*
* <p>The frame processor manages its input {@link Surface} which can be accessed via {@link
* #getInputSurface()}. The output {@link Surface} must be set by the caller using {@link
* #setOutputSurfaceInfo(SurfaceInfo)}.
*
* <p>The caller must {@linkplain #registerInputFrame() register} input frames before rendering them
* to the input {@link Surface}.
*/
@UnstableApi
public interface FrameProcessor {
// TODO(b/227625423): Allow effects to be replaced.
/** A factory for {@link FrameProcessor} instances. */
interface Factory {
/**
* Creates a new {@link FrameProcessor} instance.
*
* @param context A {@link Context}.
* @param listener A {@link Listener}.
* @param effects The {@link GlEffect} instances to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}.
* @param useHdr Whether to process the input as an HDR signal.
* @return A new instance.
* @throws FrameProcessingException If a problem occurs while creating the {@link
* FrameProcessor}.
*/
FrameProcessor create(
Context context,
FrameProcessor.Listener listener,
List<GlEffect> effects,
DebugViewProvider debugViewProvider,
boolean useHdr)
throws FrameProcessingException;
}
/** Interface for a frame processor that applies changes to individual video frames. */
/* package */ interface FrameProcessor {
/**
* Listener for asynchronous frame processing events.
*

76
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
View File

@ -29,6 +29,7 @@ import androidx.annotation.Nullable;
import androidx.annotation.WorkerThread;
import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import java.util.List;
@ -42,51 +43,48 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a
* background thread.
*/
/* package */ final class GlEffectsFrameProcessor implements FrameProcessor {
@UnstableApi
public final class GlEffectsFrameProcessor implements FrameProcessor {
// TODO(b/227625423): Replace factory method with setters once output surface and effects can be
// replaced.
/**
* Creates a new instance.
*
* @param context A {@link Context}.
* @param listener A {@link Listener}.
* @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}.
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
* EXT_YUV_target} OpenGL extension.
* @return A new instance.
* @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while
* creating and configuring the OpenGL components.
*/
public static GlEffectsFrameProcessor create(
Context context,
FrameProcessor.Listener listener,
List<GlEffect> effects,
DebugViewProvider debugViewProvider,
boolean useHdr)
throws FrameProcessingException {
/** A factory for {@link GlEffectsFrameProcessor} instances. */
public static class Factory implements FrameProcessor.Factory {
/**
* {@inheritDoc}
*
* <p>Using HDR requires the {@code EXT_YUV_target} OpenGL extension.
*/
@Override
public GlEffectsFrameProcessor create(
Context context,
FrameProcessor.Listener listener,
List<GlEffect> effects,
DebugViewProvider debugViewProvider,
boolean useHdr)
throws FrameProcessingException {
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
Future<GlEffectsFrameProcessor> glFrameProcessorFuture =
singleThreadExecutorService.submit(
() ->
createOpenGlObjectsAndFrameProcessor(
context,
listener,
effects,
debugViewProvider,
useHdr,
singleThreadExecutorService));
Future<GlEffectsFrameProcessor> glFrameProcessorFuture =
singleThreadExecutorService.submit(
() ->
createOpenGlObjectsAndFrameProcessor(
context,
listener,
effects,
debugViewProvider,
useHdr,
singleThreadExecutorService));
try {
return glFrameProcessorFuture.get();
} catch (ExecutionException e) {
throw new FrameProcessingException(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FrameProcessingException(e);
try {
return glFrameProcessorFuture.get();
} catch (ExecutionException e) {
throw new FrameProcessingException(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FrameProcessingException(e);
}
}
}

32
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -104,6 +104,7 @@ public final class Transformer {
private String containerMimeType;
private TransformationRequest transformationRequest;
private ImmutableList<GlEffect> videoEffects;
private FrameProcessor.Factory frameProcessorFactory;
private ListenerSet<Transformer.Listener> listeners;
private DebugViewProvider debugViewProvider;
private Looper looper;
@ -128,6 +129,7 @@ public final class Transformer {
containerMimeType = MimeTypes.VIDEO_MP4;
transformationRequest = new TransformationRequest.Builder().build();
videoEffects = ImmutableList.of();
frameProcessorFactory = new GlEffectsFrameProcessor.Factory();
}
/** Creates a builder with the values of the provided {@link Transformer}. */
@ -140,6 +142,7 @@ public final class Transformer {
this.containerMimeType = transformer.containerMimeType;
this.transformationRequest = transformer.transformationRequest;
this.videoEffects = transformer.videoEffects;
this.frameProcessorFactory = transformer.frameProcessorFactory;
this.listeners = transformer.listeners;
this.looper = transformer.looper;
this.encoderFactory = transformer.encoderFactory;
@ -183,6 +186,26 @@ public final class Transformer {
return this;
}
/**
* Sets the {@link FrameProcessor.Factory} for the {@link FrameProcessor} to use when applying
* {@linkplain GlEffect effects} to the video frames.
*
* <p>This factory will be used to create the {@link FrameProcessor} used for applying the
* {@link GlEffect} instances passed to {@link #setVideoEffects(List<GlEffect>)} and any
* additional {@link GlMatrixTransformation} instances derived from the {@link
* TransformationRequest} set using {@link #setTransformationRequest(TransformationRequest)}.
*
* <p>The default is {@link GlEffectsFrameProcessor.Factory}.
*
* @param frameProcessorFactory The {@link FrameProcessor.Factory} to use.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setFrameProcessorFactory(FrameProcessor.Factory frameProcessorFactory) {
this.frameProcessorFactory = frameProcessorFactory;
return this;
}
/**
* Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform.
*
@ -440,6 +463,7 @@ public final class Transformer {
containerMimeType,
transformationRequest,
videoEffects,
frameProcessorFactory,
listeners,
looper,
clock,
@ -548,6 +572,7 @@ public final class Transformer {
private final String containerMimeType;
private final TransformationRequest transformationRequest;
private final ImmutableList<GlEffect> videoEffects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Looper looper;
private final Clock clock;
private final DebugViewProvider debugViewProvider;
@ -569,6 +594,7 @@ public final class Transformer {
String containerMimeType,
TransformationRequest transformationRequest,
ImmutableList<GlEffect> videoEffects,
FrameProcessor.Factory frameProcessorFactory,
ListenerSet<Transformer.Listener> listeners,
Looper looper,
Clock clock,
@ -584,6 +610,7 @@ public final class Transformer {
this.containerMimeType = containerMimeType;
this.transformationRequest = transformationRequest;
this.videoEffects = videoEffects;
this.frameProcessorFactory = frameProcessorFactory;
this.listeners = listeners;
this.looper = looper;
this.clock = clock;
@ -733,6 +760,7 @@ public final class Transformer {
transformationRequest,
mediaItem.clippingConfiguration.startsAtKeyFrame,
videoEffects,
frameProcessorFactory,
encoderFactory,
decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest),
@ -847,6 +875,7 @@ public final class Transformer {
private final TransformationRequest transformationRequest;
private final boolean clippingStartsAtKeyFrame;
private final ImmutableList<GlEffect> videoEffects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener;
@ -861,6 +890,7 @@ public final class Transformer {
TransformationRequest transformationRequest,
boolean clippingStartsAtKeyFrame,
ImmutableList<GlEffect> videoEffects,
FrameProcessor.Factory frameProcessorFactory,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
@ -873,6 +903,7 @@ public final class Transformer {
this.transformationRequest = transformationRequest;
this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame;
this.videoEffects = videoEffects;
this.frameProcessorFactory = frameProcessorFactory;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener;
@ -912,6 +943,7 @@ public final class Transformer {
transformationRequest,
clippingStartsAtKeyFrame,
videoEffects,
frameProcessorFactory,
encoderFactory,
decoderFactory,
asyncErrorListener,

4
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
View File

@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final Context context;
private final boolean clippingStartsAtKeyFrame;
private final ImmutableList<GlEffect> effects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final DebugViewProvider debugViewProvider;
@ -53,6 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
TransformationRequest transformationRequest,
boolean clippingStartsAtKeyFrame,
ImmutableList<GlEffect> effects,
FrameProcessor.Factory frameProcessorFactory,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
Transformer.AsyncErrorListener asyncErrorListener,
@ -68,6 +70,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
this.context = context;
this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame;
this.effects = effects;
this.frameProcessorFactory = frameProcessorFactory;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.debugViewProvider = debugViewProvider;
@ -113,6 +116,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
streamOffsetUs,
transformationRequest,
effects,
frameProcessorFactory,
decoderFactory,
encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),

3
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -58,6 +58,7 @@ import org.checkerframework.dataflow.qual.Pure;
long streamOffsetUs,
TransformationRequest transformationRequest,
ImmutableList<GlEffect> effects,
FrameProcessor.Factory frameProcessorFactory,
Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes,
@ -102,7 +103,7 @@ import org.checkerframework.dataflow.qual.Pure;
try {
frameProcessor =
GlEffectsFrameProcessor.create(
frameProcessorFactory.create(
context,
new FrameProcessor.Listener() {
@Override