loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow frame processors to be set on Transformer.Builder.

Browse Source 
Also make GlFrameProcessor, ScaleToFitFrameProcessor, and
AdvancedFrameProcessor public.

PiperOrigin-RevId: 437227388
This commit is contained in:
hschlueter 2022-03-25 12:58:43 +00:00 committed by Ian Baker
parent d5056072a8
commit 0f5686fe07
8 changed files with 61 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libraries/transformer/src/main/java/androidx/media3/transformer/AdvancedFrameProcessor.java
View File

@ -23,6 +23,7 @@ import android.opengl.GLES20;
import android.util.Size;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -32,7 +33,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* ScaleToFitFrameProcessor}) are applied on the transformation. Width and height are not modified.
* The background color will default to black.
*/
/* package */ final class AdvancedFrameProcessor implements GlFrameProcessor {
@UnstableApi
public final class AdvancedFrameProcessor implements GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;

2
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
View File

@ -152,7 +152,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
this.frameProcessors = frameProcessors;
this.frameProcessors = ImmutableList.copyOf(frameProcessors);
try {
eglDisplay = GlUtil.createEglDisplay();

4
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import android.util.Size;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
/**
@ -31,7 +32,8 @@ import java.io.IOException;
* <li>{@link #release()}, upon conclusion of processing.
* </ol>
*/
/* package */ interface GlFrameProcessor {
@UnstableApi
public interface GlFrameProcessor {
// TODO(b/214975934): Investigate whether all configuration can be moved to initialize by
// using a placeholder surface until the encoder surface is known. If so, convert
// configureOutputSize to a simple getter.

4
libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java
View File

@ -24,12 +24,14 @@ import android.util.Size;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Controls how a frame is viewed, by changing resolution. */
// TODO(b/213190310): Implement crop, aspect ratio changes, etc.
/* package */ final class PresentationFrameProcessor implements GlFrameProcessor {
@UnstableApi
public final class PresentationFrameProcessor implements GlFrameProcessor {
/** A builder for {@link PresentationFrameProcessor} instances. */
public static final class Builder {

4
libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
View File

@ -24,6 +24,7 @@ import android.graphics.Matrix;
import android.util.Size;
import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -32,7 +33,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* preserved, potentially changing the width and height of the frame by scaling dimensions to fit.
* The background color will default to black.
*/
/* package */ final class ScaleToFitFrameProcessor implements GlFrameProcessor {
@UnstableApi
public final class ScaleToFitFrameProcessor implements GlFrameProcessor {
/** A builder for {@link ScaleToFitFrameProcessor} instances. */
public static final class Builder {

35
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -59,11 +59,13 @@ import androidx.media3.exoplayer.trackselection.DefaultTrackSelector;
import androidx.media3.exoplayer.video.VideoRendererEventListener;
import androidx.media3.extractor.DefaultExtractorsFactory;
import androidx.media3.extractor.mp4.Mp4Extractor;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
@ -102,6 +104,7 @@ public final class Transformer {
private boolean removeVideo;
private String containerMimeType;
private TransformationRequest transformationRequest;
private ImmutableList<GlFrameProcessor> frameProcessors;
private ListenerSet<Transformer.Listener> listeners;
private DebugViewProvider debugViewProvider;
private Looper looper;
@ -121,6 +124,7 @@ public final class Transformer {
debugViewProvider = DebugViewProvider.NONE;
containerMimeType = MimeTypes.VIDEO_MP4;
transformationRequest = new TransformationRequest.Builder().build();
frameProcessors = ImmutableList.of();
}
/**
@ -137,7 +141,8 @@ public final class Transformer {
encoderFactory = Codec.EncoderFactory.DEFAULT;
debugViewProvider = DebugViewProvider.NONE;
containerMimeType = MimeTypes.VIDEO_MP4;
this.transformationRequest = new TransformationRequest.Builder().build();
transformationRequest = new TransformationRequest.Builder().build();
frameProcessors = ImmutableList.of();
}
/** Creates a builder with the values of the provided {@link Transformer}. */
@ -149,6 +154,7 @@ public final class Transformer {
this.removeVideo = transformer.removeVideo;
this.containerMimeType = transformer.containerMimeType;
this.transformationRequest = transformer.transformationRequest;
this.frameProcessors = transformer.frameProcessors;
this.listeners = transformer.listeners;
this.looper = transformer.looper;
this.encoderFactory = transformer.encoderFactory;
@ -180,6 +186,24 @@ public final class Transformer {
return this;
}
/**
* Sets the {@linkplain GlFrameProcessor frame processors} to apply to each frame.
*
* <p>The {@linkplain GlFrameProcessor frame processors} are applied before any {@linkplain
* TransformationRequest.Builder#setScale(float, float) scale}, {@linkplain
* TransformationRequest.Builder#setRotationDegrees(float) rotation}, or {@linkplain
* TransformationRequest.Builder#setResolution(int) resolution} changes specified in the {@link
* #setTransformationRequest(TransformationRequest) TransformationRequest} but after {@linkplain
* TransformationRequest.Builder#setFlattenForSlowMotion(boolean) slow-motion flattening}.
*
* @param frameProcessors The {@linkplain GlFrameProcessor frame processors}.
* @return This builder.
*/
public Builder setFrameProcessors(List<GlFrameProcessor> frameProcessors) {
this.frameProcessors = ImmutableList.copyOf(frameProcessors);
return this;
}
/**
* Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform.
*
@ -408,6 +432,7 @@ public final class Transformer {
removeVideo,
containerMimeType,
transformationRequest,
frameProcessors,
listeners,
looper,
clock,
@ -529,6 +554,7 @@ public final class Transformer {
private final boolean removeVideo;
private final String containerMimeType;
private final TransformationRequest transformationRequest;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final Looper looper;
private final Clock clock;
private final Codec.EncoderFactory encoderFactory;
@ -549,6 +575,7 @@ public final class Transformer {
boolean removeVideo,
String containerMimeType,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ListenerSet<Transformer.Listener> listeners,
Looper looper,
Clock clock,
@ -563,6 +590,7 @@ public final class Transformer {
this.removeVideo = removeVideo;
this.containerMimeType = containerMimeType;
this.transformationRequest = transformationRequest;
this.frameProcessors = frameProcessors;
this.listeners = listeners;
this.looper = looper;
this.clock = clock;
@ -703,6 +731,7 @@ public final class Transformer {
removeAudio,
removeVideo,
transformationRequest,
frameProcessors,
encoderFactory,
decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest),
@ -814,6 +843,7 @@ public final class Transformer {
private final boolean removeAudio;
private final boolean removeVideo;
private final TransformationRequest transformationRequest;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener;
@ -825,6 +855,7 @@ public final class Transformer {
boolean removeAudio,
boolean removeVideo,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
@ -834,6 +865,7 @@ public final class Transformer {
this.removeAudio = removeAudio;
this.removeVideo = removeVideo;
this.transformationRequest = transformationRequest;
this.frameProcessors = frameProcessors;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener;
@ -869,6 +901,7 @@ public final class Transformer {
muxerWrapper,
mediaClock,
transformationRequest,
frameProcessors,
encoderFactory,
decoderFactory,
fallbackListener,

8
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
View File

@ -25,6 +25,7 @@ import androidx.media3.common.Format;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.exoplayer.FormatHolder;
import androidx.media3.exoplayer.source.SampleStream.ReadDataResult;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@ -34,6 +35,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final String TAG = "TVideoRenderer";
private final Context context;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final Transformer.DebugViewProvider debugViewProvider;
@ -46,12 +48,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
MuxerWrapper muxerWrapper,
TransformerMediaClock mediaClock,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
this.context = context;
this.frameProcessors = frameProcessors;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.debugViewProvider = debugViewProvider;
@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
context,
inputFormat,
transformationRequest,
frameProcessors,
decoderFactory,
encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
@ -126,6 +131,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
&& transformationRequest.outputHeight != inputFormat.height) {
return false;
}
if (!frameProcessors.isEmpty()) {
return false;
}
return true;
}

8
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -52,6 +52,7 @@ import org.checkerframework.dataflow.qual.Pure;
Context context,
Format inputFormat,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes,
@ -69,7 +70,6 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
// TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
// TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset.
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor.Builder(context)
@ -86,7 +86,11 @@ import org.checkerframework.dataflow.qual.Pure;
inputFormat.pixelWidthHeightRatio,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor),
new ImmutableList.Builder<GlFrameProcessor>()
.addAll(frameProcessors)
.add(scaleToFitFrameProcessor)
.add(presentationFrameProcessor)
.build(),
transformationRequest.enableHdrEditing);
Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees();