Transformer GL: Add setResolution() API.
Simple, initial implementation to allow setResolution() to set the output height, for downscaling/upscaling. Per TODOs, follow-up CLs may change layering, add UI, or allow querying decoders for more resolution options. PiperOrigin-RevId: 410203343
This commit is contained in:
huangdarwin
Ian Baker
parent
1618e0ef8e
commit
79f03bb135
@ -29,12 +29,14 @@ import android.opengl.EGLSurface;
|
||||
import android.opengl.GLES20;
|
||||
import android.view.Surface;
|
||||
import androidx.annotation.RequiresApi;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import java.io.IOException;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/** Applies OpenGL transformations to video frames. */
|
||||
/**
|
||||
* OpenGlFrameEditor applies changes to individual video frames using OpenGL. Changes include just
|
||||
* resolution for now, but may later include brightness, cropping, rotation, etc.
|
||||
*/
|
||||
@RequiresApi(18)
|
||||
/* package */ final class OpenGlFrameEditor {
|
||||
|
||||
@ -42,8 +44,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
GlUtil.glAssertionsEnabled = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new OpenGlFrameEditor for applying changes to individual frames.
|
||||
*
|
||||
* @param context A {@link Context}.
|
||||
* @param outputWidth The output width in pixels.
|
||||
* @param outputHeight The output height in pixels.
|
||||
* @param outputSurface The {@link Surface}.
|
||||
* @return A configured OpenGlFrameEditor.
|
||||
*/
|
||||
public static OpenGlFrameEditor create(
|
||||
Context context, Format inputFormat, Surface outputSurface) {
|
||||
Context context, int outputWidth, int outputHeight, Surface outputSurface) {
|
||||
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
|
||||
EGLContext eglContext;
|
||||
try {
|
||||
@ -52,7 +63,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
throw new IllegalStateException("EGL version is unsupported", e);
|
||||
}
|
||||
EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
|
||||
GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, inputFormat.width, inputFormat.height);
|
||||
GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
|
||||
int textureId = GlUtil.createExternalTexture();
|
||||
GlUtil.Program copyProgram;
|
||||
try {
|
||||
|
||||
@ -71,9 +71,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
* <p>Temporary copy of the {@link Transformer} class, which transforms by transcoding rather than
|
||||
* by muxing. This class is intended to replace the Transformer class.
|
||||
*
|
||||
* <p>TODO(http://b/202131097): Replace the Transformer class with TranscodingTransformer, and
|
||||
* rename this class to Transformer.
|
||||
*
|
||||
* <p>The same TranscodingTransformer instance can be used to transform multiple inputs
|
||||
* (sequentially, not concurrently).
|
||||
*
|
||||
@ -89,16 +86,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
@RequiresApi(18)
|
||||
@UnstableApi
|
||||
public final class TranscodingTransformer {
|
||||
// TODO(http://b/202131097): Replace the Transformer class with TranscodingTransformer, and
|
||||
// rename this class to Transformer.
|
||||
|
||||
/** A builder for {@link TranscodingTransformer} instances. */
|
||||
public static final class Builder {
|
||||
|
||||
// Mandatory field.
|
||||
private @MonotonicNonNull Context context;
|
||||
|
||||
// Optional fields.
|
||||
private @MonotonicNonNull MediaSourceFactory mediaSourceFactory;
|
||||
private Muxer.Factory muxerFactory;
|
||||
private boolean removeAudio;
|
||||
private boolean removeVideo;
|
||||
private boolean flattenForSlowMotion;
|
||||
private int outputHeight;
|
||||
private String outputMimeType;
|
||||
@Nullable private String audioMimeType;
|
||||
@Nullable private String videoMimeType;
|
||||
@ -109,6 +112,7 @@ public final class TranscodingTransformer {
|
||||
/** Creates a builder with default values. */
|
||||
public Builder() {
|
||||
muxerFactory = new FrameworkMuxer.Factory();
|
||||
outputHeight = Transformation.NO_VALUE;
|
||||
outputMimeType = MimeTypes.VIDEO_MP4;
|
||||
listener = new Listener() {};
|
||||
looper = Util.getCurrentOrMainLooper();
|
||||
@ -123,6 +127,7 @@ public final class TranscodingTransformer {
|
||||
this.removeAudio = transcodingTransformer.transformation.removeAudio;
|
||||
this.removeVideo = transcodingTransformer.transformation.removeVideo;
|
||||
this.flattenForSlowMotion = transcodingTransformer.transformation.flattenForSlowMotion;
|
||||
this.outputHeight = transcodingTransformer.transformation.outputHeight;
|
||||
this.outputMimeType = transcodingTransformer.transformation.outputMimeType;
|
||||
this.audioMimeType = transcodingTransformer.transformation.audioMimeType;
|
||||
this.videoMimeType = transcodingTransformer.transformation.videoMimeType;
|
||||
@ -215,6 +220,37 @@ public final class TranscodingTransformer {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the output resolution using the output height. The default value is {@link
|
||||
* Transformation#NO_VALUE}, which will use the same height as the input. Output width will
|
||||
* scale to preserve the input video's aspect ratio.
|
||||
*
|
||||
* <p>For now, only "popular" heights like 240, 360, 480, 720, 1080, 1440, or 2160 are
|
||||
* supported, to ensure compatibility on different devices.
|
||||
*
|
||||
* <p>For example, a 1920x1440 video can be scaled to 640x480 by calling setResolution(480).
|
||||
*
|
||||
* @param outputHeight The output height in pixels.
|
||||
* @return This builder.
|
||||
*/
|
||||
public Builder setResolution(int outputHeight) {
|
||||
// TODO(Internal b/201293185): Restructure to input a Presentation class.
|
||||
// TODO(Internal b/201293185): Check encoder codec capabilities in order to allow arbitrary
|
||||
// resolutions and reasonable fallbacks.
|
||||
if (outputHeight != 240
|
||||
&& outputHeight != 360
|
||||
&& outputHeight != 480
|
||||
&& outputHeight != 720
|
||||
&& outputHeight != 1080
|
||||
&& outputHeight != 1440
|
||||
&& outputHeight != 2160) {
|
||||
throw new IllegalArgumentException(
|
||||
"Please use a height of 240, 360, 480, 720, 1080, 1440, or 2160.");
|
||||
}
|
||||
this.outputHeight = outputHeight;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the MIME type of the output. The default value is {@link MimeTypes#VIDEO_MP4}. Supported
|
||||
* values are:
|
||||
@ -369,6 +405,7 @@ public final class TranscodingTransformer {
|
||||
removeAudio,
|
||||
removeVideo,
|
||||
flattenForSlowMotion,
|
||||
outputHeight,
|
||||
outputMimeType,
|
||||
audioMimeType,
|
||||
videoMimeType);
|
||||
|
||||
@ -21,9 +21,13 @@ import androidx.annotation.Nullable;
|
||||
/** A media transformation configuration. */
|
||||
/* package */ final class Transformation {
|
||||
|
||||
/** A value for various fields to indicate that the field's value is unknown or not set. */
|
||||
public static final int NO_VALUE = -1;
|
||||
|
||||
public final boolean removeAudio;
|
||||
public final boolean removeVideo;
|
||||
public final boolean flattenForSlowMotion;
|
||||
public final int outputHeight;
|
||||
public final String outputMimeType;
|
||||
@Nullable public final String audioMimeType;
|
||||
@Nullable public final String videoMimeType;
|
||||
@ -32,12 +36,14 @@ import androidx.annotation.Nullable;
|
||||
boolean removeAudio,
|
||||
boolean removeVideo,
|
||||
boolean flattenForSlowMotion,
|
||||
int outputHeight,
|
||||
String outputMimeType,
|
||||
@Nullable String audioMimeType,
|
||||
@Nullable String videoMimeType) {
|
||||
this.removeAudio = removeAudio;
|
||||
this.removeVideo = removeVideo;
|
||||
this.flattenForSlowMotion = flattenForSlowMotion;
|
||||
this.outputHeight = outputHeight;
|
||||
this.outputMimeType = outputMimeType;
|
||||
this.audioMimeType = audioMimeType;
|
||||
this.videoMimeType = videoMimeType;
|
||||
|
||||
@ -304,6 +304,7 @@ public final class Transformer {
|
||||
removeAudio,
|
||||
removeVideo,
|
||||
flattenForSlowMotion,
|
||||
/* outputHeight= */ Transformation.NO_VALUE,
|
||||
outputMimeType,
|
||||
/* audioMimeType= */ null,
|
||||
/* videoMimeType= */ null);
|
||||
|
||||
@ -83,7 +83,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
while (feedMuxerFromPipeline() || samplePipeline.processData() || feedPipelineFromInput()) {}
|
||||
}
|
||||
|
||||
/** Attempts to read the input format and to initialize the sample pipeline. */
|
||||
/** Attempts to read the input format and to initialize the sample or passthrough pipeline. */
|
||||
@EnsuresNonNullIf(expression = "samplePipeline", result = true)
|
||||
private boolean ensureRendererConfigured() throws ExoPlaybackException {
|
||||
if (samplePipeline != null) {
|
||||
@ -96,8 +96,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
return false;
|
||||
}
|
||||
Format decoderInputFormat = checkNotNull(formatHolder.format);
|
||||
if (transformation.videoMimeType != null
|
||||
&& !transformation.videoMimeType.equals(decoderInputFormat.sampleMimeType)) {
|
||||
if ((transformation.videoMimeType != null
|
||||
&& !transformation.videoMimeType.equals(decoderInputFormat.sampleMimeType))
|
||||
|| (transformation.outputHeight != Transformation.NO_VALUE
|
||||
&& transformation.outputHeight != decoderInputFormat.height)) {
|
||||
samplePipeline =
|
||||
new VideoSamplePipeline(context, decoderInputFormat, transformation, getIndex());
|
||||
} else {
|
||||
|
||||
@ -57,12 +57,22 @@ import java.io.IOException;
|
||||
|
||||
encoderOutputBuffer =
|
||||
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
|
||||
|
||||
int outputWidth = decoderInputFormat.width;
|
||||
int outputHeight = decoderInputFormat.height;
|
||||
if (transformation.outputHeight != Transformation.NO_VALUE
|
||||
&& transformation.outputHeight != decoderInputFormat.height) {
|
||||
outputWidth =
|
||||
decoderInputFormat.width * transformation.outputHeight / decoderInputFormat.height;
|
||||
outputHeight = transformation.outputHeight;
|
||||
}
|
||||
|
||||
try {
|
||||
encoder =
|
||||
MediaCodecAdapterWrapper.createForVideoEncoding(
|
||||
new Format.Builder()
|
||||
.setWidth(decoderInputFormat.width)
|
||||
.setHeight(decoderInputFormat.height)
|
||||
.setWidth(outputWidth)
|
||||
.setHeight(outputHeight)
|
||||
.setSampleMimeType(
|
||||
transformation.videoMimeType != null
|
||||
? transformation.videoMimeType
|
||||
@ -77,7 +87,8 @@ import java.io.IOException;
|
||||
openGlFrameEditor =
|
||||
OpenGlFrameEditor.create(
|
||||
context,
|
||||
decoderInputFormat,
|
||||
outputWidth,
|
||||
outputHeight,
|
||||
/* outputSurface= */ checkNotNull(encoder.getInputSurface()));
|
||||
try {
|
||||
decoder =
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user