From 79f03bb1355b7e370991a1aafd580324d22e070e Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Tue, 16 Nov 2021 10:48:45 +0000 Subject: [PATCH] Transformer GL: Add setResolution() API. Simple, initial implementation to allow setResolution() to set the output height, for downscaling/upscaling. Per TODOs, follow-up CLs may change layering, add UI, or allow querying decoders for more resolution options. PiperOrigin-RevId: 410203343 --- .../media3/transformer/OpenGlFrameEditor.java | 19 ++++++-- .../transformer/TranscodingTransformer.java | 43 +++++++++++++++++-- .../media3/transformer/Transformation.java | 6 +++ .../media3/transformer/Transformer.java | 1 + .../TransformerTranscodingVideoRenderer.java | 8 ++-- .../transformer/VideoSamplePipeline.java | 17 ++++++-- 6 files changed, 81 insertions(+), 13 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java index c633d562eb..c556f35c7c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java @@ -29,12 +29,14 @@ import android.opengl.EGLSurface; import android.opengl.GLES20; import android.view.Surface; import androidx.annotation.RequiresApi; -import androidx.media3.common.Format; import androidx.media3.common.util.GlUtil; import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** Applies OpenGL transformations to video frames. */ +/** + * OpenGlFrameEditor applies changes to individual video frames using OpenGL. Changes include just + * resolution for now, but may later include brightness, cropping, rotation, etc. + */ @RequiresApi(18) /* package */ final class OpenGlFrameEditor { @@ -42,8 +44,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlUtil.glAssertionsEnabled = true; } + /** + * Returns a new OpenGlFrameEditor for applying changes to individual frames. + * + * @param context A {@link Context}. + * @param outputWidth The output width in pixels. + * @param outputHeight The output height in pixels. + * @param outputSurface The {@link Surface}. + * @return A configured OpenGlFrameEditor. + */ public static OpenGlFrameEditor create( - Context context, Format inputFormat, Surface outputSurface) { + Context context, int outputWidth, int outputHeight, Surface outputSurface) { EGLDisplay eglDisplay = GlUtil.createEglDisplay(); EGLContext eglContext; try { @@ -52,7 +63,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new IllegalStateException("EGL version is unsupported", e); } EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); - GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, inputFormat.width, inputFormat.height); + GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); int textureId = GlUtil.createExternalTexture(); GlUtil.Program copyProgram; try { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TranscodingTransformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TranscodingTransformer.java index fb60d058ea..f04a11d094 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TranscodingTransformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TranscodingTransformer.java @@ -71,9 +71,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; *

Temporary copy of the {@link Transformer} class, which transforms by transcoding rather than * by muxing. This class is intended to replace the Transformer class. * - *

TODO(http://b/202131097): Replace the Transformer class with TranscodingTransformer, and - * rename this class to Transformer. - * *

The same TranscodingTransformer instance can be used to transform multiple inputs * (sequentially, not concurrently). * @@ -89,16 +86,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @RequiresApi(18) @UnstableApi public final class TranscodingTransformer { + // TODO(http://b/202131097): Replace the Transformer class with TranscodingTransformer, and + // rename this class to Transformer. /** A builder for {@link TranscodingTransformer} instances. */ public static final class Builder { + // Mandatory field. private @MonotonicNonNull Context context; + + // Optional fields. private @MonotonicNonNull MediaSourceFactory mediaSourceFactory; private Muxer.Factory muxerFactory; private boolean removeAudio; private boolean removeVideo; private boolean flattenForSlowMotion; + private int outputHeight; private String outputMimeType; @Nullable private String audioMimeType; @Nullable private String videoMimeType; @@ -109,6 +112,7 @@ public final class TranscodingTransformer { /** Creates a builder with default values. */ public Builder() { muxerFactory = new FrameworkMuxer.Factory(); + outputHeight = Transformation.NO_VALUE; outputMimeType = MimeTypes.VIDEO_MP4; listener = new Listener() {}; looper = Util.getCurrentOrMainLooper(); @@ -123,6 +127,7 @@ public final class TranscodingTransformer { this.removeAudio = transcodingTransformer.transformation.removeAudio; this.removeVideo = transcodingTransformer.transformation.removeVideo; this.flattenForSlowMotion = transcodingTransformer.transformation.flattenForSlowMotion; + this.outputHeight = transcodingTransformer.transformation.outputHeight; this.outputMimeType = transcodingTransformer.transformation.outputMimeType; this.audioMimeType = transcodingTransformer.transformation.audioMimeType; this.videoMimeType = transcodingTransformer.transformation.videoMimeType; @@ -215,6 +220,37 @@ public final class TranscodingTransformer { return this; } + /** + * Sets the output resolution using the output height. The default value is {@link + * Transformation#NO_VALUE}, which will use the same height as the input. Output width will + * scale to preserve the input video's aspect ratio. + * + *

For now, only "popular" heights like 240, 360, 480, 720, 1080, 1440, or 2160 are + * supported, to ensure compatibility on different devices. + * + *

For example, a 1920x1440 video can be scaled to 640x480 by calling setResolution(480). + * + * @param outputHeight The output height in pixels. + * @return This builder. + */ + public Builder setResolution(int outputHeight) { + // TODO(Internal b/201293185): Restructure to input a Presentation class. + // TODO(Internal b/201293185): Check encoder codec capabilities in order to allow arbitrary + // resolutions and reasonable fallbacks. + if (outputHeight != 240 + && outputHeight != 360 + && outputHeight != 480 + && outputHeight != 720 + && outputHeight != 1080 + && outputHeight != 1440 + && outputHeight != 2160) { + throw new IllegalArgumentException( + "Please use a height of 240, 360, 480, 720, 1080, 1440, or 2160."); + } + this.outputHeight = outputHeight; + return this; + } + /** * Sets the MIME type of the output. The default value is {@link MimeTypes#VIDEO_MP4}. Supported * values are: @@ -369,6 +405,7 @@ public final class TranscodingTransformer { removeAudio, removeVideo, flattenForSlowMotion, + outputHeight, outputMimeType, audioMimeType, videoMimeType); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java index 7b109478be..a3f3343281 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java @@ -21,9 +21,13 @@ import androidx.annotation.Nullable; /** A media transformation configuration. */ /* package */ final class Transformation { + /** A value for various fields to indicate that the field's value is unknown or not set. */ + public static final int NO_VALUE = -1; + public final boolean removeAudio; public final boolean removeVideo; public final boolean flattenForSlowMotion; + public final int outputHeight; public final String outputMimeType; @Nullable public final String audioMimeType; @Nullable public final String videoMimeType; @@ -32,12 +36,14 @@ import androidx.annotation.Nullable; boolean removeAudio, boolean removeVideo, boolean flattenForSlowMotion, + int outputHeight, String outputMimeType, @Nullable String audioMimeType, @Nullable String videoMimeType) { this.removeAudio = removeAudio; this.removeVideo = removeVideo; this.flattenForSlowMotion = flattenForSlowMotion; + this.outputHeight = outputHeight; this.outputMimeType = outputMimeType; this.audioMimeType = audioMimeType; this.videoMimeType = videoMimeType; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 85ff150e99..89ac90df38 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -304,6 +304,7 @@ public final class Transformer { removeAudio, removeVideo, flattenForSlowMotion, + /* outputHeight= */ Transformation.NO_VALUE, outputMimeType, /* audioMimeType= */ null, /* videoMimeType= */ null); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java index e194938d79..8c498887a4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java @@ -83,7 +83,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; while (feedMuxerFromPipeline() || samplePipeline.processData() || feedPipelineFromInput()) {} } - /** Attempts to read the input format and to initialize the sample pipeline. */ + /** Attempts to read the input format and to initialize the sample or passthrough pipeline. */ @EnsuresNonNullIf(expression = "samplePipeline", result = true) private boolean ensureRendererConfigured() throws ExoPlaybackException { if (samplePipeline != null) { @@ -96,8 +96,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } Format decoderInputFormat = checkNotNull(formatHolder.format); - if (transformation.videoMimeType != null - && !transformation.videoMimeType.equals(decoderInputFormat.sampleMimeType)) { + if ((transformation.videoMimeType != null + && !transformation.videoMimeType.equals(decoderInputFormat.sampleMimeType)) + || (transformation.outputHeight != Transformation.NO_VALUE + && transformation.outputHeight != decoderInputFormat.height)) { samplePipeline = new VideoSamplePipeline(context, decoderInputFormat, transformation, getIndex()); } else { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index 984b37227a..d120332ce9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -57,12 +57,22 @@ import java.io.IOException; encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); + + int outputWidth = decoderInputFormat.width; + int outputHeight = decoderInputFormat.height; + if (transformation.outputHeight != Transformation.NO_VALUE + && transformation.outputHeight != decoderInputFormat.height) { + outputWidth = + decoderInputFormat.width * transformation.outputHeight / decoderInputFormat.height; + outputHeight = transformation.outputHeight; + } + try { encoder = MediaCodecAdapterWrapper.createForVideoEncoding( new Format.Builder() - .setWidth(decoderInputFormat.width) - .setHeight(decoderInputFormat.height) + .setWidth(outputWidth) + .setHeight(outputHeight) .setSampleMimeType( transformation.videoMimeType != null ? transformation.videoMimeType @@ -77,7 +87,8 @@ import java.io.IOException; openGlFrameEditor = OpenGlFrameEditor.create( context, - decoderInputFormat, + outputWidth, + outputHeight, /* outputSurface= */ checkNotNull(encoder.getInputSurface())); try { decoder =