From cacec8e02fa87db793f3c4779cabccd791f05c7e Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Fri, 11 Feb 2022 13:47:20 +0000 Subject: [PATCH] Transformer GL: Implement auto-scaling to preserve input frame. PiperOrigin-RevId: 427982223 --- .../ConfigurationActivity.java | 2 +- .../transformer/DefaultEncoderFactory.java | 2 +- .../transformer/TransformationRequest.java | 20 +-- .../VideoTranscodingSamplePipeline.java | 122 +++++++++++------- 4 files changed, 82 insertions(+), 64 deletions(-) diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java index 3381935df4..795ed9fb70 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java @@ -152,7 +152,7 @@ public final class ConfigurationActivity extends AppCompatActivity { rotateAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); rotateSpinner = findViewById(R.id.rotate_spinner); rotateSpinner.setAdapter(rotateAdapter); - rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "90", "180"); + rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "60", "90", "180"); enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java index 2a6e385a40..769123ea06 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java @@ -105,7 +105,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { checkArgument(format.width != Format.NO_VALUE); checkArgument(format.height != Format.NO_VALUE); // According to interface Javadoc, format.rotationDegrees should be 0. The video should always - // be in landscape orientation. + // be encoded in landscape orientation. checkArgument(format.height <= format.width); checkArgument(format.rotationDegrees == 0); checkNotNull(format.sampleMimeType); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java index 20f249c359..9bb83df3ae 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java @@ -25,7 +25,6 @@ import com.google.android.exoplayer2.extractor.mp4.Mp4Extractor; import com.google.android.exoplayer2.source.MediaSource; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; -import com.google.common.collect.ImmutableSet; /** A media transformation request. */ public final class TransformationRequest { @@ -33,9 +32,6 @@ public final class TransformationRequest { /** A builder for {@link TransformationRequest} instances. */ public static final class Builder { - private static final ImmutableSet SUPPORTED_OUTPUT_HEIGHTS = - ImmutableSet.of(144, 240, 360, 480, 720, 1080, 1440, 2160); - private Matrix transformationMatrix; private boolean flattenForSlowMotion; private int outputHeight; @@ -122,27 +118,17 @@ public final class TransformationRequest { /** * Sets the output resolution using the output height. The default value {@link C#LENGTH_UNSET} - * corresponds to using the same height as the input. Output width will scale to preserve the - * input video's aspect ratio. - * - *

For now, only "popular" heights like 144, 240, 360, 480, 720, 1080, 1440, or 2160 are - * supported, to ensure compatibility on different devices. + * corresponds to using the same height as the input. Output width of the displayed video will + * scale to preserve the video's aspect ratio after other transformations. * *

For example, a 1920x1440 video can be scaled to 640x480 by calling setResolution(480). * - * @param outputHeight The output height in pixels. + * @param outputHeight The output height of the displayed video, in pixels. * @return This builder. * @throws IllegalArgumentException If the {@code outputHeight} is not supported. */ public Builder setResolution(int outputHeight) { - // TODO(b/209781577): Define outputHeight in the javadoc as height can be ambiguous for videos - // where rotationDegrees is set in the Format. // TODO(b/201293185): Restructure to input a Presentation class. - // TODO(b/201293185): Check encoder codec capabilities in order to allow arbitrary - // resolutions and reasonable fallbacks. - checkArgument( - outputHeight == C.LENGTH_UNSET || SUPPORTED_OUTPUT_HEIGHTS.contains(outputHeight), - "Unsupported outputHeight: " + outputHeight); this.outputHeight = outputHeight; return this; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index fb85fad7f1..c02ed99266 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -18,6 +18,8 @@ package com.google.android.exoplayer2.transformer; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Util.SDK_INT; +import static java.lang.Math.max; +import static java.lang.Math.min; import android.content.Context; import android.graphics.Matrix; @@ -63,76 +65,106 @@ import org.checkerframework.dataflow.qual.Pure; encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - // Scale width and height to desired transformationRequest.outputHeight, preserving aspect - // ratio. - // TODO(b/209781577): Think about which edge length should be set for portrait videos. - float inputFormatAspectRatio = (float) inputFormat.width / inputFormat.height; - int outputWidth = inputFormat.width; - int outputHeight = inputFormat.height; + // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. + int decodedWidth = + (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height; + int decodedHeight = + (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; + float decodedAspectRatio = (float) decodedWidth / decodedHeight; + + Matrix transformationMatrix = new Matrix(transformationRequest.transformationMatrix); + + int outputWidth = decodedWidth; + int outputHeight = decodedHeight; + if (!transformationMatrix.isIdentity()) { + // Scale frames by decodedAspectRatio, to account for FrameEditor's normalized device + // coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular display + // of input pixels during transformations (ex. rotations). With scaling, transformationMatrix + // operations operate on a rectangle for x from -decodedAspectRatio to decodedAspectRatio, and + // y from -1 to 1. + transformationMatrix.preScale(/* sx= */ decodedAspectRatio, /* sy= */ 1f); + transformationMatrix.postScale(/* sx= */ 1f / decodedAspectRatio, /* sy= */ 1f); + + float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}}; + float xMin = Float.MAX_VALUE; + float xMax = Float.MIN_VALUE; + float yMin = Float.MAX_VALUE; + float yMax = Float.MIN_VALUE; + for (float[] transformOnNdcPoint : transformOnNdcPoints) { + transformationMatrix.mapPoints(transformOnNdcPoint); + xMin = min(xMin, transformOnNdcPoint[0]); + xMax = max(xMax, transformOnNdcPoint[0]); + yMin = min(yMin, transformOnNdcPoint[1]); + yMax = max(yMax, transformOnNdcPoint[1]); + } + + float xCenter = (xMax + xMin) / 2f; + float yCenter = (yMax + yMin) / 2f; + transformationMatrix.postTranslate(-xCenter, -yCenter); + + float ndcWidthAndHeight = 2f; // Length from -1 to 1. + float xScale = (xMax - xMin) / ndcWidthAndHeight; + float yScale = (yMax - yMin) / ndcWidthAndHeight; + transformationMatrix.postScale(1f / xScale, 1f / yScale); + outputWidth = Math.round(decodedWidth * xScale); + outputHeight = Math.round(decodedHeight * yScale); + } + // Scale width and height to desired transformationRequest.outputHeight, preserving + // aspect ratio. if (transformationRequest.outputHeight != C.LENGTH_UNSET - && transformationRequest.outputHeight != inputFormat.height) { - outputWidth = Math.round(inputFormatAspectRatio * transformationRequest.outputHeight); + && transformationRequest.outputHeight != outputHeight) { + outputWidth = + Math.round((float) transformationRequest.outputHeight * outputWidth / outputHeight); outputHeight = transformationRequest.outputHeight; } - // The encoder may not support encoding in portrait orientation, so the decoded video is - // rotated to landscape orientation and a rotation is added back later to the output format. - boolean swapEncodingDimensions = inputFormat.height > inputFormat.width; + // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded + // video before encoding, so the encoded video's width >= height, and set outputRotationDegrees + // to ensure the video is displayed in the correct orientation. + int requestedEncoderWidth; + int requestedEncoderHeight; + boolean swapEncodingDimensions = outputHeight > outputWidth; if (swapEncodingDimensions) { - outputRotationDegrees = (inputFormat.rotationDegrees + 90) % 360; - int temp = outputWidth; - outputWidth = outputHeight; - outputHeight = temp; + outputRotationDegrees = 90; + requestedEncoderWidth = outputHeight; + requestedEncoderHeight = outputWidth; + // TODO(b/201293185): After fragment shader transformations are implemented, put + // postRotate in a later vertex shader. + transformationMatrix.postRotate(outputRotationDegrees); } else { - outputRotationDegrees = inputFormat.rotationDegrees; + outputRotationDegrees = 0; + requestedEncoderWidth = outputWidth; + requestedEncoderHeight = outputHeight; } - float displayAspectRatio = - (inputFormat.rotationDegrees % 180) == 0 - ? inputFormatAspectRatio - : 1.0f / inputFormatAspectRatio; - Matrix transformationMatrix = new Matrix(transformationRequest.transformationMatrix); - // Scale frames by input aspect ratio, to account for FrameEditor's square normalized device - // coordinates (-1 to 1) and preserve frame relative dimensions during transformations - // (ex. rotations). After this scaling, transformationMatrix operations operate on a rectangle - // for x from -displayAspectRatio to displayAspectRatio, and y from -1 to 1 - transformationMatrix.preScale(displayAspectRatio, 1); - transformationMatrix.postScale(1.0f / displayAspectRatio, 1); - - // The decoder rotates videos to their intended display orientation. The frameEditor rotates - // them back for improved encoder compatibility. - // TODO(b/201293185): After fragment shader transformations are implemented, put - // postRotate in a later vertex shader. - transformationMatrix.postRotate(outputRotationDegrees); - - Format requestedOutputFormat = + Format requestedEncoderFormat = new Format.Builder() - .setWidth(outputWidth) - .setHeight(outputHeight) + .setWidth(requestedEncoderWidth) + .setHeight(requestedEncoderHeight) .setRotationDegrees(0) .setSampleMimeType( transformationRequest.videoMimeType != null ? transformationRequest.videoMimeType : inputFormat.sampleMimeType) .build(); - encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat, allowedOutputMimeTypes); - Format actualOutputFormat = encoder.getConfigurationFormat(); + encoder = encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); + Format encoderSupportedFormat = encoder.getConfigurationFormat(); fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( transformationRequest, !swapEncodingDimensions, - requestedOutputFormat, - actualOutputFormat)); + requestedEncoderFormat, + encoderSupportedFormat)); if (transformationRequest.enableHdrEditing - || inputFormat.height != actualOutputFormat.height - || inputFormat.width != actualOutputFormat.width + || inputFormat.height != encoderSupportedFormat.height + || inputFormat.width != encoderSupportedFormat.width || !transformationMatrix.isIdentity()) { frameEditor = FrameEditor.create( context, - actualOutputFormat.width, - actualOutputFormat.height, + encoderSupportedFormat.width, + encoderSupportedFormat.height, inputFormat.pixelWidthHeightRatio, transformationMatrix, /* outputSurface= */ encoder.getInputSurface(),