From 2a14e3c604ba65ad2a39ee85c71d99cc86bb43b1 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Mon, 21 Mar 2022 12:08:18 +0000 Subject: [PATCH] FrameProcessor: Add a ScaleToFitFrameProcessor builder. This allows us to input scale and rotation in an easier-to-use manner. PiperOrigin-RevId: 436175982 --- .../transformer/ConfigurationActivity.java | 24 ---- .../demo/transformer/TransformerActivity.java | 35 ++---- .../res/layout/configuration_activity.xml | 11 -- .../src/main/res/values/strings.xml | 1 - .../FrameEditorDataProcessingTest.java | 12 +- .../transformer/TransformerEndToEndTest.java | 7 +- .../RepeatedTranscodeTransformationTest.java | 9 +- ...va => SetFrameEditTransformationTest.java} | 16 +-- .../transformer/ScaleToFitFrameProcessor.java | 88 ++++++++++++- .../transformer/TransformationRequest.java | 117 ++++++++++++------ .../media3/transformer/Transformer.java | 4 + .../transformer/TransformerVideoRenderer.java | 15 ++- .../VideoTranscodingSamplePipeline.java | 9 +- .../ScaleToFitFrameProcessorTest.java | 59 +++++---- .../TransformationRequestTest.java | 8 +- 15 files changed, 236 insertions(+), 179 deletions(-) rename libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/{SetTransformationMatrixTransformationTest.java => SetFrameEditTransformationTest.java} (69%) diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index b6a447768c..eaa51847e5 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -50,8 +50,6 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String AUDIO_MIME_TYPE = "audio_mime_type"; public static final String VIDEO_MIME_TYPE = "video_mime_type"; public static final String RESOLUTION_HEIGHT = "resolution_height"; - public static final String TRANSLATE_X = "translate_x"; - public static final String TRANSLATE_Y = "translate_y"; public static final String SCALE_X = "scale_x"; public static final String SCALE_Y = "scale_y"; public static final String ROTATE_DEGREES = "rotate_degrees"; @@ -81,7 +79,6 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull Spinner audioMimeSpinner; private @MonotonicNonNull Spinner videoMimeSpinner; private @MonotonicNonNull Spinner resolutionHeightSpinner; - private @MonotonicNonNull Spinner translateSpinner; private @MonotonicNonNull Spinner scaleSpinner; private @MonotonicNonNull Spinner rotateSpinner; private @MonotonicNonNull CheckBox enableFallbackCheckBox; @@ -136,14 +133,6 @@ public final class ConfigurationActivity extends AppCompatActivity { resolutionHeightAdapter.addAll( SAME_AS_INPUT_OPTION, "144", "240", "360", "480", "720", "1080", "1440", "2160"); - ArrayAdapter translateAdapter = - new ArrayAdapter<>(/* context= */ this, R.layout.spinner_item); - translateAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); - translateSpinner = findViewById(R.id.translate_spinner); - translateSpinner.setAdapter(translateAdapter); - translateAdapter.addAll( - SAME_AS_INPUT_OPTION, "-.1, -.1", "0, 0", ".5, 0", "0, .5", "1, 1", "1.9, 0", "0, 1.9"); - ArrayAdapter scaleAdapter = new ArrayAdapter<>(/* context= */ this, R.layout.spinner_item); scaleAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); @@ -185,7 +174,6 @@ public final class ConfigurationActivity extends AppCompatActivity { "audioMimeSpinner", "videoMimeSpinner", "resolutionHeightSpinner", - "translateSpinner", "scaleSpinner", "rotateSpinner", "enableFallbackCheckBox", @@ -209,13 +197,6 @@ public final class ConfigurationActivity extends AppCompatActivity { if (!SAME_AS_INPUT_OPTION.equals(selectedResolutionHeight)) { bundle.putInt(RESOLUTION_HEIGHT, Integer.parseInt(selectedResolutionHeight)); } - String selectedTranslate = String.valueOf(translateSpinner.getSelectedItem()); - if (!SAME_AS_INPUT_OPTION.equals(selectedTranslate)) { - List translateXY = Arrays.asList(selectedTranslate.split(", ")); - checkState(translateXY.size() == 2); - bundle.putFloat(TRANSLATE_X, Float.parseFloat(translateXY.get(0))); - bundle.putFloat(TRANSLATE_Y, Float.parseFloat(translateXY.get(1))); - } String selectedScale = String.valueOf(scaleSpinner.getSelectedItem()); if (!SAME_AS_INPUT_OPTION.equals(selectedScale)) { List scaleXY = Arrays.asList(selectedScale.split(", ")); @@ -258,7 +239,6 @@ public final class ConfigurationActivity extends AppCompatActivity { "audioMimeSpinner", "videoMimeSpinner", "resolutionHeightSpinner", - "translateSpinner", "scaleSpinner", "rotateSpinner", "enableHdrEditingCheckBox" @@ -277,7 +257,6 @@ public final class ConfigurationActivity extends AppCompatActivity { "audioMimeSpinner", "videoMimeSpinner", "resolutionHeightSpinner", - "translateSpinner", "scaleSpinner", "rotateSpinner", "enableHdrEditingCheckBox" @@ -295,7 +274,6 @@ public final class ConfigurationActivity extends AppCompatActivity { "audioMimeSpinner", "videoMimeSpinner", "resolutionHeightSpinner", - "translateSpinner", "scaleSpinner", "rotateSpinner", "enableHdrEditingCheckBox" @@ -304,7 +282,6 @@ public final class ConfigurationActivity extends AppCompatActivity { audioMimeSpinner.setEnabled(isAudioEnabled); videoMimeSpinner.setEnabled(isVideoEnabled); resolutionHeightSpinner.setEnabled(isVideoEnabled); - translateSpinner.setEnabled(isVideoEnabled); scaleSpinner.setEnabled(isVideoEnabled); rotateSpinner.setEnabled(isVideoEnabled); enableHdrEditingCheckBox.setEnabled(isVideoEnabled); @@ -312,7 +289,6 @@ public final class ConfigurationActivity extends AppCompatActivity { findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled); findViewById(R.id.video_mime_text_view).setEnabled(isVideoEnabled); findViewById(R.id.resolution_height_text_view).setEnabled(isVideoEnabled); - findViewById(R.id.translate).setEnabled(isVideoEnabled); findViewById(R.id.scale).setEnabled(isVideoEnabled); findViewById(R.id.rotate).setEnabled(isVideoEnabled); findViewById(R.id.hdr_editing).setEnabled(isVideoEnabled); diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 4dc5c0585a..ad83ce75f0 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -21,7 +21,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import android.app.Activity; import android.content.Intent; import android.content.pm.PackageManager; -import android.graphics.Matrix; import android.net.Uri; import android.os.Bundle; import android.os.Handler; @@ -217,10 +216,15 @@ public final class TransformerActivity extends AppCompatActivity { if (resolutionHeight != C.LENGTH_UNSET) { requestBuilder.setResolution(resolutionHeight); } - Matrix transformationMatrix = getTransformationMatrix(bundle); - if (!transformationMatrix.isIdentity()) { - requestBuilder.setTransformationMatrix(transformationMatrix); - } + + float scaleX = bundle.getFloat(ConfigurationActivity.SCALE_X, /* defaultValue= */ 1); + float scaleY = bundle.getFloat(ConfigurationActivity.SCALE_Y, /* defaultValue= */ 1); + requestBuilder.setScale(scaleX, scaleY); + + float rotateDegrees = + bundle.getFloat(ConfigurationActivity.ROTATE_DEGREES, /* defaultValue= */ 0); + requestBuilder.setRotationDegrees(rotateDegrees); + requestBuilder.experimental_setEnableHdrEditing( bundle.getBoolean(ConfigurationActivity.ENABLE_HDR_EDITING)); transformerBuilder @@ -251,27 +255,6 @@ public final class TransformerActivity extends AppCompatActivity { .build(); } - private static Matrix getTransformationMatrix(Bundle bundle) { - Matrix transformationMatrix = new Matrix(); - - float translateX = bundle.getFloat(ConfigurationActivity.TRANSLATE_X, /* defaultValue= */ 0); - float translateY = bundle.getFloat(ConfigurationActivity.TRANSLATE_Y, /* defaultValue= */ 0); - // TODO(b/201293185): Implement an AdvancedFrameEditor to handle translation, as the current - // transformationMatrix is automatically adjusted to focus on the original pixels and - // effectively undo translations. - transformationMatrix.postTranslate(translateX, translateY); - - float scaleX = bundle.getFloat(ConfigurationActivity.SCALE_X, /* defaultValue= */ 1); - float scaleY = bundle.getFloat(ConfigurationActivity.SCALE_Y, /* defaultValue= */ 1); - transformationMatrix.postScale(scaleX, scaleY); - - float rotateDegrees = - bundle.getFloat(ConfigurationActivity.ROTATE_DEGREES, /* defaultValue= */ 0); - transformationMatrix.postRotate(rotateDegrees); - - return transformationMatrix; - } - @RequiresNonNull({ "informationTextView", "progressViewGroup", diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index f58f2f7167..1ff3cafc6b 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -137,17 +137,6 @@ android:layout_gravity="right|center_vertical" android:gravity="right" /> - - - - diff --git a/demos/transformer/src/main/res/values/strings.xml b/demos/transformer/src/main/res/values/strings.xml index 3b27a515ef..8e8f97ecf9 100644 --- a/demos/transformer/src/main/res/values/strings.xml +++ b/demos/transformer/src/main/res/values/strings.xml @@ -24,7 +24,6 @@ Output audio MIME type Output video MIME type Output video resolution - Translate video Scale video Rotate video (degrees) Enable fallback diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java index 7c43044f12..78162591b6 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java @@ -38,7 +38,6 @@ import android.media.MediaExtractor; import android.media.MediaFormat; import android.util.Size; import androidx.annotation.Nullable; -import androidx.media3.common.C; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.nio.ByteBuffer; @@ -175,10 +174,8 @@ public final class FrameEditorDataProcessingTest { // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can // test that rotation doesn't distort the image. - Matrix identityMatrix = new Matrix(); GlFrameProcessor glFrameProcessor = - new ScaleToFitFrameProcessor( - getApplicationContext(), identityMatrix, /* requestedHeight= */ 480); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()).setResolution(480).build(); setUpAndPrepareFirstFrame(glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); @@ -200,11 +197,10 @@ public final class FrameEditorDataProcessingTest { // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can // test that rotation doesn't distort the image. - Matrix rotate45Matrix = new Matrix(); - rotate45Matrix.postRotate(/* degrees= */ 45); GlFrameProcessor glFrameProcessor = - new ScaleToFitFrameProcessor( - getApplicationContext(), rotate45Matrix, /* requestedHeight= */ C.LENGTH_UNSET); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setRotationDegrees(45) + .build(); setUpAndPrepareFirstFrame(glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 0a4044d08e..7a710a0d8a 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static com.google.common.truth.Truth.assertThat; import android.content.Context; -import android.graphics.Matrix; import androidx.media3.common.MimeTypes; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; @@ -67,16 +66,12 @@ public class TransformerEndToEndTest { @Test public void videoEditing_completesWithConsistentFrameCount() throws Exception { Context context = ApplicationProvider.getApplicationContext(); - Matrix transformationMatrix = new Matrix(); - transformationMatrix.postTranslate(/* dx= */ .2f, /* dy= */ .1f); FrameCountingMuxer.Factory muxerFactory = new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory()); Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( - new TransformationRequest.Builder() - .setTransformationMatrix(transformationMatrix) - .build()) + new TransformationRequest.Builder().setResolution(480).build()) .setMuxerFactory(muxerFactory) .setEncoderFactory( new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false)) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java index 8245e86c06..5b77a783be 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java @@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static com.google.common.truth.Truth.assertWithMessage; import android.content.Context; -import android.graphics.Matrix; import androidx.media3.common.MimeTypes; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.TransformationRequest; @@ -41,13 +40,11 @@ public final class RepeatedTranscodeTransformationTest { @Test public void repeatedTranscode_givesConsistentLengthOutput() throws Exception { Context context = ApplicationProvider.getApplicationContext(); - Matrix transformationMatrix = new Matrix(); - transformationMatrix.postTranslate(/* dx= */ 0.1f, /* dy= */ 0.1f); Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( new TransformationRequest.Builder() - .setTransformationMatrix(transformationMatrix) + .setRotationDegrees(45) // Video MIME type is H264. .setAudioMimeType(MimeTypes.AUDIO_AAC) .build()) @@ -74,15 +71,13 @@ public final class RepeatedTranscodeTransformationTest { @Test public void repeatedTranscodeNoAudio_givesConsistentLengthOutput() throws Exception { Context context = ApplicationProvider.getApplicationContext(); - Matrix transformationMatrix = new Matrix(); - transformationMatrix.postTranslate(/* dx= */ 0.1f, /* dy= */ 0.1f); Transformer transformer = new Transformer.Builder(context) .setRemoveAudio(true) .setTransformationRequest( new TransformationRequest.Builder() // Video MIME type is H264. - .setTransformationMatrix(transformationMatrix) + .setRotationDegrees(45) .build()) .build(); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java similarity index 69% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java index 79d1322111..92bb7b3d16 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java @@ -18,7 +18,6 @@ package androidx.media3.transformer.mh; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING; import android.content.Context; -import android.graphics.Matrix; import androidx.media3.transformer.TransformationRequest; import androidx.media3.transformer.Transformer; import androidx.media3.transformer.TransformerAndroidTestRunner; @@ -27,26 +26,21 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; import org.junit.runner.RunWith; -/** {@link Transformer} instrumentation test for setting a transformation matrix. */ +/** {@link Transformer} instrumentation test for applying a frame edit. */ @RunWith(AndroidJUnit4.class) -public class SetTransformationMatrixTransformationTest { +public class SetFrameEditTransformationTest { @Test - public void setTransformationMatrixTransform() throws Exception { + public void setFrameEditTransform() throws Exception { Context context = ApplicationProvider.getApplicationContext(); - Matrix transformationMatrix = new Matrix(); - transformationMatrix.postTranslate(/* dx= */ .2f, /* dy= */ .1f); Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( - new TransformationRequest.Builder() - .setTransformationMatrix(transformationMatrix) - .build()) + new TransformationRequest.Builder().setRotationDegrees(45).build()) .build(); new TransformerAndroidTestRunner.Builder(context, transformer) .build() .run( - /* testId= */ "setTransformationMatrixTransform", - MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + /* testId= */ "SetFrameEditTransform", MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java index f463ea1e5a..ac2f36deec 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java @@ -38,6 +38,81 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; */ /* package */ final class ScaleToFitFrameProcessor implements GlFrameProcessor { + /** A builder for {@link ScaleToFitFrameProcessor} instances. */ + public static final class Builder { + // Mandatory field. + Context context; + + // Optional field. + private int outputHeight; + private float scaleX; + private float scaleY; + private float rotationDegrees; + + /** + * Creates a builder with default values. + * + * @param context The {@link Context}. + */ + public Builder(Context context) { + this.context = context; + + outputHeight = C.LENGTH_UNSET; + scaleX = 1; + scaleY = 1; + rotationDegrees = 0; + } + + /** + * Sets the x and y axis scaling factors to apply to each frame's width and height. + * + *

The values default to 1, which corresponds to not scaling along both axes. + * + * @param scaleX The multiplier by which the frame will scale horizontally, along the x-axis. + * @param scaleY The multiplier by which the frame will scale vertically, along the y-axis. + * @return This builder. + */ + public Builder setScale(float scaleX, float scaleY) { + this.scaleX = scaleX; + this.scaleY = scaleY; + return this; + } + + /** + * Sets the counterclockwise rotation degrees. + * + *

The default value, 0, corresponds to not applying any rotation. + * + * @param rotationDegrees The counterclockwise rotation, in degrees. + * @return This builder. + */ + public Builder setRotationDegrees(float rotationDegrees) { + this.rotationDegrees = rotationDegrees; + return this; + } + + /** + * Sets the output resolution using the output height. + * + *

The default value {@link C#LENGTH_UNSET} corresponds to using the same height as the + * input. Output width of the displayed frame will scale to preserve the frame's aspect ratio + * after other transformations. + * + *

For example, a 1920x1440 frame can be scaled to 640x480 by calling setResolution(480). + * + * @param outputHeight The output height of the displayed frame, in pixels. + * @return This builder. + */ + public Builder setResolution(int outputHeight) { + this.outputHeight = outputHeight; + return this; + } + + public ScaleToFitFrameProcessor build() { + return new ScaleToFitFrameProcessor(context, scaleX, scaleY, rotationDegrees, outputHeight); + } + } + static { GlUtil.glAssertionsEnabled = true; } @@ -58,15 +133,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * Creates a new instance. * * @param context The {@link Context}. - * @param transformationMatrix The transformation matrix to apply to each frame. + * @param scaleX The multiplier by which the frame will scale horizontally, along the x-axis. + * @param scaleY The multiplier by which the frame will scale vertically, along the y-axis. + * @param rotationDegrees How much to rotate the frame counterclockwise, in degrees. * @param requestedHeight The height of the output frame, in pixels. */ - public ScaleToFitFrameProcessor( - Context context, Matrix transformationMatrix, int requestedHeight) { - // TODO(b/201293185): Replace transformationMatrix parameter with scale and rotation. + private ScaleToFitFrameProcessor( + Context context, float scaleX, float scaleY, float rotationDegrees, int requestedHeight) { this.context = context; - this.transformationMatrix = new Matrix(transformationMatrix); + this.transformationMatrix = new Matrix(); + this.transformationMatrix.postScale(scaleX, scaleY); + this.transformationMatrix.postRotate(rotationDegrees); this.requestedHeight = requestedHeight; inputWidth = C.LENGTH_UNSET; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java index 97ab61d24e..cded1514bf 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java @@ -18,7 +18,6 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; -import android.graphics.Matrix; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.MimeTypes; @@ -34,8 +33,10 @@ public final class TransformationRequest { /** A builder for {@link TransformationRequest} instances. */ public static final class Builder { - private Matrix transformationMatrix; private boolean flattenForSlowMotion; + private float scaleX; + private float scaleY; + private float rotationDegrees; private int outputHeight; @Nullable private String audioMimeType; @Nullable private String videoMimeType; @@ -48,43 +49,23 @@ public final class TransformationRequest { * {@link TransformationRequest}. */ public Builder() { - transformationMatrix = new Matrix(); + scaleX = 1; + scaleY = 1; + rotationDegrees = 0; outputHeight = C.LENGTH_UNSET; } private Builder(TransformationRequest transformationRequest) { - this.transformationMatrix = new Matrix(transformationRequest.transformationMatrix); this.flattenForSlowMotion = transformationRequest.flattenForSlowMotion; + this.scaleX = transformationRequest.scaleX; + this.scaleY = transformationRequest.scaleY; + this.rotationDegrees = transformationRequest.rotationDegrees; this.outputHeight = transformationRequest.outputHeight; this.audioMimeType = transformationRequest.audioMimeType; this.videoMimeType = transformationRequest.videoMimeType; this.enableHdrEditing = transformationRequest.enableHdrEditing; } - /** - * Sets the transformation matrix. - * - *

The default value is to apply no change. - * - *

This can be used to perform operations supported by {@link Matrix}, like scaling and - * rotating the video. - * - *

The video dimensions will be on the x axis, from -aspectRatio to aspectRatio, and on the y - * axis, from -1 to 1. - * - *

For now, resolution will not be affected by this method. - * - * @param transformationMatrix The transformation to apply to video frames. - * @return This builder. - */ - public Builder setTransformationMatrix(Matrix transformationMatrix) { - // TODO(b/201293185): Implement an AdvancedFrameEditor to handle translation, as the current - // transformationMatrix is automatically adjusted to focus on the original pixels and - // effectively undo translations. - this.transformationMatrix = new Matrix(transformationMatrix); - return this; - } - /** * Sets whether the input should be flattened for media containing slow motion markers. * @@ -116,6 +97,36 @@ public final class TransformationRequest { return this; } + /** + * Sets the x and y axis scaling factors to apply to each frame's width and height, stretching + * the video along these axes appropriately. + * + *

The values default to 1, which corresponds to not scaling along both axes. + * + * @param scaleX The multiplier by which the frame will scale horizontally, along the x-axis. + * @param scaleY The multiplier by which the frame will scale vertically, along the y-axis. + * @return This builder. + */ + public Builder setScale(float scaleX, float scaleY) { + this.scaleX = scaleX; + this.scaleY = scaleY; + return this; + } + + /** + * Sets the rotation, in degrees, counterclockwise, to apply to each frame, automatically + * adjusting the frame's width and height to preserve all input pixels. + * + *

The default value, 0, corresponds to not applying any rotation. + * + * @param rotationDegrees The counterclockwise rotation, in degrees. + * @return This builder. + */ + public Builder setRotationDegrees(float rotationDegrees) { + this.rotationDegrees = rotationDegrees; + return this; + } + /** * Sets the output resolution using the output height. * @@ -205,8 +216,10 @@ public final class TransformationRequest { /** Builds a {@link TransformationRequest} instance. */ public TransformationRequest build() { return new TransformationRequest( - transformationMatrix, flattenForSlowMotion, + scaleX, + scaleY, + rotationDegrees, outputHeight, audioMimeType, videoMimeType, @@ -214,18 +227,32 @@ public final class TransformationRequest { } } - /** - * A {@link Matrix transformation matrix} to apply to video frames. - * - * @see Builder#setTransformationMatrix(Matrix) - */ - public final Matrix transformationMatrix; /** * Whether the input should be flattened for media containing slow motion markers. * * @see Builder#setFlattenForSlowMotion(boolean) */ public final boolean flattenForSlowMotion; + /** + * The requested scale factor, on the x-axis, of the output video, or 1 if inferred from the + * input. + * + * @see Builder#setScale(float, float) + */ + public final float scaleX; + /** + * The requested scale factor, on the y-axis, of the output video, or 1 if inferred from the + * input. + * + * @see Builder#setScale(float, float) + */ + public final float scaleY; + /** + * The requested rotation, in degrees, of the output video, or 0 if inferred from the input. + * + * @see Builder#setRotationDegrees(float) + */ + public final float rotationDegrees; /** * The requested height of the output video, or {@link C#LENGTH_UNSET} if inferred from the input. * @@ -254,14 +281,18 @@ public final class TransformationRequest { public final boolean enableHdrEditing; private TransformationRequest( - Matrix transformationMatrix, boolean flattenForSlowMotion, + float scaleX, + float scaleY, + float rotationDegrees, int outputHeight, @Nullable String audioMimeType, @Nullable String videoMimeType, boolean enableHdrEditing) { - this.transformationMatrix = transformationMatrix; this.flattenForSlowMotion = flattenForSlowMotion; + this.scaleX = scaleX; + this.scaleY = scaleY; + this.rotationDegrees = rotationDegrees; this.outputHeight = outputHeight; this.audioMimeType = audioMimeType; this.videoMimeType = videoMimeType; @@ -277,8 +308,10 @@ public final class TransformationRequest { return false; } TransformationRequest that = (TransformationRequest) o; - return transformationMatrix.equals(that.transformationMatrix) - && flattenForSlowMotion == that.flattenForSlowMotion + return flattenForSlowMotion == that.flattenForSlowMotion + && scaleX == that.scaleX + && scaleY == that.scaleY + && rotationDegrees == that.rotationDegrees && outputHeight == that.outputHeight && Util.areEqual(audioMimeType, that.audioMimeType) && Util.areEqual(videoMimeType, that.videoMimeType) @@ -287,8 +320,10 @@ public final class TransformationRequest { @Override public int hashCode() { - int result = transformationMatrix.hashCode(); - result = 31 * result + (flattenForSlowMotion ? 1 : 0); + int result = (flattenForSlowMotion ? 1 : 0); + result = 31 * result + Float.floatToIntBits(scaleX); + result = 31 * result + Float.floatToIntBits(scaleY); + result = 31 * result + Float.floatToIntBits(rotationDegrees); result = 31 * result + outputHeight; result = 31 * result + (audioMimeType != null ? audioMimeType.hashCode() : 0); result = 31 * result + (videoMimeType != null ? videoMimeType.hashCode() : 0); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 9b2f8825bd..7f548104cd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -168,6 +168,10 @@ public final class Transformer { /** * Sets the {@link TransformationRequest} which configures the editing and transcoding options. * + *

Actual applied values may differ, per device capabilities. {@link + * Listener#onFallbackApplied(MediaItem, TransformationRequest, TransformationRequest)} will be + * invoked with the actual applied values. + * * @param transformationRequest The {@link TransformationRequest}. * @return This builder. */ diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index bcb5541d73..32b251d000 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -113,14 +113,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { return false; } - if (transformationRequest.outputHeight != C.LENGTH_UNSET - && transformationRequest.outputHeight != inputFormat.height) { + if (transformationRequest.rotationDegrees != 0f) { return false; } - if (!transformationRequest.transformationMatrix.isIdentity()) { - // TODO(b/201293185, b/214010296): Move FrameProcessor transformationMatrix calculation / - // adjustments out of the VideoTranscodingSamplePipeline, so that we can skip transcoding when - // adjustments result in identity matrices. + if (transformationRequest.scaleX != 1f) { + return false; + } + if (transformationRequest.scaleY != 1f) { + return false; + } + if (transformationRequest.outputHeight != C.LENGTH_UNSET + && transformationRequest.outputHeight != inputFormat.height) { return false; } return true; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 0b25425bd9..9c52664402 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -69,10 +69,11 @@ import org.checkerframework.dataflow.qual.Pure; (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor( - context, - transformationRequest.transformationMatrix, - transformationRequest.outputHeight); + new ScaleToFitFrameProcessor.Builder(context) + .setScale(transformationRequest.scaleX, transformationRequest.scaleY) + .setRotationDegrees(transformationRequest.rotationDegrees) + .setResolution(transformationRequest.outputHeight) + .build(); Size requestedEncoderDimensions = scaleToFitFrameProcessor.configureOutputSize(decodedWidth, decodedHeight); outputRotationDegrees = scaleToFitFrameProcessor.getOutputRotationDegrees(); diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java index fe6344b6b2..df24f82d0b 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java @@ -19,9 +19,7 @@ import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; -import android.graphics.Matrix; import android.util.Size; -import androidx.media3.common.C; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; import org.junit.runner.RunWith; @@ -37,11 +35,10 @@ public final class ScaleToFitFrameProcessorTest { @Test public void configureOutputDimensions_noEdits_producesExpectedOutput() { - Matrix identityMatrix = new Matrix(); int inputWidth = 200; int inputHeight = 150; ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()).build(); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); @@ -53,9 +50,8 @@ public final class ScaleToFitFrameProcessorTest { @Test public void initializeBeforeConfigure_throwsIllegalStateException() { - Matrix identityMatrix = new Matrix(); ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()).build(); // configureOutputDimensions not called before initialize. assertThrows( @@ -65,9 +61,8 @@ public final class ScaleToFitFrameProcessorTest { @Test public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() { - Matrix identityMatrix = new Matrix(); ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()).build(); // configureOutputDimensions not called before initialize. assertThrows(IllegalStateException.class, scaleToFitFrameProcessor::getOutputRotationDegrees); @@ -75,12 +70,12 @@ public final class ScaleToFitFrameProcessorTest { @Test public void configureOutputDimensions_scaleNarrow_producesExpectedOutput() { - Matrix scaleNarrowMatrix = new Matrix(); - scaleNarrowMatrix.postScale(/* sx= */ .5f, /* sy= */ 1.0f); int inputWidth = 200; int inputHeight = 150; ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor(getApplicationContext(), scaleNarrowMatrix, C.LENGTH_UNSET); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setScale(/* scaleX= */ .5f, /* scaleY= */ 1f) + .build(); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); @@ -92,12 +87,12 @@ public final class ScaleToFitFrameProcessorTest { @Test public void configureOutputDimensions_scaleWide_producesExpectedOutput() { - Matrix scaleNarrowMatrix = new Matrix(); - scaleNarrowMatrix.postScale(/* sx= */ 2f, /* sy= */ 1.0f); int inputWidth = 200; int inputHeight = 150; ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor(getApplicationContext(), scaleNarrowMatrix, C.LENGTH_UNSET); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setScale(/* scaleX= */ 2f, /* scaleY= */ 1f) + .build(); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); @@ -108,13 +103,30 @@ public final class ScaleToFitFrameProcessorTest { } @Test - public void configureOutputDimensions_rotate90_producesExpectedOutput() { - Matrix rotate90Matrix = new Matrix(); - rotate90Matrix.postRotate(/* degrees= */ 90); + public void configureOutputDimensions_scaleTall_producesExpectedOutput() { int inputWidth = 200; int inputHeight = 150; ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor(getApplicationContext(), rotate90Matrix, C.LENGTH_UNSET); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setScale(/* scaleX= */ 1f, /* scaleY= */ 2f) + .build(); + + Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); + assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); + assertThat(outputSize.getWidth()).isEqualTo(inputHeight * 2); + assertThat(outputSize.getHeight()).isEqualTo(inputWidth); + } + + @Test + public void configureOutputDimensions_rotate90_producesExpectedOutput() { + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setRotationDegrees(90) + .build(); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); @@ -126,12 +138,12 @@ public final class ScaleToFitFrameProcessorTest { @Test public void configureOutputDimensions_rotate45_producesExpectedOutput() { - Matrix rotate45Matrix = new Matrix(); - rotate45Matrix.postRotate(/* degrees= */ 45); int inputWidth = 200; int inputHeight = 150; ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor(getApplicationContext(), rotate45Matrix, C.LENGTH_UNSET); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setRotationDegrees(45) + .build(); long expectedOutputWidthHeight = 247; Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); @@ -144,12 +156,13 @@ public final class ScaleToFitFrameProcessorTest { @Test public void configureOutputDimensions_setResolution_producesExpectedOutput() { - Matrix identityMatrix = new Matrix(); int inputWidth = 200; int inputHeight = 150; int requestedHeight = 300; ScaleToFitFrameProcessor scaleToFitFrameProcessor = - new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, requestedHeight); + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setResolution(requestedHeight) + .build(); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformationRequestTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformationRequestTest.java index 848e5a0c73..ff90ef809b 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformationRequestTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformationRequestTest.java @@ -18,7 +18,6 @@ package androidx.media3.transformer; import static com.google.common.truth.Truth.assertThat; -import android.graphics.Matrix; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; @@ -35,15 +34,12 @@ public class TransformationRequestTest { } private static TransformationRequest createTestTransformationRequest() { - Matrix transformationMatrix = new Matrix(); - transformationMatrix.preRotate(36); - transformationMatrix.postTranslate((float) 0.5, (float) -0.2); - return new TransformationRequest.Builder() .setFlattenForSlowMotion(true) .setAudioMimeType(MimeTypes.AUDIO_AAC) .setVideoMimeType(MimeTypes.VIDEO_H264) - .setTransformationMatrix(transformationMatrix) + .setRotationDegrees(45) + .setScale(/* scaleX= */ 1f, /* scaleY= */ 2f) .build(); } }