Cropping is applied before setting resolution. + */ +// TODO(b/213190310): Implement aspect ratio changes, etc. public final class PresentationFrameProcessor implements GlFrameProcessor { - /** A builder for {@link PresentationFrameProcessor} instances. */ public static final class Builder { // Mandatory field. private final Context context; - // Optional field. - private int outputHeight; + // Optional fields. + private int heightPixels; + private float cropLeft; + private float cropRight; + private float cropBottom; + private float cropTop; /** * Creates a builder with default values. @@ -49,7 +58,11 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { */ public Builder(Context context) { this.context = context; - outputHeight = C.LENGTH_UNSET; + heightPixels = C.LENGTH_UNSET; + cropLeft = -1f; + cropRight = 1f; + cropBottom = -1f; + cropTop = 1f; } /** @@ -59,18 +72,49 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { * input. Output width of the displayed frame will scale to preserve the frame's aspect ratio * after other transformations. * - *
For example, a 1920x1440 frame can be scaled to 640x480 by calling setResolution(480). + *
For example, a 1920x1440 frame can be scaled to 640x480 by calling {@code + * setResolution(480)}. * - * @param outputHeight The output height of the displayed frame, in pixels. + * @param height The output height of the displayed frame, in pixels. * @return This builder. */ - public Builder setResolution(int outputHeight) { - this.outputHeight = outputHeight; + public Builder setResolution(int height) { + this.heightPixels = height; + return this; + } + + /** + * Crops a smaller (or larger frame), per normalized device coordinates (NDC), where the input + * frame corresponds to the square ranging from -1 to 1 on the x and y axes. + * + *
{@code left} and {@code bottom} default to -1, and {@code right} and {@code top} default + * to 1. To crop to a smaller subset of the input frame, use values between -1 and 1. To crop to + * a larger frame, use values below -1 and above 1. + * + *
Width and height values set may be rescaled by {@link #setResolution(int)}. + * + * @param left The left edge of the output frame, in NDC. Must be less than {@code right}. + * @param right The right edge of the output frame, in NDC. Must be greater than {@code left}. + * @param bottom The bottom edge of the output frame, in NDC. Must be less than {@code top}. + * @param top The top edge of the output frame, in NDC. Must be greater than {@code bottom}. + * @return This builder. + */ + public Builder setCrop(float left, float right, float bottom, float top) { + checkArgument( + right > left, "right value " + right + " should be greater than left value " + left); + checkArgument( + top > bottom, "top value " + top + " should be greater than bottom value " + bottom); + cropLeft = left; + cropRight = right; + cropBottom = bottom; + cropTop = top; + return this; } public PresentationFrameProcessor build() { - return new PresentationFrameProcessor(context, outputHeight); + return new PresentationFrameProcessor( + context, heightPixels, cropLeft, cropRight, cropBottom, cropTop); } } @@ -79,24 +123,34 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { } private final Context context; - private final int requestedHeight; + private final int requestedHeightPixels; + private final float cropLeft; + private final float cropRight; + private final float cropBottom; + private final float cropTop; - private @MonotonicNonNull Size outputSize; private int outputRotationDegrees; + private @MonotonicNonNull Size outputSize; private @MonotonicNonNull Matrix transformationMatrix; private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; - /** - * Creates a new instance. - * - * @param context The {@link Context}. - * @param requestedHeight The height of the output frame, in pixels. - */ - private PresentationFrameProcessor(Context context, int requestedHeight) { + /** Creates a new instance. */ + private PresentationFrameProcessor( + Context context, + int requestedHeightPixels, + float cropLeft, + float cropRight, + float cropBottom, + float cropTop) { this.context = context; - this.requestedHeight = requestedHeight; + this.requestedHeightPixels = requestedHeightPixels; + this.cropLeft = cropLeft; + this.cropRight = cropRight; + this.cropBottom = cropBottom; + this.cropTop = cropTop; outputRotationDegrees = C.LENGTH_UNSET; + transformationMatrix = new Matrix(); } @Override @@ -136,16 +190,20 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { } @EnsuresNonNull("transformationMatrix") - @VisibleForTesting // Allows roboletric testing of output size calculation without OpenGL. + @VisibleForTesting // Allows robolectric testing of output size calculation without OpenGL. /* package */ void configureOutputSizeAndTransformationMatrix(int inputWidth, int inputHeight) { transformationMatrix = new Matrix(); - int displayWidth = inputWidth; - int displayHeight = inputHeight; - // Scale width and height to desired requestedHeight, preserving aspect ratio. - if (requestedHeight != C.LENGTH_UNSET && requestedHeight != displayHeight) { - displayWidth = Math.round((float) requestedHeight * displayWidth / displayHeight); - displayHeight = requestedHeight; + + Size cropSize = applyCrop(inputWidth, inputHeight); + int displayWidth = cropSize.getWidth(); + int displayHeight = cropSize.getHeight(); + + // Scale width and height to desired requestedHeightPixels, preserving aspect ratio. + if (requestedHeightPixels != C.LENGTH_UNSET && requestedHeightPixels != displayHeight) { + displayWidth = Math.round((float) requestedHeightPixels * displayWidth / displayHeight); + displayHeight = requestedHeightPixels; } + // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded // frame before encoding, so the encoded frame's width >= height, and set // outputRotationDegrees to ensure the frame is displayed in the correct orientation. @@ -160,4 +218,19 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { outputSize = new Size(displayWidth, displayHeight); } } + + @RequiresNonNull("transformationMatrix") + private Size applyCrop(int inputWidth, int inputHeight) { + float scaleX = (cropRight - cropLeft) / GlUtil.LENGTH_NDC; + float scaleY = (cropTop - cropBottom) / GlUtil.LENGTH_NDC; + float centerX = (cropLeft + cropRight) / 2; + float centerY = (cropBottom + cropTop) / 2; + + transformationMatrix.postTranslate(-centerX, -centerY); + transformationMatrix.postScale(1f / scaleX, 1f / scaleY); + + int outputWidth = Math.round(inputWidth * scaleX); + int outputHeight = Math.round(inputHeight * scaleY); + return new Size(outputWidth, outputHeight); + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java index 052a86b1a8..984943054b 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java @@ -144,7 +144,7 @@ public final class ScaleToFitFrameProcessor implements GlFrameProcessor { } @EnsuresNonNull("adjustedTransformationMatrix") - @VisibleForTesting // Allows roboletric testing of output size calculation without OpenGL. + @VisibleForTesting // Allows robolectric testing of output size calculation without OpenGL. /* package */ void configureOutputSizeAndTransformationMatrix(int inputWidth, int inputHeight) { adjustedTransformationMatrix = new Matrix(transformationMatrix); @@ -164,22 +164,21 @@ public final class ScaleToFitFrameProcessor implements GlFrameProcessor { // Modify transformationMatrix to keep input pixels. float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}}; - float xMin = Float.MAX_VALUE; - float xMax = Float.MIN_VALUE; - float yMin = Float.MAX_VALUE; - float yMax = Float.MIN_VALUE; + float minX = Float.MAX_VALUE; + float maxX = Float.MIN_VALUE; + float minY = Float.MAX_VALUE; + float maxY = Float.MIN_VALUE; for (float[] transformOnNdcPoint : transformOnNdcPoints) { adjustedTransformationMatrix.mapPoints(transformOnNdcPoint); - xMin = min(xMin, transformOnNdcPoint[0]); - xMax = max(xMax, transformOnNdcPoint[0]); - yMin = min(yMin, transformOnNdcPoint[1]); - yMax = max(yMax, transformOnNdcPoint[1]); + minX = min(minX, transformOnNdcPoint[0]); + maxX = max(maxX, transformOnNdcPoint[0]); + minY = min(minY, transformOnNdcPoint[1]); + maxY = max(maxY, transformOnNdcPoint[1]); } - float ndcWidthAndHeight = 2f; // Length from -1 to 1. - float xScale = (xMax - xMin) / ndcWidthAndHeight; - float yScale = (yMax - yMin) / ndcWidthAndHeight; - adjustedTransformationMatrix.postScale(1f / xScale, 1f / yScale); - outputSize = new Size(Math.round(inputWidth * xScale), Math.round(inputHeight * yScale)); + float scaleX = (maxX - minX) / GlUtil.LENGTH_NDC; + float scaleY = (maxY - minY) / GlUtil.LENGTH_NDC; + adjustedTransformationMatrix.postScale(1f / scaleX, 1f / scaleY); + outputSize = new Size(Math.round(inputWidth * scaleX), Math.round(inputHeight * scaleY)); } } diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java index ae40c13938..f7cf092d17 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertThrows; import android.util.Size; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.util.GlUtil; import org.junit.Test; import org.junit.runner.RunWith; @@ -95,6 +96,83 @@ public final class PresentationFrameProcessorTest { assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); } + @Test + public void getOutputSize_setCrop_changesDimensions() { + int inputWidth = 300; + int inputHeight = 200; + float left = -.5f; + float right = .5f; + float bottom = .5f; + float top = 1f; + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()) + .setCrop(left, right, bottom, top) + .build(); + + presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); + Size outputSize = presentationFrameProcessor.getOutputSize(); + + assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC); + int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC); + assertThat(outputSize.getWidth()).isEqualTo(expectedPostCropWidth); + assertThat(outputSize.getHeight()).isEqualTo(expectedPostCropHeight); + } + + @Test + public void getOutputSize_setCropAndSetResolution_changesDimensions() { + int inputWidth = 300; + int inputHeight = 200; + float left = -.5f; + float right = .5f; + float bottom = .5f; + float top = 1f; + int requestedHeight = 100; + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()) + .setCrop(left, right, bottom, top) + .setResolution(100) + .build(); + + presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); + Size outputSize = presentationFrameProcessor.getOutputSize(); + + assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC); + int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC); + assertThat(outputSize.getWidth()) + .isEqualTo( + Math.round((float) requestedHeight * expectedPostCropWidth / expectedPostCropHeight)); + assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); + } + + @Test + public void getOutputSize_setResolutionAndCrop_changesDimensions() { + int inputWidth = 300; + int inputHeight = 200; + float left = -.5f; + float right = .5f; + float bottom = .5f; + float top = 1f; + int requestedHeight = 100; + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()) + .setResolution(100) + .setCrop(left, right, bottom, top) + .build(); + + presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); + Size outputSize = presentationFrameProcessor.getOutputSize(); + + assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC); + int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC); + assertThat(outputSize.getWidth()) + .isEqualTo( + Math.round((float) requestedHeight * expectedPostCropWidth / expectedPostCropHeight)); + assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); + } + @Test public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() { PresentationFrameProcessor presentationFrameProcessor = diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_crop_larger.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_crop_larger.png new file mode 100644 index 0000000000..6b45079102 Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_crop_larger.png differ diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_crop_smaller.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_crop_smaller.png new file mode 100644 index 0000000000..1191952f1f Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_crop_smaller.png differ