diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java index a349120dfc..aff80ddbb5 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java @@ -221,6 +221,30 @@ public final class GlUtil { } } + /** + * Asserts that dimensions are valid for a texture. + * + * @param width The width for a texture. + * @param height The height for a texture. + * @throws GlException If the texture width or height is invalid. + */ + public static void assertValidTextureDimensions(int width, int height) { + // TODO(b/201293185): Consider handling adjustments for resolutions > GL_MAX_TEXTURE_SIZE + // (ex. downscaling appropriately) in a FrameProcessor instead of asserting incorrect values. + + // For valid GL resolutions, see: + // https://www.khronos.org/registry/OpenGL-Refpages/es2.0/xhtml/glTexImage2D.xml + int[] maxTextureSizeBuffer = new int[1]; + GLES20.glGetIntegerv(GLES20.GL_MAX_TEXTURE_SIZE, maxTextureSizeBuffer, 0); + int maxTextureSize = maxTextureSizeBuffer[0]; + if (width < 0 || height < 0) { + throwGlException("width or height is less than 0"); + } + if (width > maxTextureSize || height > maxTextureSize) { + throwGlException("width or height is greater than GL_MAX_TEXTURE_SIZE " + maxTextureSize); + } + } + /** * Makes the specified {@code eglSurface} the render target, using a viewport of {@code width} by * {@code height} pixels. diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessorTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorPixelTest.java similarity index 82% rename from library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessorTest.java rename to library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorPixelTest.java index fff2c59b0f..f34ac26630 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessorTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorPixelTest.java @@ -39,7 +39,7 @@ import org.junit.Test; import org.junit.runner.RunWith; /** - * Pixel test for frame processing via {@link TransformationFrameProcessor}. + * Pixel test for frame processing via {@link AdvancedFrameProcessor}. * *

Expected images are taken from an emulator, so tests on different emulators or physical * devices may fail. To test on other devices, please increase the {@link @@ -47,7 +47,7 @@ import org.junit.runner.RunWith; * as recommended in {@link FrameEditorDataProcessingTest}. */ @RunWith(AndroidJUnit4.class) -public final class TransformationFrameProcessorTest { +public final class AdvancedFrameProcessorPixelTest { static { GlUtil.glAssertionsEnabled = true; @@ -55,7 +55,7 @@ public final class TransformationFrameProcessorTest { private final EGLDisplay eglDisplay = GlUtil.createEglDisplay(); private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay); - private @MonotonicNonNull GlFrameProcessor transformationFrameProcessor; + private @MonotonicNonNull GlFrameProcessor advancedFrameProcessor; private int inputTexId; private int outputTexId; // TODO(b/214975934): Once the frame processors are allowed to have different input and output @@ -82,8 +82,8 @@ public final class TransformationFrameProcessorTest { @After public void release() { - if (transformationFrameProcessor != null) { - transformationFrameProcessor.release(); + if (advancedFrameProcessor != null) { + advancedFrameProcessor.release(); } GlUtil.destroyEglContext(eglDisplay, eglContext); } @@ -92,12 +92,11 @@ public final class TransformationFrameProcessorTest { public void updateProgramAndDraw_noEdits_producesExpectedOutput() throws Exception { final String testId = "updateProgramAndDraw_noEdits"; Matrix identityMatrix = new Matrix(); - transformationFrameProcessor = - new TransformationFrameProcessor(getApplicationContext(), identityMatrix); - transformationFrameProcessor.initialize(inputTexId); + advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), identityMatrix); + advancedFrameProcessor.initialize(inputTexId); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING); - transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); + advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -115,13 +114,13 @@ public final class TransformationFrameProcessorTest { final String testId = "updateProgramAndDraw_translateRight"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - transformationFrameProcessor = - new TransformationFrameProcessor(getApplicationContext(), translateRightMatrix); - transformationFrameProcessor.initialize(inputTexId); + advancedFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix); + advancedFrameProcessor.initialize(inputTexId); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); - transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); + advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -139,13 +138,12 @@ public final class TransformationFrameProcessorTest { final String testId = "updateProgramAndDraw_scaleNarrow"; Matrix scaleNarrowMatrix = new Matrix(); scaleNarrowMatrix.postScale(.5f, 1.2f); - transformationFrameProcessor = - new TransformationFrameProcessor(getApplicationContext(), scaleNarrowMatrix); - transformationFrameProcessor.initialize(inputTexId); + advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), scaleNarrowMatrix); + advancedFrameProcessor.initialize(inputTexId); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING); - transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); + advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -161,17 +159,13 @@ public final class TransformationFrameProcessorTest { @Test public void updateProgramAndDraw_rotate90_producesExpectedOutput() throws Exception { final String testId = "updateProgramAndDraw_rotate90"; - // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline - // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can - // test that rotation doesn't distort the image. Matrix rotate90Matrix = new Matrix(); rotate90Matrix.postRotate(/* degrees= */ 90); - transformationFrameProcessor = - new TransformationFrameProcessor(getApplicationContext(), rotate90Matrix); - transformationFrameProcessor.initialize(inputTexId); + advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), rotate90Matrix); + advancedFrameProcessor.initialize(inputTexId); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); - transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); + advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/BitmapTestUtil.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/BitmapTestUtil.java index f933efd919..cb31c64cf5 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/BitmapTestUtil.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/BitmapTestUtil.java @@ -55,6 +55,10 @@ public class BitmapTestUtil { "media/bitmap/sample_mp4_first_frame_scale_narrow.png"; public static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING = "media/bitmap/sample_mp4_first_frame_rotate90.png"; + public static final String REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_request_output_height.png"; + public static final String ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png"; /** * Maximum allowed average pixel difference between the expected and actual edited images for the * test to pass. The value is chosen so that differences in decoder behavior across emulator diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java index 45dab1c643..dc2e7fa24d 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java @@ -18,6 +18,8 @@ package com.google.android.exoplayer2.transformer; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.android.exoplayer2.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_STRING; import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static com.google.android.exoplayer2.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING; +import static com.google.android.exoplayer2.transformer.BitmapTestUtil.ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static com.google.android.exoplayer2.transformer.BitmapTestUtil.ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static com.google.android.exoplayer2.transformer.BitmapTestUtil.SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static com.google.android.exoplayer2.transformer.BitmapTestUtil.TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING; @@ -34,8 +36,10 @@ import android.media.ImageReader; import android.media.MediaCodec; import android.media.MediaExtractor; import android.media.MediaFormat; +import android.util.Pair; import androidx.annotation.Nullable; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.util.MimeTypes; import java.nio.ByteBuffer; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -84,7 +88,9 @@ public final class FrameEditorDataProcessingTest { public void processData_noEdits_producesExpectedOutput() throws Exception { final String testId = "processData_noEdits"; Matrix identityMatrix = new Matrix(); - setUpAndPrepareFirstFrame(identityMatrix); + GlFrameProcessor glFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), identityMatrix); + setUpAndPrepareFirstFrame(glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -103,7 +109,9 @@ public final class FrameEditorDataProcessingTest { final String testId = "processData_translateRight"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - setUpAndPrepareFirstFrame(translateRightMatrix); + GlFrameProcessor glFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix); + setUpAndPrepareFirstFrame(glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); @@ -123,7 +131,9 @@ public final class FrameEditorDataProcessingTest { final String testId = "processData_scaleNarrow"; Matrix scaleNarrowMatrix = new Matrix(); scaleNarrowMatrix.postScale(.5f, 1.2f); - setUpAndPrepareFirstFrame(scaleNarrowMatrix); + GlFrameProcessor glFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), scaleNarrowMatrix); + setUpAndPrepareFirstFrame(glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING); @@ -141,12 +151,11 @@ public final class FrameEditorDataProcessingTest { @Test public void processData_rotate90_producesExpectedOutput() throws Exception { final String testId = "processData_rotate90"; - // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline - // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can - // test that rotation doesn't distort the image. Matrix rotate90Matrix = new Matrix(); rotate90Matrix.postRotate(/* degrees= */ 90); - setUpAndPrepareFirstFrame(rotate90Matrix); + GlFrameProcessor glFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), rotate90Matrix); + setUpAndPrepareFirstFrame(glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -160,12 +169,70 @@ public final class FrameEditorDataProcessingTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } - private void setUpAndPrepareFirstFrame(Matrix transformationMatrix) throws Exception { + @Test + public void processData_requestOutputHeight_producesExpectedOutput() throws Exception { + final String testId = "processData_requestOutputHeight"; + // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline + // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can + // test that rotation doesn't distort the image. + Matrix identityMatrix = new Matrix(); + GlFrameProcessor glFrameProcessor = + new ScaleToFitFrameProcessor( + getApplicationContext(), identityMatrix, /* requestedHeight= */ 480); + setUpAndPrepareFirstFrame(glFrameProcessor); + Bitmap expectedBitmap = + BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + BitmapTestUtil.saveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void processData_rotate45_scaleToFit_producesExpectedOutput() throws Exception { + final String testId = "processData_rotate45_scaleToFit"; + // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline + // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can + // test that rotation doesn't distort the image. + Matrix rotate45Matrix = new Matrix(); + rotate45Matrix.postRotate(/* degrees= */ 45); + GlFrameProcessor glFrameProcessor = + new ScaleToFitFrameProcessor( + getApplicationContext(), rotate45Matrix, /* requestedHeight= */ C.LENGTH_UNSET); + setUpAndPrepareFirstFrame(glFrameProcessor); + Bitmap expectedBitmap = + BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + BitmapTestUtil.saveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + /** + * Set up and prepare the first frame from an input video, as well as relevant test + * infrastructure. The frame will be sent towards the {@link FrameEditor}, and may be accessed on + * the {@link FrameEditor}'s output {@code frameEditorOutputImageReader}. + * + * @param glFrameProcessor The frame processor that will apply changes to the input frame. + */ + private void setUpAndPrepareFirstFrame(GlFrameProcessor glFrameProcessor) throws Exception { // Set up the extractor to read the first video frame and get its format. MediaExtractor mediaExtractor = new MediaExtractor(); @Nullable MediaCodec mediaCodec = null; - try (AssetFileDescriptor afd = - getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) { + Context context = getApplicationContext(); + try (AssetFileDescriptor afd = context.getAssets().openFd(INPUT_MP4_ASSET_STRING)) { mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength()); for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) { @@ -175,18 +242,24 @@ public final class FrameEditorDataProcessingTest { } } - int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); - int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); + int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); + int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); + Pair outputDimensions = + glFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + int outputWidth = outputDimensions.first; + int outputHeight = outputDimensions.second; frameEditorOutputImageReader = - ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); - Context context = getApplicationContext(); + ImageReader.newInstance( + outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1); frameEditor = FrameEditor.create( context, - width, - height, + inputWidth, + inputHeight, + outputWidth, + outputHeight, PIXEL_WIDTH_HEIGHT_RATIO, - new TransformationFrameProcessor(context, transformationMatrix), + glFrameProcessor, frameEditorOutputImageReader.getSurface(), /* enableExperimentalHdrEditing= */ false, Transformer.DebugViewProvider.NONE); diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java index e60658197f..70cd72b3b6 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java @@ -28,8 +28,8 @@ import org.junit.Test; import org.junit.runner.RunWith; /** - * Test for {@link FrameEditor#create(Context, int, int, float, GlFrameProcessor, Surface, boolean, - * Transformer.DebugViewProvider) creating} a {@link FrameEditor}. + * Test for {@link FrameEditor#create(Context, int, int, int, int, float, GlFrameProcessor, Surface, + * boolean, Transformer.DebugViewProvider) creating} a {@link FrameEditor}. */ @RunWith(AndroidJUnit4.class) public final class FrameEditorTest { @@ -43,10 +43,12 @@ public final class FrameEditorTest { FrameEditor.create( context, + /* inputWidth= */ 200, + /* inputHeight= */ 100, /* outputWidth= */ 200, /* outputHeight= */ 100, /* pixelWidthHeightRatio= */ 1, - new TransformationFrameProcessor(context, new Matrix()), + new AdvancedFrameProcessor(context, new Matrix()), new Surface(new SurfaceTexture(false)), /* enableExperimentalHdrEditing= */ false, Transformer.DebugViewProvider.NONE); @@ -62,10 +64,12 @@ public final class FrameEditorTest { () -> FrameEditor.create( context, + /* inputWidth= */ 200, + /* inputHeight= */ 100, /* outputWidth= */ 200, /* outputHeight= */ 100, /* pixelWidthHeightRatio= */ 2, - new TransformationFrameProcessor(context, new Matrix()), + new AdvancedFrameProcessor(context, new Matrix()), new Surface(new SurfaceTexture(false)), /* enableExperimentalHdrEditing= */ false, Transformer.DebugViewProvider.NONE)); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessor.java similarity index 85% rename from library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessor.java rename to library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessor.java index f605bb1c0e..7d89981652 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessor.java @@ -20,13 +20,19 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import android.content.Context; import android.graphics.Matrix; import android.opengl.GLES20; +import android.util.Pair; import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlUtil; import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** Applies a transformation matrix in the vertex shader. */ -/* package */ class TransformationFrameProcessor implements GlFrameProcessor { +/** + * Applies a transformation matrix in the vertex shader. Operations are done on normalized device + * coordinates (-1 to 1 on x and y axes). No automatic adjustments (like done in {@link + * ScaleToFitFrameProcessor}) are applied on the transformation. Width and height are not modified. + * The background color will default to black. + */ +/* package */ final class AdvancedFrameProcessor implements GlFrameProcessor { static { GlUtil.glAssertionsEnabled = true; @@ -85,13 +91,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * Creates a new instance. * * @param context The {@link Context}. - * @param transformationMatrix The transformation matrix to apply to each frame. + * @param transformationMatrix The transformation matrix to apply to each frame. Operations are + * done on normalized device coordinates (-1 to 1 on x and y), and no automatic adjustments + * are applied on the transformation matrix. */ - public TransformationFrameProcessor(Context context, Matrix transformationMatrix) { + public AdvancedFrameProcessor(Context context, Matrix transformationMatrix) { this.context = context; this.transformationMatrix = transformationMatrix; } + @Override + public Pair configureOutputDimensions(int inputWidth, int inputHeight) { + return new Pair<>(inputWidth, inputHeight); + } + @Override public void initialize(int inputTexId) throws IOException { // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalCopyFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalCopyFrameProcessor.java index a57649c60a..2e5bb11bad 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalCopyFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalCopyFrameProcessor.java @@ -19,6 +19,7 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.GLES20; +import android.util.Pair; import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlUtil; import java.io.IOException; @@ -57,6 +58,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; } + @Override + public Pair configureOutputDimensions(int inputWidth, int inputHeight) { + return new Pair<>(inputWidth, inputHeight); + } + @Override public void initialize(int inputTexId) throws IOException { // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java index 184d559afc..d157cdddca 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java @@ -49,7 +49,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * is processed on a background thread as it becomes available. All input frames should be {@link * #registerInputFrame() registered} before they are rendered to the input surface. {@link * #hasPendingFrames()} can be used to check whether there are frames that have not been fully - * processed yet. Output is written to its {@link #create(Context, int, int, float, + * processed yet. Output is written to its {@link #create(Context, int, int, int, int, float, * GlFrameProcessor, Surface, boolean, Transformer.DebugViewProvider) output surface}. */ /* package */ final class FrameEditor { @@ -62,6 +62,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * Returns a new {@code FrameEditor} for applying changes to individual frames. * * @param context A {@link Context}. + * @param inputWidth The input width in pixels. + * @param inputHeight The input height in pixels. * @param outputWidth The output width in pixels. * @param outputHeight The output height in pixels. * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. @@ -76,8 +78,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; */ // TODO(b/214975934): Take a List as input and rename FrameEditor to // FrameProcessorChain. + // TODO(b/218488308): Remove the need to input outputWidth and outputHeight into FrameEditor, that + // stems from encoder fallback resolution. This could maybe be input into the last + // GlFrameProcessor in the FrameEditor instead of being input directly into the FrameEditor. public static FrameEditor create( Context context, + int inputWidth, + int inputHeight, int outputWidth, int outputHeight, float pixelWidthHeightRatio, @@ -121,6 +128,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; externalCopyFrameProcessor, transformationFrameProcessor, outputSurface, + inputWidth, + inputHeight, outputWidth, outputHeight, enableExperimentalHdrEditing, @@ -152,6 +161,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ExternalCopyFrameProcessor externalCopyFrameProcessor, GlFrameProcessor transformationFrameProcessor, Surface outputSurface, + int inputWidth, + int inputHeight, int outputWidth, int outputHeight, boolean enableExperimentalHdrEditing, @@ -182,7 +193,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); + GlUtil.assertValidTextureDimensions(outputWidth, outputHeight); int inputExternalTexId = GlUtil.createExternalTexture(); + externalCopyFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); externalCopyFrameProcessor.initialize(inputExternalTexId); int intermediateTexId = GlUtil.createTexture(outputWidth, outputHeight); int frameBuffer = GlUtil.createFboForTexture(intermediateTexId); @@ -227,9 +240,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final float[] textureTransformMatrix; /** - * Identifier of a framebuffer object associated with the intermediate texture that the output of - * the {@link ExternalCopyFrameProcessor} is written to and the {@link - * TransformationFrameProcessor} reads its input from. + * Identifier of a framebuffer object associated with the intermediate texture that receives + * output from the prior {@link ExternalCopyFrameProcessor}, and provides input for the following + * {@link GlFrameProcessor}. */ private final int frameBuffer; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlFrameProcessor.java index 3466fb1fbc..d21d697d23 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlFrameProcessor.java @@ -15,21 +15,36 @@ */ package com.google.android.exoplayer2.transformer; +import android.util.Pair; import java.io.IOException; -/** Manages a GLSL shader program for processing a frame. */ +/** + * Manages a GLSL shader program for processing a frame. + * + *

Methods must be called in the following order: + * + *

    + *
  1. The constructor, for implementation-specific arguments. + *
  2. {@link #configureOutputDimensions(int, int)}, to configure based on input dimensions. + *
  3. {@link #initialize(int)}, to set up graphics initialization. + *
  4. {@link #updateProgramAndDraw(long)}, to process one frame. + *
  5. {@link #release()}, upon conclusion of processing. + *
+ */ /* package */ interface GlFrameProcessor { - // TODO(b/214975934): Add getOutputDimensions(inputWidth, inputHeight) and move output dimension - // calculations out of the VideoTranscodingSamplePipeline into the frame processors. + /** + * Returns the output dimensions of frames processed through {@link #updateProgramAndDraw(long)}. + * + *

This method must be called before {@link #initialize(int)} and does not use OpenGL. + */ + Pair configureOutputDimensions(int inputWidth, int inputHeight); /** * Does any initialization necessary such as loading and compiling a GLSL shader programs. * *

This method may only be called after creating the OpenGL context and focusing a render * target. - * - * @param inputTexId The identifier of an OpenGL texture that the fragment shader can sample from. */ void initialize(int inputTexId) throws IOException; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java new file mode 100644 index 0000000000..ca44d71596 --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java @@ -0,0 +1,197 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.util.Assertions.checkState; +import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; +import static java.lang.Math.max; +import static java.lang.Math.min; + +import android.content.Context; +import android.graphics.Matrix; +import android.util.Pair; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.util.GlUtil; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; + +/** + * Applies a simple rotation and/or scale in the vertex shader. All input frames' pixels will be + * preserved, potentially changing the width and height of the video by scaling dimensions to fit. + * The background color will default to black. + */ +/* package */ final class ScaleToFitFrameProcessor implements GlFrameProcessor { + + static { + GlUtil.glAssertionsEnabled = true; + } + + private final Context context; + private final Matrix transformationMatrix; + private final int requestedHeight; + + private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; + private int inputWidth; + private int inputHeight; + private int outputWidth; + private int outputHeight; + private int outputRotationDegrees; + private @MonotonicNonNull Matrix adjustedTransformationMatrix; + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param transformationMatrix The transformation matrix to apply to each frame. + * @param requestedHeight The height of the output frame, in pixels. + */ + public ScaleToFitFrameProcessor( + Context context, Matrix transformationMatrix, int requestedHeight) { + // TODO(b/201293185): Replace transformationMatrix parameter with scale and rotation. + + this.context = context; + this.transformationMatrix = new Matrix(transformationMatrix); + this.requestedHeight = requestedHeight; + + inputWidth = C.LENGTH_UNSET; + inputHeight = C.LENGTH_UNSET; + outputWidth = C.LENGTH_UNSET; + outputHeight = C.LENGTH_UNSET; + outputRotationDegrees = C.LENGTH_UNSET; + } + + /** + * Returns {@link Format#rotationDegrees} for the output frame. + * + *

Return values may be {@code 0} or {@code 90} degrees. + * + *

This method can only be called after {@link #configureOutputDimensions(int, int)}. + */ + public int getOutputRotationDegrees() { + checkState(outputRotationDegrees != C.LENGTH_UNSET); + return outputRotationDegrees; + } + + /** + * Returns whether this ScaleToFitFrameProcessor will apply any changes on a frame. + * + *

The ScaleToFitFrameProcessor should only be used if this returns true. + * + *

This method can only be called after {@link #configureOutputDimensions(int, int)}. + */ + @RequiresNonNull("adjustedTransformationMatrix") + public boolean shouldProcess() { + return inputWidth != outputWidth + || inputHeight != outputHeight + || !adjustedTransformationMatrix.isIdentity(); + } + + @Override + @EnsuresNonNull("adjustedTransformationMatrix") + public Pair configureOutputDimensions(int inputWidth, int inputHeight) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + adjustedTransformationMatrix = new Matrix(transformationMatrix); + + int displayWidth = inputWidth; + int displayHeight = inputHeight; + if (!transformationMatrix.isIdentity()) { + float inputAspectRatio = (float) inputWidth / inputHeight; + // Scale frames by inputAspectRatio, to account for FrameEditor's normalized device + // coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular + // display of input pixels during transformations (ex. rotations). With scaling, + // transformationMatrix operations operate on a rectangle for x from -inputAspectRatio to + // inputAspectRatio, and y from -1 to 1. + adjustedTransformationMatrix.preScale(/* sx= */ inputAspectRatio, /* sy= */ 1f); + adjustedTransformationMatrix.postScale(/* sx= */ 1f / inputAspectRatio, /* sy= */ 1f); + + // Modify transformationMatrix to keep input pixels. + float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}}; + float xMin = Float.MAX_VALUE; + float xMax = Float.MIN_VALUE; + float yMin = Float.MAX_VALUE; + float yMax = Float.MIN_VALUE; + for (float[] transformOnNdcPoint : transformOnNdcPoints) { + adjustedTransformationMatrix.mapPoints(transformOnNdcPoint); + xMin = min(xMin, transformOnNdcPoint[0]); + xMax = max(xMax, transformOnNdcPoint[0]); + yMin = min(yMin, transformOnNdcPoint[1]); + yMax = max(yMax, transformOnNdcPoint[1]); + } + + float xCenter = (xMax + xMin) / 2f; + float yCenter = (yMax + yMin) / 2f; + adjustedTransformationMatrix.postTranslate(-xCenter, -yCenter); + + float ndcWidthAndHeight = 2f; // Length from -1 to 1. + float xScale = (xMax - xMin) / ndcWidthAndHeight; + float yScale = (yMax - yMin) / ndcWidthAndHeight; + adjustedTransformationMatrix.postScale(1f / xScale, 1f / yScale); + displayWidth = Math.round(inputWidth * xScale); + displayHeight = Math.round(inputHeight * yScale); + } + + // TODO(b/214975934): Move following requestedHeight and outputRotationDegrees logic into + // separate GlFrameProcessors (ex. Presentation). + + // Scale width and height to desired requestedHeight, preserving aspect ratio. + if (requestedHeight != C.LENGTH_UNSET && requestedHeight != displayHeight) { + displayWidth = Math.round((float) requestedHeight * displayWidth / displayHeight); + displayHeight = requestedHeight; + } + + // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded + // video before encoding, so the encoded video's width >= height, and set + // outputRotationDegrees to ensure the video is displayed in the correct orientation. + if (displayHeight > displayWidth) { + outputRotationDegrees = 90; + outputWidth = displayHeight; + outputHeight = displayWidth; + // TODO(b/201293185): After fragment shader transformations are implemented, put + // postRotate in a later GlFrameProcessor. + adjustedTransformationMatrix.postRotate(outputRotationDegrees); + } else { + outputRotationDegrees = 0; + outputWidth = displayWidth; + outputHeight = displayHeight; + } + + return new Pair<>(outputWidth, outputHeight); + } + + @Override + public void initialize(int inputTexId) throws IOException { + checkStateNotNull(adjustedTransformationMatrix); + advancedFrameProcessor = new AdvancedFrameProcessor(context, adjustedTransformationMatrix); + advancedFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + advancedFrameProcessor.initialize(inputTexId); + } + + @Override + public void updateProgramAndDraw(long presentationTimeNs) { + checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeNs); + } + + @Override + public void release() { + if (advancedFrameProcessor != null) { + advancedFrameProcessor.release(); + } + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index 98ff515a2e..4d119a91fe 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -18,16 +18,13 @@ package com.google.android.exoplayer2.transformer; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Util.SDK_INT; -import static java.lang.Math.max; -import static java.lang.Math.min; import android.content.Context; -import android.graphics.Matrix; import android.media.MediaCodec; import android.media.MediaFormat; +import android.util.Pair; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; -import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.util.Util; @@ -70,77 +67,20 @@ import org.checkerframework.dataflow.qual.Pure; (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; - float decodedAspectRatio = (float) decodedWidth / decodedHeight; - Matrix transformationMatrix = new Matrix(transformationRequest.transformationMatrix); - - int outputWidth = decodedWidth; - int outputHeight = decodedHeight; - if (!transformationMatrix.isIdentity()) { - // Scale frames by decodedAspectRatio, to account for FrameEditor's normalized device - // coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular display - // of input pixels during transformations (ex. rotations). With scaling, transformationMatrix - // operations operate on a rectangle for x from -decodedAspectRatio to decodedAspectRatio, and - // y from -1 to 1. - transformationMatrix.preScale(/* sx= */ decodedAspectRatio, /* sy= */ 1f); - transformationMatrix.postScale(/* sx= */ 1f / decodedAspectRatio, /* sy= */ 1f); - - float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}}; - float xMin = Float.MAX_VALUE; - float xMax = Float.MIN_VALUE; - float yMin = Float.MAX_VALUE; - float yMax = Float.MIN_VALUE; - for (float[] transformOnNdcPoint : transformOnNdcPoints) { - transformationMatrix.mapPoints(transformOnNdcPoint); - xMin = min(xMin, transformOnNdcPoint[0]); - xMax = max(xMax, transformOnNdcPoint[0]); - yMin = min(yMin, transformOnNdcPoint[1]); - yMax = max(yMax, transformOnNdcPoint[1]); - } - - float xCenter = (xMax + xMin) / 2f; - float yCenter = (yMax + yMin) / 2f; - transformationMatrix.postTranslate(-xCenter, -yCenter); - - float ndcWidthAndHeight = 2f; // Length from -1 to 1. - float xScale = (xMax - xMin) / ndcWidthAndHeight; - float yScale = (yMax - yMin) / ndcWidthAndHeight; - transformationMatrix.postScale(1f / xScale, 1f / yScale); - outputWidth = Math.round(decodedWidth * xScale); - outputHeight = Math.round(decodedHeight * yScale); - } - // Scale width and height to desired transformationRequest.outputHeight, preserving - // aspect ratio. - if (transformationRequest.outputHeight != C.LENGTH_UNSET - && transformationRequest.outputHeight != outputHeight) { - outputWidth = - Math.round((float) transformationRequest.outputHeight * outputWidth / outputHeight); - outputHeight = transformationRequest.outputHeight; - } - - // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded - // video before encoding, so the encoded video's width >= height, and set outputRotationDegrees - // to ensure the video is displayed in the correct orientation. - int requestedEncoderWidth; - int requestedEncoderHeight; - boolean swapEncodingDimensions = outputHeight > outputWidth; - if (swapEncodingDimensions) { - outputRotationDegrees = 90; - requestedEncoderWidth = outputHeight; - requestedEncoderHeight = outputWidth; - // TODO(b/201293185): After fragment shader transformations are implemented, put - // postRotate in a later vertex shader. - transformationMatrix.postRotate(outputRotationDegrees); - } else { - outputRotationDegrees = 0; - requestedEncoderWidth = outputWidth; - requestedEncoderHeight = outputHeight; - } + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor( + context, + transformationRequest.transformationMatrix, + transformationRequest.outputHeight); + Pair requestedEncoderDimensions = + scaleToFitFrameProcessor.configureOutputDimensions(decodedWidth, decodedHeight); + outputRotationDegrees = scaleToFitFrameProcessor.getOutputRotationDegrees(); Format requestedEncoderFormat = new Format.Builder() - .setWidth(requestedEncoderWidth) - .setHeight(requestedEncoderHeight) + .setWidth(requestedEncoderDimensions.first) + .setHeight(requestedEncoderDimensions.second) .setRotationDegrees(0) .setSampleMimeType( transformationRequest.videoMimeType != null @@ -152,21 +92,23 @@ import org.checkerframework.dataflow.qual.Pure; fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( transformationRequest, - /* hasOutputFormatRotation= */ swapEncodingDimensions, + /* hasOutputFormatRotation= */ outputRotationDegrees == 0, requestedEncoderFormat, encoderSupportedFormat)); if (transformationRequest.enableHdrEditing || inputFormat.height != encoderSupportedFormat.height || inputFormat.width != encoderSupportedFormat.width - || !transformationMatrix.isIdentity()) { + || scaleToFitFrameProcessor.shouldProcess()) { frameEditor = FrameEditor.create( context, - encoderSupportedFormat.width, - encoderSupportedFormat.height, + /* inputWidth= */ decodedWidth, + /* inputHeight= */ decodedHeight, + /* outputWidth= */ encoderSupportedFormat.width, + /* outputHeight= */ encoderSupportedFormat.height, inputFormat.pixelWidthHeightRatio, - new TransformationFrameProcessor(context, transformationMatrix), + scaleToFitFrameProcessor, /* outputSurface= */ encoder.getInputSurface(), transformationRequest.enableHdrEditing, debugViewProvider); diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorTest.java new file mode 100644 index 0000000000..77bb7a9cf6 --- /dev/null +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorTest.java @@ -0,0 +1,66 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Matrix; +import android.util.Pair; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Unit tests for {@link AdvancedFrameProcessor}. + * + *

See {@link AdvancedFrameProcessorPixelTest} for pixel tests testing {@link + * AdvancedFrameProcessor} given a transformation matrix. + */ +@RunWith(AndroidJUnit4.class) +public final class AdvancedFrameProcessorTest { + @Test + public void getOutputDimensions_withIdentityMatrix_leavesDimensionsUnchanged() { + Matrix identityMatrix = new Matrix(); + int inputWidth = 200; + int inputHeight = 150; + AdvancedFrameProcessor advancedFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), identityMatrix); + + Pair outputDimensions = + advancedFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + + assertThat(outputDimensions.first).isEqualTo(inputWidth); + assertThat(outputDimensions.second).isEqualTo(inputHeight); + } + + @Test + public void getOutputDimensions_withTransformationMatrix_leavesDimensionsUnchanged() { + Matrix transformationMatrix = new Matrix(); + transformationMatrix.postRotate(/* degrees= */ 90); + transformationMatrix.postScale(/* sx= */ .5f, /* sy= */ 1.2f); + int inputWidth = 200; + int inputHeight = 150; + AdvancedFrameProcessor advancedFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), transformationMatrix); + + Pair outputDimensions = + advancedFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + + assertThat(outputDimensions.first).isEqualTo(inputWidth); + assertThat(outputDimensions.second).isEqualTo(inputHeight); + } +} diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessorTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessorTest.java new file mode 100644 index 0000000000..183a275005 --- /dev/null +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessorTest.java @@ -0,0 +1,167 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.graphics.Matrix; +import android.util.Pair; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Unit tests for {@link ScaleToFitFrameProcessor}. + * + *

See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link + * AdvancedFrameProcessor} given a transformation matrix. + */ +@RunWith(AndroidJUnit4.class) +public final class ScaleToFitFrameProcessorTest { + + @Test + public void configureOutputDimensions_noEdits_producesExpectedOutput() { + Matrix identityMatrix = new Matrix(); + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET); + + Pair outputDimensions = + scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + + assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + assertThat(scaleToFitFrameProcessor.shouldProcess()).isFalse(); + assertThat(outputDimensions.first).isEqualTo(inputWidth); + assertThat(outputDimensions.second).isEqualTo(inputHeight); + } + + @Test + public void initializeBeforeConfigure_throwsIllegalStateException() { + Matrix identityMatrix = new Matrix(); + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET); + + // configureOutputDimensions not called before initialize. + assertThrows( + IllegalStateException.class, + () -> scaleToFitFrameProcessor.initialize(/* inputTexId= */ 0)); + } + + @Test + public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() { + Matrix identityMatrix = new Matrix(); + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET); + + // configureOutputDimensions not called before initialize. + assertThrows(IllegalStateException.class, scaleToFitFrameProcessor::getOutputRotationDegrees); + } + + @Test + public void configureOutputDimensions_scaleNarrow_producesExpectedOutput() { + Matrix scaleNarrowMatrix = new Matrix(); + scaleNarrowMatrix.postScale(/* sx= */ .5f, /* sy= */ 1.0f); + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor(getApplicationContext(), scaleNarrowMatrix, C.LENGTH_UNSET); + + Pair outputDimensions = + scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + + assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); + assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); + assertThat(outputDimensions.first).isEqualTo(inputHeight); + assertThat(outputDimensions.second).isEqualTo(Math.round(inputWidth * .5f)); + } + + @Test + public void configureOutputDimensions_scaleWide_producesExpectedOutput() { + Matrix scaleNarrowMatrix = new Matrix(); + scaleNarrowMatrix.postScale(/* sx= */ 2f, /* sy= */ 1.0f); + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor(getApplicationContext(), scaleNarrowMatrix, C.LENGTH_UNSET); + + Pair outputDimensions = + scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + + assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); + assertThat(outputDimensions.first).isEqualTo(inputWidth * 2); + assertThat(outputDimensions.second).isEqualTo(inputHeight); + } + + @Test + public void configureOutputDimensions_rotate90_producesExpectedOutput() { + Matrix rotate90Matrix = new Matrix(); + rotate90Matrix.postRotate(/* degrees= */ 90); + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor(getApplicationContext(), rotate90Matrix, C.LENGTH_UNSET); + + Pair outputDimensions = + scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + + assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); + assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); + assertThat(outputDimensions.first).isEqualTo(inputWidth); + assertThat(outputDimensions.second).isEqualTo(inputHeight); + } + + @Test + public void configureOutputDimensions_rotate45_producesExpectedOutput() { + Matrix rotate45Matrix = new Matrix(); + rotate45Matrix.postRotate(/* degrees= */ 45); + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor(getApplicationContext(), rotate45Matrix, C.LENGTH_UNSET); + long expectedOutputWidthHeight = 247; + + Pair outputDimensions = + scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + + assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); + assertThat(outputDimensions.first).isEqualTo(expectedOutputWidthHeight); + assertThat(outputDimensions.second).isEqualTo(expectedOutputWidthHeight); + } + + @Test + public void configureOutputDimensions_setResolution_producesExpectedOutput() { + Matrix identityMatrix = new Matrix(); + int inputWidth = 200; + int inputHeight = 150; + int requestedHeight = 300; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, requestedHeight); + + Pair outputDimensions = + scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight); + + assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); + assertThat(outputDimensions.first).isEqualTo(requestedHeight * inputWidth / inputHeight); + assertThat(outputDimensions.second).isEqualTo(requestedHeight); + } +} diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_request_output_height.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_request_output_height.png new file mode 100644 index 0000000000..a0c7903b9b Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_request_output_height.png differ diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png new file mode 100644 index 0000000000..b0cdf20aeb Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png differ