From 0bba5c6329e69b7fe582a664908b5bd6dfa4cb99 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Tue, 3 May 2022 18:58:59 +0100 Subject: [PATCH] Separate matrix effect specification and implementation. This change splits AdvancedFrameProcessor into 4 files: - MatrixTransformationFrameProcessor for the GlFrameProcessor implementation - MatrixTransformation and GlMatrixTransformation for the GlEffect specification - MatrixUtils for the static matrix helpers PiperOrigin-RevId: 446236384 --- ....java => MatrixTransformationFactory.java} | 35 ++- .../transformerdemo/TransformerActivity.java | 6 +- .../FrameProcessorChainPixelTest.java | 54 ++--- ...ransformationFrameProcessorPixelTest.java} | 42 ++-- .../transformer/AdvancedFrameProcessor.java | 204 ------------------ .../transformer/GlMatrixTransformation.java | 53 +++++ .../transformer/MatrixTransformation.java | 39 ++++ .../MatrixTransformationFrameProcessor.java | 119 ++++++++++ .../exoplayer2/transformer/MatrixUtils.java | 64 ++++++ .../PresentationFrameProcessor.java | 17 +- .../transformer/ScaleToFitFrameProcessor.java | 17 +- .../AdvancedFrameProcessorTest.java | 44 ---- 12 files changed, 365 insertions(+), 329 deletions(-) rename demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/{AdvancedFrameProcessorFactory.java => MatrixTransformationFactory.java} (67%) rename library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/{AdvancedFrameProcessorPixelTest.java => MatrixTransformationFrameProcessorPixelTest.java} (80%) delete mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessor.java create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlMatrixTransformation.java create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformation.java create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationFrameProcessor.java create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixUtils.java delete mode 100644 library/transformer/src/test/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorTest.java diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/AdvancedFrameProcessorFactory.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/MatrixTransformationFactory.java similarity index 67% rename from demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/AdvancedFrameProcessorFactory.java rename to demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/MatrixTransformationFactory.java index 7f3bacf9b7..93a993c812 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/AdvancedFrameProcessorFactory.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/MatrixTransformationFactory.java @@ -17,40 +17,39 @@ package com.google.android.exoplayer2.transformerdemo; import android.graphics.Matrix; import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.transformer.AdvancedFrameProcessor; -import com.google.android.exoplayer2.transformer.GlFrameProcessor; +import com.google.android.exoplayer2.transformer.GlMatrixTransformation; +import com.google.android.exoplayer2.transformer.MatrixTransformation; import com.google.android.exoplayer2.util.Util; /** - * Factory for {@link GlFrameProcessor GlFrameProcessors} that create video effects by applying - * transformation matrices to the individual video frames using {@link AdvancedFrameProcessor}. + * Factory for {@link GlMatrixTransformation GlMatrixTransformations} and {@link + * MatrixTransformation MatrixTransformations} that create video effects by applying transformation + * matrices to the individual video frames. */ -/* package */ final class AdvancedFrameProcessorFactory { +/* package */ final class MatrixTransformationFactory { /** - * Returns a {@link GlFrameProcessor} that rescales the frames over the first {@value + * Returns a {@link MatrixTransformation} that rescales the frames over the first {@value * #ZOOM_DURATION_SECONDS} seconds, such that the rectangle filled with the input frame increases * linearly in size from a single point to filling the full output frame. */ - public static GlFrameProcessor createZoomInTransitionFrameProcessor() { - return new AdvancedFrameProcessor( - /* matrixProvider= */ AdvancedFrameProcessorFactory::calculateZoomInTransitionMatrix); + public static MatrixTransformation createZoomInTransition() { + return MatrixTransformationFactory::calculateZoomInTransitionMatrix; } /** - * Returns a {@link GlFrameProcessor} that crops frames to a rectangle that moves on an ellipse. + * Returns a {@link MatrixTransformation} that crops frames to a rectangle that moves on an + * ellipse. */ - public static GlFrameProcessor createDizzyCropFrameProcessor() { - return new AdvancedFrameProcessor( - /* matrixProvider= */ AdvancedFrameProcessorFactory::calculateDizzyCropMatrix); + public static MatrixTransformation createDizzyCropEffect() { + return MatrixTransformationFactory::calculateDizzyCropMatrix; } /** - * Returns a {@link GlFrameProcessor} that rotates a frame in 3D around the y-axis and applies - * perspective projection to 2D. + * Returns a {@link GlMatrixTransformation} that rotates a frame in 3D around the y-axis and + * applies perspective projection to 2D. */ - public static GlFrameProcessor createSpin3dFrameProcessor() { - return new AdvancedFrameProcessor( - /* matrixProvider= */ AdvancedFrameProcessorFactory::calculate3dSpinMatrix); + public static GlMatrixTransformation createSpin3dEffect() { + return MatrixTransformationFactory::calculate3dSpinMatrix; } private static final float ZOOM_DURATION_SECONDS = 2f; diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java index ff2403492e..1c4ba3e145 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java @@ -246,7 +246,7 @@ public final class TransformerActivity extends AppCompatActivity { bundle.getBooleanArray(ConfigurationActivity.DEMO_EFFECTS_SELECTIONS); if (selectedEffects != null) { if (selectedEffects[0]) { - effects.add(AdvancedFrameProcessorFactory::createDizzyCropFrameProcessor); + effects.add(MatrixTransformationFactory.createDizzyCropEffect()); } if (selectedEffects[1]) { effects.add( @@ -261,13 +261,13 @@ public final class TransformerActivity extends AppCompatActivity { bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); } if (selectedEffects[2]) { - effects.add(AdvancedFrameProcessorFactory::createSpin3dFrameProcessor); + effects.add(MatrixTransformationFactory.createSpin3dEffect()); } if (selectedEffects[3]) { effects.add(BitmapOverlayFrameProcessor::new); } if (selectedEffects[4]) { - effects.add(AdvancedFrameProcessorFactory::createZoomInTransitionFrameProcessor); + effects.add(MatrixTransformationFactory.createZoomInTransition()); } transformerBuilder.setVideoFrameEffects(effects.build()); } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java index a632b44626..ab88a9b3d3 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java @@ -124,13 +124,14 @@ public final class FrameProcessorChainPixelTest { } @Test - public void processData_withAdvancedFrameProcessor_translateRight_producesExpectedOutput() + public void processData_withMatrixTransformation_translateRight_producesExpectedOutput() throws Exception { - String testId = "processData_withAdvancedFrameProcessor_translateRight"; + String testId = "processData_withMatrixTransformation_translateRight"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - GlFrameProcessor glFrameProcessor = new AdvancedFrameProcessor(translateRightMatrix); - setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor); + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, + (MatrixTransformation) (long presentationTimeNs) -> translateRightMatrix); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -145,19 +146,15 @@ public final class FrameProcessorChainPixelTest { } @Test - public void processData_withAdvancedAndScaleToFitFrameProcessors_producesExpectedOutput() + public void processData_withMatrixAndScaleToFitTransformation_producesExpectedOutput() throws Exception { - String testId = "processData_withAdvancedAndScaleToFitFrameProcessors"; + String testId = "processData_withMatrixAndScaleToFitTransformation"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - GlFrameProcessor translateRightFrameProcessor = - new AdvancedFrameProcessor(translateRightMatrix); - GlFrameProcessor rotate45FrameProcessor = - new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build(); setUpAndPrepareFirstFrame( DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, - () -> translateRightFrameProcessor, - () -> rotate45FrameProcessor); + (MatrixTransformation) (long presentationTimeUs) -> translateRightMatrix, + () -> new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -172,19 +169,15 @@ public final class FrameProcessorChainPixelTest { } @Test - public void processData_withScaleToFitAndAdvancedFrameProcessors_producesExpectedOutput() + public void processData_withScaleToFitAndMatrixTransformation_producesExpectedOutput() throws Exception { - String testId = "processData_withScaleToFitAndAdvancedFrameProcessors"; - GlFrameProcessor rotate45FrameProcessor = - new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build(); + String testId = "processData_withScaleToFitAndMatrixTransformation"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - GlFrameProcessor translateRightFrameProcessor = - new AdvancedFrameProcessor(translateRightMatrix); setUpAndPrepareFirstFrame( DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, - () -> rotate45FrameProcessor, - () -> translateRightFrameProcessor); + () -> new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build(), + (MatrixTransformation) (long presentationTimeUs) -> translateRightMatrix); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -199,12 +192,11 @@ public final class FrameProcessorChainPixelTest { } @Test - public void processData_withPresentationFrameProcessor_setResolution_producesExpectedOutput() - throws Exception { - String testId = "processData_withPresentationFrameProcessor_setResolution"; - GlFrameProcessor glFrameProcessor = - new PresentationFrameProcessor.Builder().setResolution(480).build(); - setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor); + public void processData_withPresentation_setResolution_producesExpectedOutput() throws Exception { + String testId = "processData_withPresentation_setResolution"; + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, + () -> new PresentationFrameProcessor.Builder().setResolution(480).build()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -219,12 +211,12 @@ public final class FrameProcessorChainPixelTest { } @Test - public void processData_withScaleToFitFrameProcessor_rotate45_producesExpectedOutput() + public void processData_withScaleToFitTransformation_rotate45_producesExpectedOutput() throws Exception { - String testId = "processData_withScaleToFitFrameProcessor_rotate45"; - GlFrameProcessor glFrameProcessor = - new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build(); - setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor); + String testId = "processData_withScaleToFitTransformation_rotate45"; + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, + () -> new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorPixelTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/MatrixTransformationFrameProcessorPixelTest.java similarity index 80% rename from library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorPixelTest.java rename to library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/MatrixTransformationFrameProcessorPixelTest.java index c2d44ba5ef..cfa593b60c 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorPixelTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/MatrixTransformationFrameProcessorPixelTest.java @@ -34,7 +34,7 @@ import org.junit.Test; import org.junit.runner.RunWith; /** - * Pixel test for frame processing via {@link AdvancedFrameProcessor}. + * Pixel test for frame processing via {@link MatrixTransformationFrameProcessor}. * *

Expected images are taken from an emulator, so tests on different emulators or physical * devices may fail. To test on other devices, please increase the {@link @@ -42,7 +42,7 @@ import org.junit.runner.RunWith; * as recommended in {@link FrameProcessorChainPixelTest}. */ @RunWith(AndroidJUnit4.class) -public final class AdvancedFrameProcessorPixelTest { +public final class MatrixTransformationFrameProcessorPixelTest { public static final String ORIGINAL_PNG_ASSET_PATH = "media/bitmap/sample_mp4_first_frame/original.png"; public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH = @@ -58,7 +58,7 @@ public final class AdvancedFrameProcessorPixelTest { private final EGLDisplay eglDisplay = GlUtil.createEglDisplay(); private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay); - private @MonotonicNonNull GlFrameProcessor advancedFrameProcessor; + private @MonotonicNonNull GlFrameProcessor matrixTransformationFrameProcessor; private int inputTexId; private int outputTexId; private int width; @@ -80,8 +80,8 @@ public final class AdvancedFrameProcessorPixelTest { @After public void release() { - if (advancedFrameProcessor != null) { - advancedFrameProcessor.release(); + if (matrixTransformationFrameProcessor != null) { + matrixTransformationFrameProcessor.release(); } GlUtil.destroyEglContext(eglDisplay, eglContext); } @@ -90,11 +90,13 @@ public final class AdvancedFrameProcessorPixelTest { public void drawFrame_noEdits_producesExpectedOutput() throws Exception { String testId = "drawFrame_noEdits"; Matrix identityMatrix = new Matrix(); - advancedFrameProcessor = new AdvancedFrameProcessor(identityMatrix); - advancedFrameProcessor.initialize(getApplicationContext(), inputTexId, width, height); + matrixTransformationFrameProcessor = + new MatrixTransformationFrameProcessor((long presentationTimeUs) -> identityMatrix); + matrixTransformationFrameProcessor.initialize( + getApplicationContext(), inputTexId, width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); - advancedFrameProcessor.drawFrame(/* presentationTimeUs= */ 0); + matrixTransformationFrameProcessor.drawFrame(/* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -112,11 +114,13 @@ public final class AdvancedFrameProcessorPixelTest { String testId = "drawFrame_translateRight"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - advancedFrameProcessor = new AdvancedFrameProcessor(translateRightMatrix); - advancedFrameProcessor.initialize(getApplicationContext(), inputTexId, width, height); + matrixTransformationFrameProcessor = + new MatrixTransformationFrameProcessor((long presentationTimeUs) -> translateRightMatrix); + matrixTransformationFrameProcessor.initialize( + getApplicationContext(), inputTexId, width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); - advancedFrameProcessor.drawFrame(/* presentationTimeUs= */ 0); + matrixTransformationFrameProcessor.drawFrame(/* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -134,11 +138,13 @@ public final class AdvancedFrameProcessorPixelTest { String testId = "drawFrame_scaleNarrow"; Matrix scaleNarrowMatrix = new Matrix(); scaleNarrowMatrix.postScale(.5f, 1.2f); - advancedFrameProcessor = new AdvancedFrameProcessor(scaleNarrowMatrix); - advancedFrameProcessor.initialize(getApplicationContext(), inputTexId, width, height); + matrixTransformationFrameProcessor = + new MatrixTransformationFrameProcessor((long presentationTimeUs) -> scaleNarrowMatrix); + matrixTransformationFrameProcessor.initialize( + getApplicationContext(), inputTexId, width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH); - advancedFrameProcessor.drawFrame(/* presentationTimeUs= */ 0); + matrixTransformationFrameProcessor.drawFrame(/* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -156,11 +162,13 @@ public final class AdvancedFrameProcessorPixelTest { String testId = "drawFrame_rotate90"; Matrix rotate90Matrix = new Matrix(); rotate90Matrix.postRotate(/* degrees= */ 90); - advancedFrameProcessor = new AdvancedFrameProcessor(rotate90Matrix); - advancedFrameProcessor.initialize(getApplicationContext(), inputTexId, width, height); + matrixTransformationFrameProcessor = + new MatrixTransformationFrameProcessor((long presentationTimeUs) -> rotate90Matrix); + matrixTransformationFrameProcessor.initialize( + getApplicationContext(), inputTexId, width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH); - advancedFrameProcessor.drawFrame(/* presentationTimeUs= */ 0); + matrixTransformationFrameProcessor.drawFrame(/* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessor.java deleted file mode 100644 index c1a5d28489..0000000000 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessor.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.transformer; - -import static com.google.android.exoplayer2.util.Assertions.checkArgument; -import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; - -import android.content.Context; -import android.opengl.GLES20; -import android.util.Size; -import com.google.android.exoplayer2.util.GlProgram; -import com.google.android.exoplayer2.util.GlUtil; -import java.io.IOException; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -/** - * Applies a transformation matrix in the vertex shader, and copies input pixels into an output - * frame based on their locations after applying this matrix. - * - *

Operations are done on normalized device coordinates (-1 to 1 on x and y axes). No automatic - * adjustments (like done in {@link ScaleToFitFrameProcessor}) are applied on the transformation. - * Width and height are not modified. - * - *

The background color of the output frame will be black. - */ -@SuppressWarnings("FunctionalInterfaceClash") // b/228192298 -public final class AdvancedFrameProcessor implements GlFrameProcessor { - - static { - GlUtil.glAssertionsEnabled = true; - } - - /** Updates the transformation {@link android.opengl.Matrix} for each frame. */ - public interface GlMatrixProvider { - /** - * Updates the transformation {@link android.opengl.Matrix} to apply to the frame with the given - * timestamp in place. - */ - float[] getGlMatrixArray(long presentationTimeUs); - } - - /** Provides a {@link android.graphics.Matrix} for each frame. */ - public interface MatrixProvider extends GlMatrixProvider { - /** - * Returns the transformation {@link android.graphics.Matrix} to apply to the frame with the - * given timestamp. - */ - android.graphics.Matrix getMatrix(long presentationTimeUs); - - @Override - default float[] getGlMatrixArray(long presentationTimeUs) { - return AdvancedFrameProcessor.getGlMatrixArray(getMatrix(presentationTimeUs)); - } - } - - private static final String VERTEX_SHADER_TRANSFORMATION_PATH = - "shaders/vertex_shader_transformation_es2.glsl"; - private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; - - /** - * Returns a 4x4, column-major {@link android.opengl.Matrix} float array, from an input {@link - * android.graphics.Matrix}. - * - *

This is useful for converting to the 4x4 column-major format commonly used in OpenGL. - */ - private static float[] getGlMatrixArray(android.graphics.Matrix matrix) { - float[] matrix3x3Array = new float[9]; - matrix.getValues(matrix3x3Array); - float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array); - - // Transpose from row-major to column-major representations. - float[] transposedMatrix4x4Array = new float[16]; - android.opengl.Matrix.transposeM( - transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0); - - return transposedMatrix4x4Array; - } - - /** - * Returns a 4x4 matrix array containing the 3x3 matrix array's contents. - * - *

The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected - * to be in 3 dimensions. The output will have the third row/column's values be an identity - * matrix's values, so that vertex transformations using this matrix will not affect the z axis. - *
- * Input format: [a, b, c, d, e, f, g, h, i]
- * Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i] - */ - private static float[] getMatrix4x4Array(float[] matrix3x3Array) { - float[] matrix4x4Array = new float[16]; - matrix4x4Array[10] = 1; - for (int inputRow = 0; inputRow < 3; inputRow++) { - for (int inputColumn = 0; inputColumn < 3; inputColumn++) { - int outputRow = (inputRow == 2) ? 3 : inputRow; - int outputColumn = (inputColumn == 2) ? 3 : inputColumn; - matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn]; - } - } - return matrix4x4Array; - } - - private final GlMatrixProvider matrixProvider; - - private @MonotonicNonNull Size size; - private @MonotonicNonNull GlProgram glProgram; - - /** - * Creates a new instance. - * - * @param transformationMatrix The transformation {@link android.graphics.Matrix} to apply to each - * frame. Operations are done on normalized device coordinates (-1 to 1 on x and y), and no - * automatic adjustments are applied on the transformation matrix. - */ - public AdvancedFrameProcessor(android.graphics.Matrix transformationMatrix) { - this(getGlMatrixArray(transformationMatrix)); - } - - /** - * Creates a new instance. - * - * @param matrixProvider A {@link MatrixProvider} that provides the transformation matrix to apply - * to each frame. - */ - public AdvancedFrameProcessor(MatrixProvider matrixProvider) { - this.matrixProvider = matrixProvider; - } - - /** - * Creates a new instance. - * - * @param transformationMatrix The 4x4 transformation {@link android.opengl.Matrix} to apply to - * each frame. Operations are done on normalized device coordinates (-1 to 1 on x and y), and - * no automatic adjustments are applied on the transformation matrix. - */ - public AdvancedFrameProcessor(float[] transformationMatrix) { - this(/* matrixProvider= */ (long presentationTimeUs) -> transformationMatrix.clone()); - checkArgument( - transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements."); - } - - /** - * Creates a new instance. - * - * @param matrixProvider A {@link GlMatrixProvider} that updates the transformation matrix for - * each frame. - */ - public AdvancedFrameProcessor(GlMatrixProvider matrixProvider) { - this.matrixProvider = matrixProvider; - } - - @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) - throws IOException { - checkArgument(inputWidth > 0, "inputWidth must be positive"); - checkArgument(inputHeight > 0, "inputHeight must be positive"); - - size = new Size(inputWidth, inputHeight); - // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms - // expected in the code. - glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); - // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. - glProgram.setBufferAttribute( - "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); - glProgram.setBufferAttribute( - "aTexSamplingCoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); - } - - @Override - public Size getOutputSize() { - return checkStateNotNull(size); - } - - @Override - public void drawFrame(long presentationTimeUs) { - checkStateNotNull(glProgram).use(); - glProgram.setFloatsUniform( - "uTransformationMatrix", matrixProvider.getGlMatrixArray(presentationTimeUs)); - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - GlUtil.checkGlError(); - } - - @Override - public void release() { - if (glProgram != null) { - glProgram.delete(); - } - } -} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlMatrixTransformation.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlMatrixTransformation.java new file mode 100644 index 0000000000..07d2916681 --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlMatrixTransformation.java @@ -0,0 +1,53 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import android.opengl.Matrix; +import android.util.Size; + +/** + * Specifies a 4x4 transformation {@link Matrix} to apply in the vertex shader for each frame. + * + *

The matrix is applied to points given in normalized device coordinates (-1 to 1 on x, y, and z + * axes). Transformed pixels that are moved outside of the normal device coordinate range are + * clipped. + * + *

Output frame pixels outside of the transformed input frame will be black. + */ +public interface GlMatrixTransformation extends GlEffect { + /** + * Configures the input and output dimensions. + * + *

Must be called before {@link #getGlMatrixArray(long)}. + * + * @param inputWidth The input frame width, in pixels. + * @param inputHeight The input frame height, in pixels. + * @return The output frame {@link Size}, in pixels. + */ + default Size configure(int inputWidth, int inputHeight) { + return new Size(inputWidth, inputHeight); + } + + /** + * Returns the 4x4 transformation {@link Matrix} to apply to the frame with the given timestamp. + */ + float[] getGlMatrixArray(long presentationTimeUs); + + @Override + default GlFrameProcessor toGlFrameProcessor() { + return new MatrixTransformationFrameProcessor(this); + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformation.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformation.java new file mode 100644 index 0000000000..eb7ea0c5af --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformation.java @@ -0,0 +1,39 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import android.graphics.Matrix; + +/** + * Specifies a 3x3 transformation {@link Matrix} to apply in the vertex shader for each frame. + * + *

The matrix is applied to points given in normalized device coordinates (-1 to 1 on x and y + * axes). Transformed pixels that are moved outside of the normal device coordinate range are + * clipped. + * + *

Output frame pixels outside of the transformed input frame will be black. + */ +public interface MatrixTransformation extends GlMatrixTransformation { + /** + * Returns the 3x3 transformation {@link Matrix} to apply to the frame with the given timestamp. + */ + Matrix getMatrix(long presentationTimeUs); + + @Override + default float[] getGlMatrixArray(long presentationTimeUs) { + return MatrixUtils.getGlMatrixArray(getMatrix(presentationTimeUs)); + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationFrameProcessor.java new file mode 100644 index 0000000000..78caf6d65d --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationFrameProcessor.java @@ -0,0 +1,119 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.util.Assertions.checkArgument; +import static com.google.android.exoplayer2.util.Assertions.checkState; +import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.opengl.GLES20; +import android.util.Size; +import com.google.android.exoplayer2.util.GlProgram; +import com.google.android.exoplayer2.util.GlUtil; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Applies a transformation matrix in the vertex shader, and copies input pixels into an output + * frame based on their locations after applying this matrix. + * + *

Operations are done on normalized device coordinates (-1 to 1 on x and y axes). No automatic + * adjustments (like done in {@link ScaleToFitFrameProcessor}) are applied on the transformation. + * + *

The background color of the output frame will be black. + */ +// TODO(b/227625423): Compose multiple transformation matrices in a single shader with clipping +// after each matrix. +@SuppressWarnings("FunctionalInterfaceClash") // b/228192298 +public final class MatrixTransformationFrameProcessor implements GlFrameProcessor { + + static { + GlUtil.glAssertionsEnabled = true; + } + + private static final String VERTEX_SHADER_TRANSFORMATION_PATH = + "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; + + private final GlMatrixTransformation matrixTransformation; + + private @MonotonicNonNull Size outputSize; + private @MonotonicNonNull GlProgram glProgram; + + /** + * Creates a new instance. + * + * @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation + * matrix to use for each frame. + */ + public MatrixTransformationFrameProcessor(MatrixTransformation matrixTransformation) { + this.matrixTransformation = matrixTransformation; + } + + /** + * Creates a new instance. + * + * @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation + * matrix to use for each frame. + */ + public MatrixTransformationFrameProcessor(GlMatrixTransformation matrixTransformation) { + this.matrixTransformation = matrixTransformation; + } + + @Override + public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) + throws IOException { + checkArgument(inputWidth > 0, "inputWidth must be positive"); + checkArgument(inputHeight > 0, "inputHeight must be positive"); + + outputSize = matrixTransformation.configure(inputWidth, inputHeight); + // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms + // expected in the code. + glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.setBufferAttribute( + "aTexSamplingCoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + } + + @Override + public Size getOutputSize() { + return checkStateNotNull(outputSize); + } + + @Override + public void drawFrame(long presentationTimeUs) { + checkStateNotNull(glProgram).use(); + float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs); + checkState( + transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements"); + glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); + } + + @Override + public void release() { + if (glProgram != null) { + glProgram.delete(); + } + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixUtils.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixUtils.java new file mode 100644 index 0000000000..62bcbda1da --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixUtils.java @@ -0,0 +1,64 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +/** Utility functions for working with matrices. */ +/* package */ class MatrixUtils { + /** + * Returns a 4x4, column-major {@link android.opengl.Matrix} float array, from an input {@link + * android.graphics.Matrix}. + * + *

This is useful for converting to the 4x4 column-major format commonly used in OpenGL. + */ + public static float[] getGlMatrixArray(android.graphics.Matrix matrix) { + float[] matrix3x3Array = new float[9]; + matrix.getValues(matrix3x3Array); + float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array); + + // Transpose from row-major to column-major representations. + float[] transposedMatrix4x4Array = new float[16]; + android.opengl.Matrix.transposeM( + transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0); + + return transposedMatrix4x4Array; + } + + /** + * Returns a 4x4 matrix array containing the 3x3 matrix array's contents. + * + *

The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected + * to be in 3 dimensions. The output will have the third row/column's values be an identity + * matrix's values, so that vertex transformations using this matrix will not affect the z axis. + *
+ * Input format: [a, b, c, d, e, f, g, h, i]
+ * Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i] + */ + private static float[] getMatrix4x4Array(float[] matrix3x3Array) { + float[] matrix4x4Array = new float[16]; + matrix4x4Array[10] = 1; + for (int inputRow = 0; inputRow < 3; inputRow++) { + for (int inputColumn = 0; inputColumn < 3; inputColumn++) { + int outputRow = (inputRow == 2) ? 3 : inputRow; + int outputColumn = (inputColumn == 2) ? 3 : inputColumn; + matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn]; + } + } + return matrix4x4Array; + } + + /** Class only contains static methods. */ + private MatrixUtils() {} +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java index 44cb876fcd..7146f04b33 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java @@ -46,6 +46,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * *

The background color of the output frame will be black. */ +// TODO(b/227625423): Implement MatrixTransformation instead of wrapping +// MatrixTransformationFrameProcessor. public final class PresentationFrameProcessor implements GlFrameProcessor { /** * Strategies controlling the layout of input pixels in the output frame. @@ -235,7 +237,7 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { private float outputHeight; private @MonotonicNonNull Size outputSize; private @MonotonicNonNull Matrix transformationMatrix; - private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; + private @MonotonicNonNull MatrixTransformationFrameProcessor matrixTransformationFrameProcessor; /** Creates a new instance. */ private PresentationFrameProcessor( @@ -263,8 +265,11 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) throws IOException { configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); - advancedFrameProcessor = new AdvancedFrameProcessor(transformationMatrix); - advancedFrameProcessor.initialize(context, inputTexId, inputWidth, inputHeight); + matrixTransformationFrameProcessor = + new MatrixTransformationFrameProcessor( + /* matrixTransformation= */ (long presentationTimeUs) -> + checkStateNotNull(transformationMatrix)); + matrixTransformationFrameProcessor.initialize(context, inputTexId, inputWidth, inputHeight); } @Override @@ -277,13 +282,13 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { @Override public void drawFrame(long presentationTimeUs) { - checkStateNotNull(advancedFrameProcessor).drawFrame(presentationTimeUs); + checkStateNotNull(matrixTransformationFrameProcessor).drawFrame(presentationTimeUs); } @Override public void release() { - if (advancedFrameProcessor != null) { - advancedFrameProcessor.release(); + if (matrixTransformationFrameProcessor != null) { + matrixTransformationFrameProcessor.release(); } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java index 1924d5e942..d4dc441324 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java @@ -37,6 +37,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * *

The background color of the output frame will be black. */ +// TODO(b/227625423): Implement MatrixTransformation instead of wrapping +// MatrixTransformationFrameProcessor. public final class ScaleToFitFrameProcessor implements GlFrameProcessor { /** A builder for {@link ScaleToFitFrameProcessor} instances. */ @@ -93,7 +95,7 @@ public final class ScaleToFitFrameProcessor implements GlFrameProcessor { private final Matrix transformationMatrix; - private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; + private @MonotonicNonNull MatrixTransformationFrameProcessor matrixTransformationFrameProcessor; private @MonotonicNonNull Size outputSize; private @MonotonicNonNull Matrix adjustedTransformationMatrix; @@ -114,8 +116,11 @@ public final class ScaleToFitFrameProcessor implements GlFrameProcessor { public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) throws IOException { configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); - advancedFrameProcessor = new AdvancedFrameProcessor(adjustedTransformationMatrix); - advancedFrameProcessor.initialize(context, inputTexId, inputWidth, inputHeight); + matrixTransformationFrameProcessor = + new MatrixTransformationFrameProcessor( + /* matrixTransformation= */ (long presentationTimeUs) -> + checkStateNotNull(adjustedTransformationMatrix)); + matrixTransformationFrameProcessor.initialize(context, inputTexId, inputWidth, inputHeight); } @Override @@ -125,13 +130,13 @@ public final class ScaleToFitFrameProcessor implements GlFrameProcessor { @Override public void drawFrame(long presentationTimeUs) { - checkStateNotNull(advancedFrameProcessor).drawFrame(presentationTimeUs); + checkStateNotNull(matrixTransformationFrameProcessor).drawFrame(presentationTimeUs); } @Override public void release() { - if (advancedFrameProcessor != null) { - advancedFrameProcessor.release(); + if (matrixTransformationFrameProcessor != null) { + matrixTransformationFrameProcessor.release(); } } diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorTest.java deleted file mode 100644 index 65242651b8..0000000000 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.transformer; - -import static org.junit.Assert.assertThrows; - -import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - * Unit tests for {@link AdvancedFrameProcessor}. - * - *

See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link - * AdvancedFrameProcessor} given a transformation matrix. - */ -@RunWith(AndroidJUnit4.class) -public final class AdvancedFrameProcessorTest { - - @Test - public void construct_withInvalidMatrixSize_throwsException() { - assertThrows( - IllegalArgumentException.class, - () -> new AdvancedFrameProcessor(/* transformationMatrix= */ new float[4])); - } - - @Test - public void construct_withValidMatrixSize_completesSuccessfully() { - new AdvancedFrameProcessor(/* transformationMatrix= */ new float[16]); - } -}