diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index 6aaaedbe39..3fb355bcc1 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -35,6 +35,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; +import java.util.List; import javax.microedition.khronos.egl.EGL10; /** OpenGL ES utilities. */ @@ -122,6 +123,20 @@ public final class GlUtil { }; } + /** Flattens the list of 4 element NDC coordinate vectors into a buffer. */ + public static float[] createVertexBuffer(List vertexList) { + float[] vertexBuffer = new float[HOMOGENEOUS_COORDINATE_VECTOR_SIZE * vertexList.size()]; + for (int i = 0; i < vertexList.size(); i++) { + System.arraycopy( + /* src= */ vertexList.get(i), + /* srcPos= */ 0, + /* dest= */ vertexBuffer, + /* destPos= */ HOMOGENEOUS_COORDINATE_VECTOR_SIZE * i, + /* length= */ HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + } + return vertexBuffer; + } + /** * Returns whether creating a GL context with {@value #EXTENSION_PROTECTED_CONTENT} is possible. * diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/rotate_then_translate.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/rotate_then_translate.png index 5cd7a9e989..a07e747547 100644 Binary files a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/rotate_then_translate.png and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/rotate_then_translate.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/translate_then_rotate.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/translate_then_rotate.png index a2efe9118c..a4f212bc59 100644 Binary files a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/translate_then_rotate.png and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/translate_then_rotate.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index cef6f29d1e..912d4d7f8b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -36,7 +36,9 @@ import android.util.Size; import androidx.annotation.Nullable; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; +import java.util.List; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; @@ -235,12 +237,38 @@ public final class FrameProcessorChainPixelTest { } @Test - public void processData_withFrameProcessingException_callsListener() throws Exception { - setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, ThrowingFrameProcessor::new); + public void + processData_withManyComposedMatrixTransformations_producesSameOutputAsCombinedTransformation() + throws Exception { + String testId = + "processData_withManyComposedMatrixTransformations_producesSameOutputAsCombinedTransformation"; + Presentation centerCrop = + new Presentation.Builder() + .setCrop(/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f) + .build(); + ImmutableList.Builder full10StepRotationAndCenterCrop = new ImmutableList.Builder<>(); + for (int i = 0; i < 10; i++) { + full10StepRotationAndCenterCrop.add(new Rotation(/* degrees= */ 36)); + } + full10StepRotationAndCenterCrop.add(centerCrop); - Thread.sleep(FRAME_PROCESSING_WAIT_MS); + setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, centerCrop); + Bitmap centerCropResultBitmap = processFirstFrameAndEnd(); + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, full10StepRotationAndCenterCrop.build()); + Bitmap fullRotationAndCenterCropResultBitmap = processFirstFrameAndEnd(); - assertThat(frameProcessingException.get()).isNotNull(); + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "centerCrop", centerCropResultBitmap); + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, + /* bitmapLabel= */ "full10StepRotationAndCenterCrop", + fullRotationAndCenterCropResultBitmap); + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + centerCropResultBitmap, fullRotationAndCenterCropResultBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } /** @@ -253,6 +281,11 @@ public final class FrameProcessorChainPixelTest { */ private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, GlEffect... effects) throws Exception { + setUpAndPrepareFirstFrame(pixelWidthHeightRatio, asList(effects)); + } + + private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, List effects) + throws Exception { // Set up the extractor to read the first video frame and get its format. MediaExtractor mediaExtractor = new MediaExtractor(); @Nullable MediaCodec mediaCodec = null; @@ -276,7 +309,7 @@ public final class FrameProcessorChainPixelTest { pixelWidthHeightRatio, inputWidth, inputHeight, - asList(effects), + effects, /* enableExperimentalHdrEditing= */ false); Size outputSize = frameProcessorChain.getOutputSize(); outputImageReader = @@ -349,26 +382,36 @@ public final class FrameProcessorChainPixelTest { return actualBitmap; } - private static class ThrowingFrameProcessor implements GlFrameProcessor { + /** + * Specifies a counter-clockwise rotation while accounting for the aspect ratio difference between + * the input frame in pixel coordinates and NDC. + * + *

Unlike {@link ScaleToFitTransformation}, this does not adjust the output size or scale to + * preserve input pixels. Pixels rotated out of the frame are clipped. + */ + private static final class Rotation implements MatrixTransformation { - private @MonotonicNonNull Size outputSize; + private final float degrees; + private @MonotonicNonNull Matrix adjustedTransformationMatrix; - @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) { - outputSize = new Size(inputWidth, inputHeight); + public Rotation(float degrees) { + this.degrees = degrees; } @Override - public Size getOutputSize() { - return checkStateNotNull(outputSize); + public Size configure(int inputWidth, int inputHeight) { + adjustedTransformationMatrix = new Matrix(); + adjustedTransformationMatrix.postRotate(degrees); + float inputAspectRatio = (float) inputWidth / inputHeight; + adjustedTransformationMatrix.preScale(/* sx= */ inputAspectRatio, /* sy= */ 1f); + adjustedTransformationMatrix.postScale(/* sx= */ 1f / inputAspectRatio, /* sy= */ 1f); + + return new Size(inputWidth, inputHeight); } @Override - public void drawFrame(long presentationTimeUs) throws FrameProcessingException { - throw new FrameProcessingException("An exception occurred."); + public Matrix getMatrix(long presentationTimeUs) { + return checkStateNotNull(adjustedTransformationMatrix); } - - @Override - public void release() {} } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index 6512b5bdb9..686278bcfc 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -201,24 +201,43 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ImmutableList.Builder frameProcessors = new ImmutableList.Builder().add(externalCopyFrameProcessor); + ImmutableList.Builder matrixTransformationListBuilder = + new ImmutableList.Builder<>(); // Scale to expand the frame to apply the pixelWidthHeightRatio. if (pixelWidthHeightRatio > 1f) { - frameProcessors.add( + matrixTransformationListBuilder.add( new ScaleToFitTransformation.Builder() .setScale(/* scaleX= */ pixelWidthHeightRatio, /* scaleY= */ 1f) - .build() - .toGlFrameProcessor()); + .build()); } else if (pixelWidthHeightRatio < 1f) { - frameProcessors.add( + matrixTransformationListBuilder.add( new ScaleToFitTransformation.Builder() .setScale(/* scaleX= */ 1f, /* scaleY= */ 1f / pixelWidthHeightRatio) - .build() - .toGlFrameProcessor()); + .build()); } + // Combine consecutive GlMatrixTransformations into a single GlFrameProcessor and convert + // all other GlEffects to GlFrameProcessors. for (int i = 0; i < effects.size(); i++) { - frameProcessors.add(effects.get(i).toGlFrameProcessor()); + GlEffect effect = effects.get(i); + if (effect instanceof GlMatrixTransformation) { + matrixTransformationListBuilder.add((GlMatrixTransformation) effect); + continue; + } + ImmutableList matrixTransformations = + matrixTransformationListBuilder.build(); + if (!matrixTransformations.isEmpty()) { + frameProcessors.add(new MatrixTransformationFrameProcessor(matrixTransformations)); + matrixTransformationListBuilder = new ImmutableList.Builder<>(); + } + frameProcessors.add(effect.toGlFrameProcessor()); } + ImmutableList matrixTransformations = + matrixTransformationListBuilder.build(); + if (!matrixTransformations.isEmpty()) { + frameProcessors.add(new MatrixTransformationFrameProcessor(matrixTransformations)); + } + return frameProcessors.build(); } @@ -256,6 +275,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final int[] framebuffers; private final Listener listener; + /** * Prevents further frame processing tasks from being scheduled or executed after {@link * #release()} is called or an exception occurred. @@ -415,7 +435,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; public void release() { stopProcessing.set(true); while (!futures.isEmpty()) { - checkNotNull(futures.poll()).cancel(/* mayInterruptIfRunning= */ true); + checkNotNull(futures.poll()).cancel(/* mayInterruptIfRunning= */ false); } futures.add( singleThreadExecutorService.submit(this::releaseFrameProcessorsAndDestroyGlContext)); @@ -490,6 +510,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; .setTextureTransformMatrix(textureTransformMatrix); for (int i = 0; i < frameProcessors.size() - 1; i++) { + if (stopProcessing.get()) { + return; + } + Size intermediateSize = frameProcessors.get(i).getOutputSize(); GlUtil.focusFramebuffer( eglDisplay, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java index cace85cbdc..850f30d06c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java @@ -21,23 +21,26 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.GLES20; +import android.opengl.Matrix; import android.util.Size; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; +import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.util.Arrays; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** - * Applies a transformation matrix in the vertex shader, and copies input pixels into an output - * frame based on their locations after applying this matrix. + * Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into + * an output frame based on their locations after applying the sequence of transformation matrices. * - *

Operations are done on normalized device coordinates (-1 to 1 on x and y axes). + *

Operations are done on normalized device coordinates (-1 to 1 on x, y, and z axes). + * Transformed vertices that are moved outside of this range after any of the transformation + * matrices are clipped to the NDC range. * *

The background color of the output frame will be black. */ -// TODO(b/227625423): Compose multiple transformation matrices in a single shader with clipping -// after each matrix. @UnstableApi @SuppressWarnings("FunctionalInterfaceClash") // b/228192298 /* package */ final class MatrixTransformationFrameProcessor implements GlFrameProcessor { @@ -49,8 +52,37 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final String VERTEX_SHADER_TRANSFORMATION_PATH = "shaders/vertex_shader_transformation_es2.glsl"; private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; + private static final ImmutableList NDC_SQUARE = + ImmutableList.of( + new float[] {-1, -1, 0, 1}, + new float[] {-1, 1, 0, 1}, + new float[] {1, 1, 0, 1}, + new float[] {1, -1, 0, 1}); - private final GlMatrixTransformation matrixTransformation; + /** The {@link MatrixTransformation MatrixTransformations} to apply. */ + private final ImmutableList matrixTransformations; + /** + * The transformation matrices provided by the {@link MatrixTransformation MatrixTransformations} + * for the most recent frame. + */ + private final float[][] transformationMatrixCache; + /** + * The product of the {@link #transformationMatrixCache} for the most recent frame, to be applied + * in the vertex shader. + */ + private final float[] compositeTransformationMatrix; + /** Matrix for storing an intermediate calculation result. */ + private final float[] tempResultMatrix; + + /** + * A polygon in the input space chosen such that no additional clipping is needed to keep vertices + * inside the NDC range when applying each of the {@link #matrixTransformations}. + * + *

This means that this polygon and {@link #compositeTransformationMatrix} can be used instead + * of applying each of the {@link #matrixTransformations} to {@link #NDC_SQUARE} in separate + * shaders. + */ + private ImmutableList visiblePolygon; private @MonotonicNonNull Size outputSize; private @MonotonicNonNull GlProgram glProgram; @@ -62,7 +94,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * matrix to use for each frame. */ public MatrixTransformationFrameProcessor(MatrixTransformation matrixTransformation) { - this.matrixTransformation = matrixTransformation; + this(ImmutableList.of(matrixTransformation)); } /** @@ -72,7 +104,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * matrix to use for each frame. */ public MatrixTransformationFrameProcessor(GlMatrixTransformation matrixTransformation) { - this.matrixTransformation = matrixTransformation; + this(ImmutableList.of(matrixTransformation)); + } + + /** + * Creates a new instance. + * + * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to + * apply to each frame in order. + */ + public MatrixTransformationFrameProcessor( + ImmutableList matrixTransformations) { + this.matrixTransformations = matrixTransformations; + + transformationMatrixCache = new float[matrixTransformations.size()][16]; + compositeTransformationMatrix = new float[16]; + tempResultMatrix = new float[16]; + Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); + visiblePolygon = NDC_SQUARE; } @Override @@ -81,14 +130,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkArgument(inputWidth > 0, "inputWidth must be positive"); checkArgument(inputHeight > 0, "inputHeight must be positive"); - outputSize = matrixTransformation.configure(inputWidth, inputHeight); + outputSize = new Size(inputWidth, inputHeight); + for (int i = 0; i < matrixTransformations.size(); i++) { + outputSize = + matrixTransformations.get(i).configure(outputSize.getWidth(), outputSize.getHeight()); + } + glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); - // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. - glProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); } @Override @@ -98,18 +147,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + updateCompositeTransformationMatrixAndVisiblePolygon(presentationTimeUs); + if (visiblePolygon.size() < 3) { + return; // Need at least three visible vertices for a triangle. + } + try { checkStateNotNull(glProgram).use(); - float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs); - checkState( - transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements"); - glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix); + glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrix); + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.createVertexBuffer(visiblePolygon), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GLES20.glDrawArrays( + GLES20.GL_TRIANGLE_FAN, /* first= */ 0, /* count= */ visiblePolygon.size()); GlUtil.checkGlError(); } catch (GlUtil.GlException e) { - throw new FrameProcessingException(e); + throw new FrameProcessingException(e, presentationTimeUs); } } @@ -119,4 +174,69 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; glProgram.delete(); } } + + /** + * Updates {@link #compositeTransformationMatrix} and {@link #visiblePolygon} based on the given + * frame timestamp. + */ + private void updateCompositeTransformationMatrixAndVisiblePolygon(long presentationTimeUs) { + if (!updateTransformationMatrixCache(presentationTimeUs)) { + return; + } + + // Compute the compositeTransformationMatrix and transform and clip the visiblePolygon for each + // MatrixTransformation's matrix. + Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); + visiblePolygon = NDC_SQUARE; + for (float[] transformationMatrix : transformationMatrixCache) { + Matrix.multiplyMM( + tempResultMatrix, + /* resultOffset= */ 0, + transformationMatrix, + /* lhsOffset= */ 0, + compositeTransformationMatrix, + /* rhsOffset= */ 0); + System.arraycopy( + /* src= */ tempResultMatrix, + /* srcPos= */ 0, + /* dest= */ compositeTransformationMatrix, + /* destPost= */ 0, + /* length= */ tempResultMatrix.length); + visiblePolygon = + MatrixUtils.clipConvexPolygonToNdcRange( + MatrixUtils.transformPoints(transformationMatrix, visiblePolygon)); + if (visiblePolygon.size() < 3) { + // Can ignore remaining matrices as there are not enough vertices left to form a polygon. + return; + } + } + // Calculate the input frame vertices corresponding to the output frame's visible polygon. + Matrix.invertM( + tempResultMatrix, /* mInvOffset= */ 0, compositeTransformationMatrix, /* mOffset= */ 0); + visiblePolygon = MatrixUtils.transformPoints(tempResultMatrix, visiblePolygon); + } + + /** + * Updates {@link #transformationMatrixCache} with the transformation matrices provided by the + * {@link #matrixTransformations} for the given frame timestamp and returns whether any matrix in + * {@link #transformationMatrixCache} changed. + */ + private boolean updateTransformationMatrixCache(long presentationTimeUs) { + boolean matrixChanged = false; + for (int i = 0; i < matrixTransformations.size(); i++) { + float[] cachedMatrix = transformationMatrixCache[i]; + float[] matrix = matrixTransformations.get(i).getGlMatrixArray(presentationTimeUs); + if (!Arrays.equals(cachedMatrix, matrix)) { + checkState(matrix.length == 16, "A 4x4 transformation matrix must have 16 elements"); + System.arraycopy( + /* src= */ matrix, + /* srcPos= */ 0, + /* dest= */ cachedMatrix, + /* destPost= */ 0, + /* length= */ matrix.length); + matrixChanged = true; + } + } + return matrixChanged; + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixUtils.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixUtils.java index 41775c1668..206d7cf16d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixUtils.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixUtils.java @@ -15,10 +15,31 @@ */ package androidx.media3.transformer; -/** Utility functions for working with matrices. */ -/* package */ class MatrixUtils { +import static androidx.media3.common.util.Assertions.checkArgument; + +import android.opengl.Matrix; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; + +/** Utility functions for working with matrices, vertices, and polygons. */ +/* package */ final class MatrixUtils { /** - * Returns a 4x4, column-major {@link android.opengl.Matrix} float array, from an input {@link + * Contains the normal vectors of the clipping planes in homogeneous coordinates which + * conveniently also double as origin vectors and parameters of the normal form of the planes ax + + * by + cz = d. + */ + private static final float[][] NDC_CUBE = + new float[][] { + new float[] {1, 0, 0, 1}, + new float[] {-1, 0, 0, 1}, + new float[] {0, 1, 0, 1}, + new float[] {0, -1, 0, 1}, + new float[] {0, 0, 1, 1}, + new float[] {0, 0, -1, 1} + }; + + /** + * Returns a 4x4, column-major {@link Matrix} float array, from an input {@link * android.graphics.Matrix}. * *

This is useful for converting to the 4x4 column-major format commonly used in OpenGL. @@ -30,7 +51,7 @@ package androidx.media3.transformer; // Transpose from row-major to column-major representations. float[] transposedMatrix4x4Array = new float[16]; - android.opengl.Matrix.transposeM( + Matrix.transposeM( transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0); return transposedMatrix4x4Array; @@ -59,6 +80,143 @@ package androidx.media3.transformer; return matrix4x4Array; } + /** + * Clips a convex polygon to normalized device coordinates (-1 to 1 on x, y, and z axes). + * + *

The input and output vertices are given in homogeneous coordinates (x,y,z,1) where the last + * element must always be 1. To convert a general vector in homogeneous coordinates (xw,yw,zw,w) + * to this form, simply divide all elements by w. + * + * @param polygonVertices The vertices in counter-clockwise order as 4 element vectors of + * homogeneous coordinates. + * @return The vertices of the clipped polygon, in counter-clockwise order, or an empty list if + * the polygon doesn't intersect with the NDC range. + */ + public static ImmutableList clipConvexPolygonToNdcRange( + ImmutableList polygonVertices) { + checkArgument(polygonVertices.size() >= 3, "A polygon must have at least 3 vertices."); + + // This is a 3D generalization of the Sutherland-Hodgman algorithm + // https://en.wikipedia.org/wiki/Sutherland%E2%80%93Hodgman_algorithm + // using a convex clipping volume (the NDC cube) instead of a convex clipping polygon to clip a + // given subject polygon. + // For this algorithm, the subject polygon doesn't necessarily need to be convex. But since we + // require that it is convex, we can assume that the clipped result is a single connected + // convex polygon. + ImmutableList.Builder outputVertices = + new ImmutableList.Builder().addAll(polygonVertices); + for (float[] clippingPlane : NDC_CUBE) { + ImmutableList inputVertices = outputVertices.build(); + outputVertices = new ImmutableList.Builder<>(); + + for (int i = 0; i < inputVertices.size(); i++) { + float[] currentVertex = inputVertices.get(i); + float[] previousVertex = + inputVertices.get((inputVertices.size() + i - 1) % inputVertices.size()); + if (isInsideClippingHalfSpace(currentVertex, clippingPlane)) { + if (!isInsideClippingHalfSpace(previousVertex, clippingPlane)) { + float[] intersectionPoint = + computeIntersectionPoint( + clippingPlane, clippingPlane, previousVertex, currentVertex); + if (!Arrays.equals(currentVertex, intersectionPoint)) { + outputVertices.add(intersectionPoint); + } + } + outputVertices.add(currentVertex); + } else if (isInsideClippingHalfSpace(previousVertex, clippingPlane)) { + float[] intersection = + computeIntersectionPoint(clippingPlane, clippingPlane, previousVertex, currentVertex); + if (!Arrays.equals(previousVertex, intersection)) { + outputVertices.add(intersection); + } + } + } + } + + return outputVertices.build(); + } + + /** + * Returns whether the given point is inside the half-space bounded by the clipping plane and + * facing away from its normal vector. + * + *

The clipping plane has the form ax + by + cz = d. + * + * @param point A point in homogeneous coordinates (x,y,z,1). + * @param clippingPlane The parameters (a,b,c,d) of the plane's normal form. + * @return Whether the point is on the inside of the plane. + */ + private static boolean isInsideClippingHalfSpace(float[] point, float[] clippingPlane) { + checkArgument(clippingPlane.length == 4, "Expecting 4 plane parameters"); + + return clippingPlane[0] * point[0] + clippingPlane[1] * point[1] + clippingPlane[2] * point[2] + <= clippingPlane[3]; + } + + /** + * Returns the intersection point of the given line and plane. + * + *

This method may only be called if such an intersection exists. + * + *

The plane has the form ax + by + cz = d. + * + *

The points are given in homogeneous coordinates (x,y,z,1). + * + * @param planePoint A point on the plane. + * @param planeParameters The parameters of the plane's normal form. + * @param linePoint1 A point on the line. + * @param linePoint2 Another point on the line. + * @return The point of intersection. + */ + private static float[] computeIntersectionPoint( + float[] planePoint, float[] planeParameters, float[] linePoint1, float[] linePoint2) { + checkArgument(planeParameters.length == 4, "Expecting 4 plane parameters"); + + // See https://en.wikipedia.org/wiki/Line%E2%80%93plane_intersection#Algebraic_form for the + // derivation of this solution formula. + float lineEquationParameter = + ((planePoint[0] - linePoint1[0]) * planeParameters[0] + + (planePoint[1] - linePoint1[1]) * planeParameters[1] + + (planePoint[2] - linePoint1[2]) * planeParameters[2]) + / ((linePoint2[0] - linePoint1[0]) * planeParameters[0] + + (linePoint2[1] - linePoint1[1]) * planeParameters[1] + + (linePoint2[2] - linePoint1[2]) * planeParameters[2]); + float x = linePoint1[0] + (linePoint2[0] - linePoint1[0]) * lineEquationParameter; + float y = linePoint1[1] + (linePoint2[1] - linePoint1[1]) * lineEquationParameter; + float z = linePoint1[2] + (linePoint2[2] - linePoint1[2]) * lineEquationParameter; + return new float[] {x, y, z, 1}; + } + + /** + * Applies a transformation matrix to each point. + * + * @param transformationMatrix The 4x4 transformation matrix. + * @param points The points as 4 element vectors of homogeneous coordinates (x,y,z,1). + * @return The transformed points as 4 element vectors of homogeneous coordinates (x,y,z,1). + */ + public static ImmutableList transformPoints( + float[] transformationMatrix, ImmutableList points) { + ImmutableList.Builder transformedPoints = new ImmutableList.Builder<>(); + for (int i = 0; i < points.size(); i++) { + float[] transformedPoint = new float[4]; + Matrix.multiplyMV( + transformedPoint, + /* resultVecOffset= */ 0, + transformationMatrix, + /* lhsMatOffset= */ 0, + points.get(i), + /* rhsVecOffset= */ 0); + // Multiplication result is in homogeneous coordinates (xw,yw,zw,w) with any w. Divide by w + // to get (x,y,z,1). + transformedPoint[0] /= transformedPoint[3]; + transformedPoint[1] /= transformedPoint[3]; + transformedPoint[2] /= transformedPoint[3]; + transformedPoint[3] = 1; + transformedPoints.add(transformedPoint); + } + return transformedPoints.build(); + } + /** Class only contains static methods. */ private MatrixUtils() {} } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/MatrixUtilsTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/MatrixUtilsTest.java new file mode 100644 index 0000000000..8c80ef6103 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/MatrixUtilsTest.java @@ -0,0 +1,177 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.opengl.Matrix; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Tests for {@link MatrixUtils}. */ +@RunWith(AndroidJUnit4.class) +public class MatrixUtilsTest { + + @Test + public void clipConvexPolygonToNdcRange_withTwoVertices_throwsException() { + ImmutableList vertices = + ImmutableList.of(new float[] {1, 0, 1, 1}, new float[] {-0.5f, 0, 1, 1}); + + assertThrows( + IllegalArgumentException.class, () -> MatrixUtils.clipConvexPolygonToNdcRange(vertices)); + } + + @Test + public void clipConvexPolygonToNdcRange_insideRange_returnsPolygon() { + ImmutableList vertices = + ImmutableList.of( + new float[] {-0.5f, 0, 0, 1}, new float[] {0.5f, 0, 0, 1}, new float[] {0, 0.5f, 0, 1}); + + ImmutableList clippedVertices = MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + assertThat(clippedVertices).isEqualTo(vertices); + } + + @Test + public void clipConvexPolygonToNdcRange_onXClippingPlane_returnsPolygon() { + ImmutableList vertices = + ImmutableList.of( + new float[] {1, -0.5f, 0, 1}, new float[] {1, 0.5f, 0, 1}, new float[] {1, 0, 0.5f, 1}); + + ImmutableList clippedVertices = MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + assertThat(clippedVertices).isEqualTo(vertices); + } + + @Test + public void clipConvexPolygonToNdcRange_onYClippingPlane_returnsPolygon() { + ImmutableList vertices = + ImmutableList.of( + new float[] {0, 1, -0.5f, 1}, new float[] {0, 1, 0.5f, 1}, new float[] {0.5f, 1, 0, 1}); + + ImmutableList clippedVertices = MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + assertThat(clippedVertices).isEqualTo(vertices); + } + + @Test + public void clipConvexPolygonToNdcRange_onZClippingPlane_returnsPolygon() { + ImmutableList vertices = + ImmutableList.of( + new float[] {-0.5f, 0, 1, 1}, new float[] {0.5f, 0, 1, 1}, new float[] {0, 0.5f, 1, 1}); + + ImmutableList clippedVertices = MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + assertThat(clippedVertices).isEqualTo(vertices); + } + + @Test + public void clipConvexPolygonToNdcRange_onClippingVolumeCorners_returnsPolygon() { + ImmutableList vertices = + ImmutableList.of( + new float[] {-1, 0, 1, 1}, new float[] {1, 0, 1, 1}, new float[] {0, 1, 1, 1}); + + ImmutableList clippedVertices = MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + assertThat(clippedVertices).isEqualTo(vertices); + } + + @Test + public void clipConvexPolygonToNdcRange_outsideRange_returnsEmptyList() { + ImmutableList vertices = + ImmutableList.of( + new float[] {-0.5f, 0, 2, 1}, new float[] {0.5f, 0, 2, 1}, new float[] {0, 0.5f, 2, 1}); + + ImmutableList clippedVertices = MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + assertThat(clippedVertices).isEmpty(); + } + + @Test + public void clipConvexPolygonToNdcRange_withOneVertexOutsideRange_returnsClippedPolygon() { + ImmutableList vertices = + ImmutableList.of( + new float[] {-1, 0, 1, 1}, new float[] {1, 0, 1, 1}, new float[] {1, 2, 1, 1}); + + ImmutableList actualClippedVertices = + MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + ImmutableList expectedClippedVertices = + ImmutableList.of( + new float[] {0, 1, 1, 1}, + new float[] {-1, 0, 1, 1}, + new float[] {1, 0, 1, 1}, + new float[] {1, 1, 1, 1}); + assertThat(actualClippedVertices.toArray()).isEqualTo(expectedClippedVertices.toArray()); + } + + @Test + public void clipConvexPolygonToNdcRange_withTwoVerticesOutsideRange_returnsClippedPolygon() { + ImmutableList vertices = + ImmutableList.of( + new float[] {0, 1, 1, 1}, new float[] {-2, -3, 1, 1}, new float[] {2, -3, 1, 1}); + + ImmutableList actualClippedVertices = + MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + ImmutableList expectedClippedVertices = + ImmutableList.of( + new float[] {1, -1, 1, 1}, new float[] {0, 1, 1, 1}, new float[] {-1, -1, 1, 1}); + assertThat(actualClippedVertices.toArray()).isEqualTo(expectedClippedVertices.toArray()); + } + + @Test + public void clipConvexPolygonToNdcRange_enclosingRange_returnsRange() { + ImmutableList vertices = + ImmutableList.of( + new float[] {-2, -2, 1, 1}, + new float[] {2, -2, 1, 1}, + new float[] {2, 2, 1, 1}, + new float[] {-2, 2, 1, 1}); + + ImmutableList actualClippedVertices = + MatrixUtils.clipConvexPolygonToNdcRange(vertices); + + ImmutableList expectedClippedVertices = + ImmutableList.of( + new float[] {-1, 1, 1, 1}, + new float[] {-1, -1, 1, 1}, + new float[] {1, -1, 1, 1}, + new float[] {1, 1, 1, 1}); + assertThat(actualClippedVertices.toArray()).isEqualTo(expectedClippedVertices.toArray()); + } + + @Test + public void transformPoints_returnsExpectedResult() { + ImmutableList points = + ImmutableList.of( + new float[] {-1, 0, 1, 1}, new float[] {1, 0, 1, 1}, new float[] {0, 1, 1, 1}); + float[] scaleMatrix = new float[16]; + Matrix.setIdentityM(scaleMatrix, /* smOffset= */ 0); + Matrix.scaleM(scaleMatrix, /* mOffset= */ 0, /* x= */ 2, /* y= */ 3, /* z= */ 4); + + ImmutableList actualTransformedPoints = + MatrixUtils.transformPoints(scaleMatrix, points); + + ImmutableList expectedTransformedPoints = + ImmutableList.of( + new float[] {-2, 0, 4, 1}, new float[] {2, 0, 4, 1}, new float[] {0, 3, 4, 1}); + assertThat(actualTransformedPoints.toArray()).isEqualTo(expectedTransformedPoints.toArray()); + } +}