diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java index 77068f83dc..06c05c70a9 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java @@ -24,6 +24,7 @@ import static java.lang.Math.max; import android.content.res.AssetFileDescriptor; import android.graphics.Bitmap; import android.graphics.BitmapFactory; +import android.graphics.Matrix; import android.graphics.PixelFormat; import android.media.Image; import android.media.ImageReader; @@ -84,9 +85,14 @@ public final class FrameEditorTest { int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); frameEditorOutputImageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); + Matrix identityMatrix = new Matrix(); frameEditor = FrameEditor.create( - getApplicationContext(), width, height, frameEditorOutputImageReader.getSurface()); + getApplicationContext(), + width, + height, + identityMatrix, + frameEditorOutputImageReader.getSurface()); // Queue the first video frame from the extractor. String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java new file mode 100644 index 0000000000..6fdcfe0d33 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer.mh; + +import static androidx.media3.transformer.mh.AndroidTestUtil.REMOTE_MP4_10_SECONDS_URI_STRING; +import static androidx.media3.transformer.mh.AndroidTestUtil.runTransformer; + +import android.content.Context; +import android.graphics.Matrix; +import androidx.media3.transformer.Transformer; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** {@link Transformer} instrumentation test for setting a transformation matrix. */ +@RunWith(AndroidJUnit4.class) +public class SetTransformationMatrixTransformationTest { + @Test + public void setTransformationMatrixTransform() throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + Matrix transformationMatrix = new Matrix(); + transformationMatrix.postTranslate(/* dx= */ .2f, /* dy= */ .1f); + Transformer transformer = + new Transformer.Builder(context).setTransformationMatrix(transformationMatrix).build(); + + runTransformer( + context, + /* testId= */ "setTransformationMatrixTransform", + transformer, + REMOTE_MP4_10_SECONDS_URI_STRING, + /* timeoutSeconds= */ 120); + } +} diff --git a/libraries/transformer/src/main/assets/shaders/copy_external_fragment_shader.glsl b/libraries/transformer/src/main/assets/shaders/fragment_shader.glsl similarity index 100% rename from libraries/transformer/src/main/assets/shaders/copy_external_fragment_shader.glsl rename to libraries/transformer/src/main/assets/shaders/fragment_shader.glsl diff --git a/libraries/transformer/src/main/assets/shaders/blit_vertex_shader.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader.glsl similarity index 87% rename from libraries/transformer/src/main/assets/shaders/blit_vertex_shader.glsl rename to libraries/transformer/src/main/assets/shaders/vertex_shader.glsl index 502a8c4493..993760ce29 100644 --- a/libraries/transformer/src/main/assets/shaders/blit_vertex_shader.glsl +++ b/libraries/transformer/src/main/assets/shaders/vertex_shader.glsl @@ -14,8 +14,9 @@ attribute vec4 a_position; attribute vec4 a_texcoord; uniform mat4 tex_transform; +uniform mat4 transformation_matrix; varying vec2 v_texcoord; void main() { gl_Position = a_position; - v_texcoord = (tex_transform * a_texcoord).xy; + v_texcoord = (transformation_matrix * tex_transform * a_texcoord).xy; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java index dc552d26f1..687e5850b9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; import android.content.Context; +import android.graphics.Matrix; import android.graphics.SurfaceTexture; import android.opengl.EGL14; import android.opengl.EGLContext; @@ -27,10 +28,7 @@ import android.view.Surface; import androidx.media3.common.util.GlUtil; import java.io.IOException; -/** - * FrameEditor applies changes to individual video frames. Changes include just resolution for now, - * but may later include brightness, cropping, rotation, etc. - */ +/** FrameEditor applies changes to individual video frames. */ /* package */ final class FrameEditor { static { @@ -43,11 +41,16 @@ import java.io.IOException; * @param context A {@link Context}. * @param outputWidth The output width in pixels. * @param outputHeight The output height in pixels. + * @param transformationMatrix The transformation matrix to apply to each frame. * @param outputSurface The {@link Surface}. * @return A configured {@code FrameEditor}. */ public static FrameEditor create( - Context context, int outputWidth, int outputHeight, Surface outputSurface) { + Context context, + int outputWidth, + int outputHeight, + Matrix transformationMatrix, + Surface outputSurface) { EGLDisplay eglDisplay = GlUtil.createEglDisplay(); EGLContext eglContext; try { @@ -58,15 +61,16 @@ import java.io.IOException; EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); int textureId = GlUtil.createExternalTexture(); - GlUtil.Program copyProgram; + GlUtil.Program glProgram; try { // TODO(internal b/205002913): check the loaded program is consistent with the attributes // and uniforms expected in the code. - copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); + glProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); } catch (IOException e) { throw new IllegalStateException(e); } - copyProgram.setBufferAttribute( + + glProgram.setBufferAttribute( "a_position", new float[] { -1.0f, -1.0f, 0.0f, 1.0f, @@ -75,7 +79,7 @@ import java.io.IOException; 1.0f, 1.0f, 0.0f, 1.0f, }, /* size= */ 4); - copyProgram.setBufferAttribute( + glProgram.setBufferAttribute( "a_texcoord", new float[] { 0.0f, 0.0f, 0.0f, 1.0f, @@ -84,14 +88,57 @@ import java.io.IOException; 1.0f, 1.0f, 0.0f, 1.0f, }, /* size= */ 4); - copyProgram.setSamplerTexIdUniform("tex_sampler", textureId, /* unit= */ 0); - return new FrameEditor(eglDisplay, eglContext, eglSurface, textureId, copyProgram); + glProgram.setSamplerTexIdUniform("tex_sampler", textureId, /* unit= */ 0); + + float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix); + glProgram.setFloatsUniform("transformation_matrix", transformationMatrixArray); + + return new FrameEditor(eglDisplay, eglContext, eglSurface, textureId, glProgram); + } + + /** + * Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful + * for converting to the 4x4 column-major format commonly used in OpenGL. + */ + private static final float[] getGlMatrixArray(Matrix matrix) { + float[] matrix3x3Array = new float[9]; + matrix.getValues(matrix3x3Array); + float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array); + + // Transpose from row-major to column-major representations. + float[] transposedMatrix4x4Array = new float[16]; + android.opengl.Matrix.transposeM( + transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0); + + return transposedMatrix4x4Array; + } + + /** + * Returns a 4x4 matrix array containing the 3x3 matrix array's contents. + * + *

The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected + * to be in 3 dimensions. The output will have the third row/column's values be an identity + * matrix's values, so that vertex transformations using this matrix will not affect the z axis. + *
+ * Input format: [a, b, c, d, e, f, g, h, i]
+ * Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i] + */ + private static final float[] getMatrix4x4Array(float[] matrix3x3Array) { + float[] matrix4x4Array = new float[16]; + matrix4x4Array[10] = 1; + for (int inputRow = 0; inputRow < 3; inputRow++) { + for (int inputColumn = 0; inputColumn < 3; inputColumn++) { + int outputRow = (inputRow == 2) ? 3 : inputRow; + int outputColumn = (inputColumn == 2) ? 3 : inputColumn; + matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn]; + } + } + return matrix4x4Array; } // Predefined shader values. - private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl"; - private static final String FRAGMENT_SHADER_FILE_PATH = - "shaders/copy_external_fragment_shader.glsl"; + private static final String VERTEX_SHADER_FILE_PATH = "shaders/vertex_shader.glsl"; + private static final String FRAGMENT_SHADER_FILE_PATH = "shaders/fragment_shader.glsl"; private final float[] textureTransformMatrix; private final EGLDisplay eglDisplay; @@ -101,7 +148,7 @@ import java.io.IOException; private final SurfaceTexture inputSurfaceTexture; private final Surface inputSurface; - private final GlUtil.Program copyProgram; + private final GlUtil.Program glProgram; private volatile boolean hasInputData; @@ -110,12 +157,12 @@ import java.io.IOException; EGLContext eglContext, EGLSurface eglSurface, int textureId, - GlUtil.Program copyProgram) { + GlUtil.Program glProgram) { this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.eglSurface = eglSurface; this.textureId = textureId; - this.copyProgram = copyProgram; + this.glProgram = glProgram; textureTransformMatrix = new float[16]; inputSurfaceTexture = new SurfaceTexture(textureId); inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true); @@ -135,12 +182,12 @@ import java.io.IOException; return hasInputData; } - /** Processes pending input data. */ + /** Processes pending input frame. */ public void processData() { inputSurfaceTexture.updateTexImage(); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); - copyProgram.setFloatsUniform("tex_transform", textureTransformMatrix); - copyProgram.bindAttributesAndUniforms(); + glProgram.setFloatsUniform("tex_transform", textureTransformMatrix); + glProgram.bindAttributesAndUniforms(); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); @@ -150,7 +197,7 @@ import java.io.IOException; /** Releases all resources. */ public void release() { - copyProgram.delete(); + glProgram.delete(); GlUtil.deleteTexture(textureId); GlUtil.destroyEglContext(eglDisplay, eglContext); inputSurfaceTexture.release(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java index 633bd44e81..206784e661 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformation.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; +import android.graphics.Matrix; import androidx.annotation.Nullable; /** A media transformation configuration. */ @@ -25,6 +26,7 @@ import androidx.annotation.Nullable; public final boolean removeVideo; public final boolean flattenForSlowMotion; public final int outputHeight; + public final Matrix transformationMatrix; public final String containerMimeType; @Nullable public final String audioMimeType; @Nullable public final String videoMimeType; @@ -34,6 +36,7 @@ import androidx.annotation.Nullable; boolean removeVideo, boolean flattenForSlowMotion, int outputHeight, + Matrix transformationMatrix, String containerMimeType, @Nullable String audioMimeType, @Nullable String videoMimeType) { @@ -41,6 +44,7 @@ import androidx.annotation.Nullable; this.removeVideo = removeVideo; this.flattenForSlowMotion = flattenForSlowMotion; this.outputHeight = outputHeight; + this.transformationMatrix = transformationMatrix; this.containerMimeType = containerMimeType; this.audioMimeType = audioMimeType; this.videoMimeType = videoMimeType; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index a51b93b64e..4ca78ca71a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -25,6 +25,7 @@ import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_MIN_BUFFER_MS import static java.lang.Math.min; import android.content.Context; +import android.graphics.Matrix; import android.media.MediaFormat; import android.media.MediaMuxer; import android.os.Handler; @@ -102,6 +103,7 @@ public final class Transformer { private boolean removeVideo; private boolean flattenForSlowMotion; private int outputHeight; + private Matrix transformationMatrix; private String containerMimeType; @Nullable private String audioMimeType; @Nullable private String videoMimeType; @@ -114,6 +116,7 @@ public final class Transformer { public Builder() { muxerFactory = new FrameworkMuxer.Factory(); outputHeight = Format.NO_VALUE; + transformationMatrix = new Matrix(); containerMimeType = MimeTypes.VIDEO_MP4; listener = new Listener() {}; looper = Util.getCurrentOrMainLooper(); @@ -129,6 +132,7 @@ public final class Transformer { this.context = context.getApplicationContext(); muxerFactory = new FrameworkMuxer.Factory(); outputHeight = Format.NO_VALUE; + transformationMatrix = new Matrix(); containerMimeType = MimeTypes.VIDEO_MP4; listener = new Listener() {}; looper = Util.getCurrentOrMainLooper(); @@ -144,6 +148,7 @@ public final class Transformer { this.removeVideo = transformer.transformation.removeVideo; this.flattenForSlowMotion = transformer.transformation.flattenForSlowMotion; this.outputHeight = transformer.transformation.outputHeight; + this.transformationMatrix = transformer.transformation.transformationMatrix; this.containerMimeType = transformer.transformation.containerMimeType; this.audioMimeType = transformer.transformation.audioMimeType; this.videoMimeType = transformer.transformation.videoMimeType; @@ -260,6 +265,26 @@ public final class Transformer { return this; } + /** + * Sets the transformation matrix. The default value is to apply no change. + * + *

This can be used to perform operations supported by {@link Matrix}, like scaling and + * rotating the video. + * + *

For now, resolution will not be affected by this method. + * + * @param transformationMatrix The transformation to apply to video frames. + * @return This builder. + */ + public Builder setTransformationMatrix(Matrix transformationMatrix) { + // TODO(Internal b/201293185): After {@link #setResolution} supports arbitrary resolutions, + // allow transformations to change the resolution, by scaling to the appropriate min/max + // values. This will also be required to create the VertexTransformation class, in order to + // have aspect ratio helper methods (which require resolution to change). + this.transformationMatrix = transformationMatrix; + return this; + } + /** * @deprecated This feature will be removed in a following release and the MIME type of the * output will always be MP4. @@ -411,6 +436,7 @@ public final class Transformer { removeVideo, flattenForSlowMotion, outputHeight, + transformationMatrix, containerMimeType, audioMimeType, videoMimeType); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index 60d79eda20..164d7d21ee 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -68,10 +68,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } Format inputFormat = checkNotNull(formatHolder.format); - if ((transformation.videoMimeType != null - && !transformation.videoMimeType.equals(inputFormat.sampleMimeType)) - || (transformation.outputHeight != Format.NO_VALUE - && transformation.outputHeight != inputFormat.height)) { + if (shouldTranscode(inputFormat)) { samplePipeline = new VideoSamplePipeline(context, inputFormat, transformation, getIndex()); } else { samplePipeline = new PassthroughSamplePipeline(inputFormat); @@ -82,6 +79,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return true; } + private boolean shouldTranscode(Format inputFormat) { + if (transformation.videoMimeType != null + && !transformation.videoMimeType.equals(inputFormat.sampleMimeType)) { + return true; + } + if (transformation.outputHeight != Format.NO_VALUE + && transformation.outputHeight != inputFormat.height) { + return true; + } + if (!transformation.transformationMatrix.isIdentity()) { + return true; + } + return false; + } + /** * Queues the input buffer to the sample pipeline unless it should be dropped because of slow * motion flattening. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index ac1f792dcd..fbcd9ab94c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -84,6 +84,7 @@ import java.io.IOException; context, outputWidth, outputHeight, + transformation.transformationMatrix, /* outputSurface= */ checkNotNull(encoder.getInputSurface())); try { decoder =