diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java index ee55e71477..8b95a7b0c6 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java @@ -116,8 +116,8 @@ public final class FrameEditorDataProcessingTest { public void processData_noEdits_producesExpectedOutput() throws Exception { Matrix identityMatrix = new Matrix(); setUpAndPrepareFirstFrame(identityMatrix); - Bitmap expectedBitmap = getBitmap(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING); + checkNotNull(frameEditor).processData(); Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); @@ -134,8 +134,8 @@ public final class FrameEditorDataProcessingTest { Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); setUpAndPrepareFirstFrame(translateRightMatrix); - Bitmap expectedBitmap = getBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); + checkNotNull(frameEditor).processData(); Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); @@ -174,8 +174,8 @@ public final class FrameEditorDataProcessingTest { Matrix rotate90Matrix = new Matrix(); rotate90Matrix.postRotate(/* degrees= */ 90); setUpAndPrepareFirstFrame(rotate90Matrix); - Bitmap expectedBitmap = getBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); + checkNotNull(frameEditor).processData(); Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); @@ -206,13 +206,14 @@ public final class FrameEditorDataProcessingTest { int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); frameEditorOutputImageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); + Context context = getApplicationContext(); frameEditor = FrameEditor.create( - getApplicationContext(), + context, width, height, PIXEL_WIDTH_HEIGHT_RATIO, - transformationMatrix, + new TransformationFrameProcessor(context, transformationMatrix), frameEditorOutputImageReader.getSurface(), /* enableExperimentalHdrEditing= */ false, Transformer.DebugViewProvider.NONE); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java index e92c045b66..a35b2985a4 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java @@ -28,7 +28,7 @@ import org.junit.Test; import org.junit.runner.RunWith; /** - * Test for {@link FrameEditor#create(Context, int, int, float, Matrix, Surface, boolean, + * Test for {@link FrameEditor#create(Context, int, int, float, GlFrameProcessor, Surface, boolean, * Transformer.DebugViewProvider) creating} a {@link FrameEditor}. */ @RunWith(AndroidJUnit4.class) @@ -39,12 +39,14 @@ public final class FrameEditorTest { @Test public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully() throws TransformationException { + Context context = getApplicationContext(); + FrameEditor.create( - getApplicationContext(), + context, /* outputWidth= */ 200, /* outputHeight= */ 100, /* pixelWidthHeightRatio= */ 1, - new Matrix(), + new TransformationFrameProcessor(context, new Matrix()), new Surface(new SurfaceTexture(false)), /* enableExperimentalHdrEditing= */ false, Transformer.DebugViewProvider.NONE); @@ -52,16 +54,18 @@ public final class FrameEditorTest { @Test public void create_withUnsupportedPixelWidthHeightRatio_throwsException() { + Context context = getApplicationContext(); + TransformationException exception = assertThrows( TransformationException.class, () -> FrameEditor.create( - getApplicationContext(), + context, /* outputWidth= */ 200, /* outputHeight= */ 100, /* pixelWidthHeightRatio= */ 2, - new Matrix(), + new TransformationFrameProcessor(context, new Matrix()), new Surface(new SurfaceTexture(false)), /* enableExperimentalHdrEditing= */ false, Transformer.DebugViewProvider.NONE)); diff --git a/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_es2.glsl b/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_es2.glsl new file mode 100644 index 0000000000..4d11dbba78 --- /dev/null +++ b/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_es2.glsl @@ -0,0 +1,24 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 fragment shader that samples from a (non-external) texture with uTexSampler, +// copying from this texture to the current output. + +precision mediump float; +uniform sampler2D uTexSampler; +varying vec2 vTexCoords; +void main() { + gl_FragColor = texture2D(uTexSampler, vTexCoords); +} diff --git a/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external.glsl b/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl similarity index 90% rename from libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external.glsl rename to libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl index 17667e636d..56f2477812 100644 --- a/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external.glsl +++ b/libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl @@ -1,3 +1,4 @@ +#version 100 // Copyright 2021 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Fragment shader that samples from an external texture with uTexSampler, +// ES 2 fragment shader that samples from an external texture with uTexSampler, // copying from this texture to the current output. #extension GL_OES_EGL_image_external : require diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl new file mode 100644 index 0000000000..7eb9cda916 --- /dev/null +++ b/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl @@ -0,0 +1,27 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 vertex shader that applies an external surface texture's 4 * 4 texture +// transformation matrix to convert the texture coordinates to the sampling +// locations. + +attribute vec4 aFramePosition; +attribute vec4 aTexCoords; +uniform mat4 uTexTransform; +varying vec2 vTexCoords; +void main() { + gl_Position = aFramePosition; + vTexCoords = (uTexTransform * aTexCoords).xy; +} diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl similarity index 79% rename from libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl rename to libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl index 56da553bad..f65570b2a9 100644 --- a/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl +++ b/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl @@ -13,15 +13,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -// ES 3 vertex shader that applies the 4 * 4 transformation matrix -// uTransformationMatrix. +// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture +// transformation matrix to convert the texture coordinates to the sampling +// locations. in vec4 aFramePosition; in vec4 aTexCoords; uniform mat4 uTexTransform; -uniform mat4 uTransformationMatrix; out vec2 vTexCoords; void main() { - gl_Position = uTransformationMatrix * aFramePosition; + gl_Position = aFramePosition; vTexCoords = (uTexTransform * aTexCoords).xy; } diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl similarity index 85% rename from libraries/transformer/src/main/assets/shaders/vertex_shader_transformation.glsl rename to libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl index 1268e9265c..a8320492ba 100644 --- a/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation.glsl +++ b/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl @@ -1,3 +1,4 @@ +#version 100 // Copyright 2021 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,15 +13,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Vertex shader that applies the 4 * 4 transformation matrix +// ES 2 vertex shader that applies the 4 * 4 transformation matrix // uTransformationMatrix. attribute vec4 aFramePosition; attribute vec4 aTexCoords; -uniform mat4 uTexTransform; uniform mat4 uTransformationMatrix; varying vec2 vTexCoords; void main() { gl_Position = uTransformationMatrix * aFramePosition; - vTexCoords = (uTexTransform * aTexCoords).xy; + vTexCoords = aTexCoords.xy; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java new file mode 100644 index 0000000000..4bce183cc6 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java @@ -0,0 +1,114 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.opengl.GLES20; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Copies frames from an external texture and applies color transformations for HDR if needed. */ +/* package */ class ExternalCopyFrameProcessor implements GlFrameProcessor { + + static { + GlUtil.glAssertionsEnabled = true; + } + + private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH = + "shaders/vertex_shader_tex_transform_es2.glsl"; + private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH = + "shaders/vertex_shader_tex_transform_es3.glsl"; + private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = + "shaders/fragment_shader_copy_external_es2.glsl"; + private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = + "shaders/fragment_shader_copy_external_yuv_es3.glsl"; + // Color transform coefficients from + // https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f. + private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = { + 1.168f, 1.168f, 1.168f, + 0.0f, -0.188f, 2.148f, + 1.683f, -0.652f, 0.0f, + }; + + private final Context context; + private final boolean enableExperimentalHdrEditing; + + private @MonotonicNonNull GlProgram glProgram; + + public ExternalCopyFrameProcessor(Context context, boolean enableExperimentalHdrEditing) { + this.context = context; + this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; + } + + @Override + public void initialize() throws IOException { + // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms + // expected in the code. + String vertexShaderFilePath = + enableExperimentalHdrEditing + ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH + : VERTEX_SHADER_TEX_TRANSFORM_PATH; + String fragmentShaderFilePath = + enableExperimentalHdrEditing + ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH + : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.setBufferAttribute( + "aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + if (enableExperimentalHdrEditing) { + // In HDR editing mode the decoder output is sampled in YUV. + glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); + } + } + + /** + * Sets the texture transform matrix for converting an external surface texture's coordinates to + * sampling locations. + * + * @param textureTransformMatrix The external surface texture's {@link + * android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}. + */ + public void setTextureTransformMatrix(float[] textureTransformMatrix) { + checkStateNotNull(glProgram); + glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix); + } + + @Override + public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) { + checkStateNotNull(glProgram); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0); + glProgram.use(); + glProgram.bindAttributesAndUniforms(); + GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } + + @Override + public void release() { + if (glProgram != null) { + glProgram.delete(); + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java index f3dc89c94d..10b5ccdd77 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java @@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; -import android.graphics.Matrix; import android.graphics.SurfaceTexture; import android.opengl.EGL14; import android.opengl.EGLContext; @@ -40,7 +39,7 @@ import java.util.concurrent.atomic.AtomicInteger; * *

Input becomes available on its {@link #getInputSurface() input surface} asynchronously so * {@link #canProcessData()} needs to be checked before calling {@link #processData()}. Output is - * written to its {@link #create(Context, int, int, float, Matrix, Surface, boolean, + * written to its {@link #create(Context, int, int, float, GlFrameProcessor, Surface, boolean, * Transformer.DebugViewProvider) output surface}. */ /* package */ final class FrameEditor { @@ -56,7 +55,7 @@ import java.util.concurrent.atomic.AtomicInteger; * @param outputWidth The output width in pixels. * @param outputHeight The output height in pixels. * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. - * @param transformationMatrix The transformation matrix to apply to each frame. + * @param transformationFrameProcessor The {@link GlFrameProcessor} to apply to each frame. * @param outputSurface The {@link Surface}. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @param debugViewProvider Provider for optional debug views to show intermediate output. @@ -65,12 +64,14 @@ import java.util.concurrent.atomic.AtomicInteger; * files fails, or an OpenGL error occurs while creating and configuring the OpenGL * components. */ + // TODO(b/214975934): Take a List as input and rename FrameEditor to + // FrameProcessorChain. public static FrameEditor create( Context context, int outputWidth, int outputHeight, float pixelWidthHeightRatio, - Matrix transformationMatrix, + GlFrameProcessor transformationFrameProcessor, Surface outputSurface, boolean enableExperimentalHdrEditing, Transformer.DebugViewProvider debugViewProvider) @@ -85,8 +86,8 @@ import java.util.concurrent.atomic.AtomicInteger; TransformationException.ERROR_CODE_GL_INIT_FAILED); } - GlFrameProcessor frameProcessor = - new GlFrameProcessor(context, transformationMatrix, enableExperimentalHdrEditing); + ExternalCopyFrameProcessor externalCopyFrameProcessor = + new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); @Nullable SurfaceView debugSurfaceView = debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight); @@ -94,7 +95,9 @@ import java.util.concurrent.atomic.AtomicInteger; EGLDisplay eglDisplay; EGLContext eglContext; EGLSurface eglSurface; - int textureId; + int inputExternalTexId; + int intermediateTexId; + int frameBuffer; @Nullable EGLSurface debugPreviewEglSurface = null; try { eglDisplay = GlUtil.createEglDisplay(); @@ -117,9 +120,12 @@ import java.util.concurrent.atomic.AtomicInteger; } GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); - textureId = GlUtil.createExternalTexture(); - frameProcessor.initialize(); - } catch (IOException | GlUtil.GlException e) { + inputExternalTexId = GlUtil.createExternalTexture(); + intermediateTexId = GlUtil.createTexture(outputWidth, outputHeight); + frameBuffer = GlUtil.createFboForTexture(intermediateTexId); + externalCopyFrameProcessor.initialize(); + transformationFrameProcessor.initialize(); + } catch (GlUtil.GlException | IOException e) { throw TransformationException.createForFrameEditor( e, TransformationException.ERROR_CODE_GL_INIT_FAILED); } @@ -138,8 +144,11 @@ import java.util.concurrent.atomic.AtomicInteger; eglDisplay, eglContext, eglSurface, - textureId, - frameProcessor, + inputExternalTexId, + intermediateTexId, + frameBuffer, + externalCopyFrameProcessor, + transformationFrameProcessor, outputWidth, outputHeight, debugPreviewEglSurface, @@ -147,12 +156,25 @@ import java.util.concurrent.atomic.AtomicInteger; debugPreviewHeight); } - private final GlFrameProcessor frameProcessor; + // TODO(b/214975934): Write javadoc for fields where the purpose might be unclear to someone less + // familiar with this class and consider grouping some of these fields into new classes to + // reduce the number of constructor parameters. + private final ExternalCopyFrameProcessor externalCopyFrameProcessor; + private final GlFrameProcessor transformationFrameProcessor; private final float[] textureTransformMatrix; private final EGLDisplay eglDisplay; private final EGLContext eglContext; private final EGLSurface eglSurface; - private final int textureId; + /** Indentifier of the external texture the {@code FrameEditor} reads its input from. */ + private final int inputExternalTexId; + /** + * Indentifier of the texture where the output of the {@link ExternalCopyFrameProcessor} is + * written to and the {@link TransformationFrameProcessor} reads its input from. + */ + private final int intermediateTexId; + /** Identifier of a framebuffer object associated with the intermediate texture. */ + private final int frameBuffer; + private final AtomicInteger pendingInputFrameCount; private final AtomicInteger availableInputFrameCount; private final SurfaceTexture inputSurfaceTexture; @@ -169,8 +191,11 @@ import java.util.concurrent.atomic.AtomicInteger; EGLDisplay eglDisplay, EGLContext eglContext, EGLSurface eglSurface, - int textureId, - GlFrameProcessor frameProcessor, + int inputExternalTexId, + int intermediateTexId, + int frameBuffer, + ExternalCopyFrameProcessor externalCopyFrameProcessor, + GlFrameProcessor transformationFrameProcessor, int outputWidth, int outputHeight, @Nullable EGLSurface debugPreviewEglSurface, @@ -179,8 +204,11 @@ import java.util.concurrent.atomic.AtomicInteger; this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.eglSurface = eglSurface; - this.textureId = textureId; - this.frameProcessor = frameProcessor; + this.inputExternalTexId = inputExternalTexId; + this.intermediateTexId = intermediateTexId; + this.frameBuffer = frameBuffer; + this.externalCopyFrameProcessor = externalCopyFrameProcessor; + this.transformationFrameProcessor = transformationFrameProcessor; this.outputWidth = outputWidth; this.outputHeight = outputHeight; this.debugPreviewEglSurface = debugPreviewEglSurface; @@ -189,7 +217,7 @@ import java.util.concurrent.atomic.AtomicInteger; pendingInputFrameCount = new AtomicInteger(); availableInputFrameCount = new AtomicInteger(); textureTransformMatrix = new float[16]; - inputSurfaceTexture = new SurfaceTexture(textureId); + inputSurfaceTexture = new SurfaceTexture(inputExternalTexId); inputSurfaceTexture.setOnFrameAvailableListener( surfaceTexture -> { checkState(pendingInputFrameCount.getAndDecrement() > 0); @@ -235,10 +263,15 @@ import java.util.concurrent.atomic.AtomicInteger; try { inputSurfaceTexture.updateTexImage(); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); - GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); - frameProcessor.setTextureTransformMatrix(textureTransformMatrix); - frameProcessor.updateProgramAndDraw(textureId); long presentationTimeNs = inputSurfaceTexture.getTimestamp(); + + GlUtil.focusFramebuffer( + eglDisplay, eglContext, eglSurface, frameBuffer, outputWidth, outputHeight); + externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix); + externalCopyFrameProcessor.updateProgramAndDraw(inputExternalTexId, presentationTimeNs); + + GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); + transformationFrameProcessor.updateProgramAndDraw(intermediateTexId, presentationTimeNs); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs); EGL14.eglSwapBuffers(eglDisplay, eglSurface); @@ -260,8 +293,10 @@ import java.util.concurrent.atomic.AtomicInteger; /** Releases all resources. */ public void release() { - frameProcessor.release(); - GlUtil.deleteTexture(textureId); + externalCopyFrameProcessor.release(); + transformationFrameProcessor.release(); + GlUtil.deleteTexture(inputExternalTexId); + GlUtil.deleteTexture(intermediateTexId); GlUtil.destroyEglContext(eglDisplay, eglContext); inputSurfaceTexture.release(); inputSurface.release(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java index 55bd315111..18be0193e4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java @@ -15,98 +15,13 @@ */ package androidx.media3.transformer; -import static androidx.media3.common.util.Assertions.checkStateNotNull; - -import android.content.Context; -import android.graphics.Matrix; -import android.opengl.GLES20; -import androidx.media3.common.util.GlProgram; -import androidx.media3.common.util.GlUtil; import java.io.IOException; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** Manages a GLSL shader program for applying a transformation matrix to a frame. */ -/* package */ class GlFrameProcessor { +/** Manages a GLSL shader program for processing a frame. */ +/* package */ interface GlFrameProcessor { - static { - GlUtil.glAssertionsEnabled = true; - } - - /** - * Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful - * for converting to the 4x4 column-major format commonly used in OpenGL. - */ - private static float[] getGlMatrixArray(Matrix matrix) { - float[] matrix3x3Array = new float[9]; - matrix.getValues(matrix3x3Array); - float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array); - - // Transpose from row-major to column-major representations. - float[] transposedMatrix4x4Array = new float[16]; - android.opengl.Matrix.transposeM( - transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0); - - return transposedMatrix4x4Array; - } - - /** - * Returns a 4x4 matrix array containing the 3x3 matrix array's contents. - * - *

The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected - * to be in 3 dimensions. The output will have the third row/column's values be an identity - * matrix's values, so that vertex transformations using this matrix will not affect the z axis. - *
- * Input format: [a, b, c, d, e, f, g, h, i]
- * Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i] - */ - private static float[] getMatrix4x4Array(float[] matrix3x3Array) { - float[] matrix4x4Array = new float[16]; - matrix4x4Array[10] = 1; - for (int inputRow = 0; inputRow < 3; inputRow++) { - for (int inputColumn = 0; inputColumn < 3; inputColumn++) { - int outputRow = (inputRow == 2) ? 3 : inputRow; - int outputColumn = (inputColumn == 2) ? 3 : inputColumn; - matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn]; - } - } - return matrix4x4Array; - } - - private static final String VERTEX_SHADER_TRANSFORMATION_PATH = - "shaders/vertex_shader_transformation.glsl"; - private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = - "shaders/fragment_shader_copy_external.glsl"; - private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH = - "shaders/vertex_shader_transformation_es3.glsl"; - private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = - "shaders/fragment_shader_copy_external_yuv_es3.glsl"; - // Color transform coefficients from - // https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f. - private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = { - 1.168f, 1.168f, 1.168f, - 0.0f, -0.188f, 2.148f, - 1.683f, -0.652f, 0.0f, - }; - - private final Context context; - private final Matrix transformationMatrix; - private final boolean enableExperimentalHdrEditing; - - private @MonotonicNonNull GlProgram glProgram; - - /** - * Creates a new instance. - * - * @param context A {@link Context}. - * @param transformationMatrix The transformation matrix to apply to each frame. - * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. - */ - public GlFrameProcessor( - Context context, Matrix transformationMatrix, boolean enableExperimentalHdrEditing) { - this.context = context; - this.transformationMatrix = transformationMatrix; - this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; - } + // TODO(b/214975934): Add getOutputDimensions(inputWidth, inputHeight) and move output dimension + // calculations out of the VideoTranscodingSamplePipeline into the frame processors. /** * Does any initialization necessary such as loading and compiling a GLSL shader programs. @@ -114,43 +29,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; *

This method may only be called after creating the OpenGL context and focusing a render * target. */ - public void initialize() throws IOException { - // TODO(b/205002913): check the loaded program is consistent with the attributes - // and uniforms expected in the code. - String vertexShaderFilePath = - enableExperimentalHdrEditing - ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH - : VERTEX_SHADER_TRANSFORMATION_PATH; - String fragmentShaderFilePath = - enableExperimentalHdrEditing - ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH - : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; - - glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); - // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. - glProgram.setBufferAttribute( - "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); - glProgram.setBufferAttribute( - "aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); - if (enableExperimentalHdrEditing) { - // In HDR editing mode the decoder output is sampled in YUV. - glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); - } - float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix); - glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrixArray); - } - - /** - * Sets the texture transform matrix for converting an external surface texture's coordinates to - * sampling locations. - * - * @param textureTransformMatrix The external surface texture's {@link - * android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}. - */ - public void setTextureTransformMatrix(float[] textureTransformMatrix) { - checkStateNotNull(glProgram); - glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix); - } + void initialize() throws IOException; /** * Updates the shader program's vertex attributes and uniforms, binds them, and draws. @@ -159,23 +38,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * focussing the correct render target before calling this method. * * @param inputTexId The identifier of an OpenGL texture that the fragment shader can sample from. + * @param presentationTimeNs The presentation timestamp of the current frame, in nanoseconds. */ - // TODO(b/214975934): Also pass presentationTimeNs. - public void updateProgramAndDraw(int inputTexId) { - checkStateNotNull(glProgram); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0); - glProgram.use(); - glProgram.bindAttributesAndUniforms(); - GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - } + void updateProgramAndDraw(int inputTexId, long presentationTimeNs); /** Releases all resources. */ - public void release() { - if (glProgram != null) { - glProgram.delete(); - } - } + void release(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationFrameProcessor.java new file mode 100644 index 0000000000..09246eacfb --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationFrameProcessor.java @@ -0,0 +1,126 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.graphics.Matrix; +import android.opengl.GLES20; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Applies a transformation matrix in the vertex shader. */ +/* package */ class TransformationFrameProcessor implements GlFrameProcessor { + + static { + GlUtil.glAssertionsEnabled = true; + } + + private static final String VERTEX_SHADER_TRANSFORMATION_PATH = + "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; + + /** + * Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful + * for converting to the 4x4 column-major format commonly used in OpenGL. + */ + private static float[] getGlMatrixArray(Matrix matrix) { + float[] matrix3x3Array = new float[9]; + matrix.getValues(matrix3x3Array); + float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array); + + // Transpose from row-major to column-major representations. + float[] transposedMatrix4x4Array = new float[16]; + android.opengl.Matrix.transposeM( + transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0); + + return transposedMatrix4x4Array; + } + + /** + * Returns a 4x4 matrix array containing the 3x3 matrix array's contents. + * + *

The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected + * to be in 3 dimensions. The output will have the third row/column's values be an identity + * matrix's values, so that vertex transformations using this matrix will not affect the z axis. + *
+ * Input format: [a, b, c, d, e, f, g, h, i]
+ * Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i] + */ + private static float[] getMatrix4x4Array(float[] matrix3x3Array) { + float[] matrix4x4Array = new float[16]; + matrix4x4Array[10] = 1; + for (int inputRow = 0; inputRow < 3; inputRow++) { + for (int inputColumn = 0; inputColumn < 3; inputColumn++) { + int outputRow = (inputRow == 2) ? 3 : inputRow; + int outputColumn = (inputColumn == 2) ? 3 : inputColumn; + matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn]; + } + } + return matrix4x4Array; + } + + private final Context context; + private final Matrix transformationMatrix; + + private @MonotonicNonNull GlProgram glProgram; + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param transformationMatrix The transformation matrix to apply to each frame. + */ + public TransformationFrameProcessor(Context context, Matrix transformationMatrix) { + this.context = context; + this.transformationMatrix = transformationMatrix; + } + + @Override + public void initialize() throws IOException { + // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms + // expected in the code. + glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.setBufferAttribute( + "aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.setFloatsUniform("uTransformationMatrix", getGlMatrixArray(transformationMatrix)); + } + + @Override + public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) { + checkStateNotNull(glProgram); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0); + glProgram.use(); + glProgram.bindAttributesAndUniforms(); + GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } + + @Override + public void release() { + if (glProgram != null) { + glProgram.delete(); + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 17431ea26d..96fc4e12ea 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -166,7 +166,7 @@ import org.checkerframework.dataflow.qual.Pure; encoderSupportedFormat.width, encoderSupportedFormat.height, inputFormat.pixelWidthHeightRatio, - transformationMatrix, + new TransformationFrameProcessor(context, transformationMatrix), /* outputSurface= */ encoder.getInputSurface(), transformationRequest.enableHdrEditing, debugViewProvider);