loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split out ExternalCopyFrameProcessor.

Browse Source 
All (later customizable) GlFrameProcessors after the
ExternalCopyFrameProcessor receive their input from a normal OpenGL
texture not an external texture, so they won't need to worry about
the textureTransformMatrix.

PiperOrigin-RevId: 430165652
This commit is contained in:
hschlueter 2022-02-22 10:35:36 +00:00 committed by Ian Baker
parent bdc3af1416
commit e8977d00c5
12 changed files with 383 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java
View File

@ -116,8 +116,8 @@ public final class FrameEditorDataProcessingTest {
public void processData_noEdits_producesExpectedOutput() throws Exception {
Matrix identityMatrix = new Matrix();
setUpAndPrepareFirstFrame(identityMatrix);
Bitmap expectedBitmap = getBitmap(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData();
Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
@ -134,8 +134,8 @@ public final class FrameEditorDataProcessingTest {
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
setUpAndPrepareFirstFrame(translateRightMatrix);
Bitmap expectedBitmap = getBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData();
Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
@ -174,8 +174,8 @@ public final class FrameEditorDataProcessingTest {
Matrix rotate90Matrix = new Matrix();
rotate90Matrix.postRotate(/* degrees= */ 90);
setUpAndPrepareFirstFrame(rotate90Matrix);
Bitmap expectedBitmap = getBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData();
Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
@ -206,13 +206,14 @@ public final class FrameEditorDataProcessingTest {
int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
frameEditorOutputImageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
Context context = getApplicationContext();
frameEditor =
FrameEditor.create(
getApplicationContext(),
context,
width,
height,
PIXEL_WIDTH_HEIGHT_RATIO,
transformationMatrix,
new TransformationFrameProcessor(context, transformationMatrix),
frameEditorOutputImageReader.getSurface(),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE);

14
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java
View File

@ -28,7 +28,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Test for {@link FrameEditor#create(Context, int, int, float, Matrix, Surface, boolean,
* Test for {@link FrameEditor#create(Context, int, int, float, GlFrameProcessor, Surface, boolean,
* Transformer.DebugViewProvider) creating} a {@link FrameEditor}.
*/
@RunWith(AndroidJUnit4.class)
@ -39,12 +39,14 @@ public final class FrameEditorTest {
@Test
public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully()
throws TransformationException {
Context context = getApplicationContext();
FrameEditor.create(
getApplicationContext(),
context,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 1,
new Matrix(),
new TransformationFrameProcessor(context, new Matrix()),
new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE);
@ -52,16 +54,18 @@ public final class FrameEditorTest {
@Test
public void create_withUnsupportedPixelWidthHeightRatio_throwsException() {
Context context = getApplicationContext();
TransformationException exception =
assertThrows(
TransformationException.class,
() ->
FrameEditor.create(
getApplicationContext(),
context,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 2,
new Matrix(),
new TransformationFrameProcessor(context, new Matrix()),
new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE));

24
libraries/transformer/src/main/assets/shaders/fragment_shader_copy_es2.glsl Normal file
View File

@ -0,0 +1,24 @@
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that samples from a (non-external) texture with uTexSampler,
// copying from this texture to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
varying vec2 vTexCoords;
void main() {
gl_FragColor = texture2D(uTexSampler, vTexCoords);
}

3
libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external.glsl → libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl
View File

@ -1,3 +1,4 @@
#version 100
// Copyright 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
@ -12,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Fragment shader that samples from an external texture with uTexSampler,
// ES 2 fragment shader that samples from an external texture with uTexSampler,
// copying from this texture to the current output.
#extension GL_OES_EGL_image_external : require

27
libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl Normal file
View File

@ -0,0 +1,27 @@
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 vertex shader that applies an external surface texture's 4 * 4 texture
// transformation matrix to convert the texture coordinates to the sampling
// locations.
attribute vec4 aFramePosition;
attribute vec4 aTexCoords;
uniform mat4 uTexTransform;
varying vec2 vTexCoords;
void main() {
gl_Position = aFramePosition;
vTexCoords = (uTexTransform * aTexCoords).xy;
}

8
libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl → libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl
View File

@ -13,15 +13,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 vertex shader that applies the 4 * 4 transformation matrix
// uTransformationMatrix.
// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture
// transformation matrix to convert the texture coordinates to the sampling
// locations.
in vec4 aFramePosition;
in vec4 aTexCoords;
uniform mat4 uTexTransform;
uniform mat4 uTransformationMatrix;
out vec2 vTexCoords;
void main() {
gl_Position = uTransformationMatrix * aFramePosition;
gl_Position = aFramePosition;
vTexCoords = (uTexTransform * aTexCoords).xy;
}

6
libraries/transformer/src/main/assets/shaders/vertex_shader_transformation.glsl → libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl
View File

@ -1,3 +1,4 @@
#version 100
// Copyright 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
@ -12,15 +13,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Vertex shader that applies the 4 * 4 transformation matrix
// ES 2 vertex shader that applies the 4 * 4 transformation matrix
// uTransformationMatrix.
attribute vec4 aFramePosition;
attribute vec4 aTexCoords;
uniform mat4 uTexTransform;
uniform mat4 uTransformationMatrix;
varying vec2 vTexCoords;
void main() {
gl_Position = uTransformationMatrix * aFramePosition;
vTexCoords = (uTexTransform * aTexCoords).xy;
vTexCoords = aTexCoords.xy;
}

114
libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.opengl.GLES20;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Copies frames from an external texture and applies color transformations for HDR if needed. */
/* package */ class ExternalCopyFrameProcessor implements GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH =
"shaders/vertex_shader_tex_transform_es2.glsl";
private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH =
"shaders/vertex_shader_tex_transform_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
private final Context context;
private final boolean enableExperimentalHdrEditing;
private @MonotonicNonNull GlProgram glProgram;
public ExternalCopyFrameProcessor(Context context, boolean enableExperimentalHdrEditing) {
this.context = context;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
}
@Override
public void initialize() throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms
// expected in the code.
String vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH
: VERTEX_SHADER_TEX_TRANSFORM_PATH;
String fragmentShaderFilePath =
enableExperimentalHdrEditing
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setBufferAttribute(
"aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
if (enableExperimentalHdrEditing) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
}
/**
* Sets the texture transform matrix for converting an external surface texture's coordinates to
* sampling locations.
*
* @param textureTransformMatrix The external surface texture's {@link
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
*/
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
checkStateNotNull(glProgram);
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
}
@Override
public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) {
checkStateNotNull(glProgram);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
glProgram.use();
glProgram.bindAttributesAndUniforms();
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
}
@Override
public void release() {
if (glProgram != null) {
glProgram.delete();
}
}
}

83
libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java
View File

@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.opengl.EGLContext;
@ -40,7 +39,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*
* <p>Input becomes available on its {@link #getInputSurface() input surface} asynchronously so
* {@link #canProcessData()} needs to be checked before calling {@link #processData()}. Output is
* written to its {@link #create(Context, int, int, float, Matrix, Surface, boolean,
* written to its {@link #create(Context, int, int, float, GlFrameProcessor, Surface, boolean,
* Transformer.DebugViewProvider) output surface}.
*/
/* package */ final class FrameEditor {
@ -56,7 +55,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* @param outputWidth The output width in pixels.
* @param outputHeight The output height in pixels.
* @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
* @param transformationMatrix The transformation matrix to apply to each frame.
* @param transformationFrameProcessor The {@link GlFrameProcessor} to apply to each frame.
* @param outputSurface The {@link Surface}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @param debugViewProvider Provider for optional debug views to show intermediate output.
@ -65,12 +64,14 @@ import java.util.concurrent.atomic.AtomicInteger;
* files fails, or an OpenGL error occurs while creating and configuring the OpenGL
* components.
*/
// TODO(b/214975934): Take a List<GlFrameProcessor> as input and rename FrameEditor to
// FrameProcessorChain.
public static FrameEditor create(
Context context,
int outputWidth,
int outputHeight,
float pixelWidthHeightRatio,
Matrix transformationMatrix,
GlFrameProcessor transformationFrameProcessor,
Surface outputSurface,
boolean enableExperimentalHdrEditing,
Transformer.DebugViewProvider debugViewProvider)
@ -85,8 +86,8 @@ import java.util.concurrent.atomic.AtomicInteger;
TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
GlFrameProcessor frameProcessor =
new GlFrameProcessor(context, transformationMatrix, enableExperimentalHdrEditing);
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing);
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
@ -94,7 +95,9 @@ import java.util.concurrent.atomic.AtomicInteger;
EGLDisplay eglDisplay;
EGLContext eglContext;
EGLSurface eglSurface;
int textureId;
int inputExternalTexId;
int intermediateTexId;
int frameBuffer;
@Nullable EGLSurface debugPreviewEglSurface = null;
try {
eglDisplay = GlUtil.createEglDisplay();
@ -117,9 +120,12 @@ import java.util.concurrent.atomic.AtomicInteger;
}
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
textureId = GlUtil.createExternalTexture();
frameProcessor.initialize();
} catch (IOException | GlUtil.GlException e) {
inputExternalTexId = GlUtil.createExternalTexture();
intermediateTexId = GlUtil.createTexture(outputWidth, outputHeight);
frameBuffer = GlUtil.createFboForTexture(intermediateTexId);
externalCopyFrameProcessor.initialize();
transformationFrameProcessor.initialize();
} catch (GlUtil.GlException | IOException e) {
throw TransformationException.createForFrameEditor(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
@ -138,8 +144,11 @@ import java.util.concurrent.atomic.AtomicInteger;
eglDisplay,
eglContext,
eglSurface,
textureId,
frameProcessor,
inputExternalTexId,
intermediateTexId,
frameBuffer,
externalCopyFrameProcessor,
transformationFrameProcessor,
outputWidth,
outputHeight,
debugPreviewEglSurface,
@ -147,12 +156,25 @@ import java.util.concurrent.atomic.AtomicInteger;
debugPreviewHeight);
}
private final GlFrameProcessor frameProcessor;
// TODO(b/214975934): Write javadoc for fields where the purpose might be unclear to someone less
// familiar with this class and consider grouping some of these fields into new classes to
// reduce the number of constructor parameters.
private final ExternalCopyFrameProcessor externalCopyFrameProcessor;
private final GlFrameProcessor transformationFrameProcessor;
private final float[] textureTransformMatrix;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final EGLSurface eglSurface;
private final int textureId;
/** Indentifier of the external texture the {@code FrameEditor} reads its input from. */
private final int inputExternalTexId;
/**
* Indentifier of the texture where the output of the {@link ExternalCopyFrameProcessor} is
* written to and the {@link TransformationFrameProcessor} reads its input from.
*/
private final int intermediateTexId;
/** Identifier of a framebuffer object associated with the intermediate texture. */
private final int frameBuffer;
private final AtomicInteger pendingInputFrameCount;
private final AtomicInteger availableInputFrameCount;
private final SurfaceTexture inputSurfaceTexture;
@ -169,8 +191,11 @@ import java.util.concurrent.atomic.AtomicInteger;
EGLDisplay eglDisplay,
EGLContext eglContext,
EGLSurface eglSurface,
int textureId,
GlFrameProcessor frameProcessor,
int inputExternalTexId,
int intermediateTexId,
int frameBuffer,
ExternalCopyFrameProcessor externalCopyFrameProcessor,
GlFrameProcessor transformationFrameProcessor,
int outputWidth,
int outputHeight,
@Nullable EGLSurface debugPreviewEglSurface,
@ -179,8 +204,11 @@ import java.util.concurrent.atomic.AtomicInteger;
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.eglSurface = eglSurface;
this.textureId = textureId;
this.frameProcessor = frameProcessor;
this.inputExternalTexId = inputExternalTexId;
this.intermediateTexId = intermediateTexId;
this.frameBuffer = frameBuffer;
this.externalCopyFrameProcessor = externalCopyFrameProcessor;
this.transformationFrameProcessor = transformationFrameProcessor;
this.outputWidth = outputWidth;
this.outputHeight = outputHeight;
this.debugPreviewEglSurface = debugPreviewEglSurface;
@ -189,7 +217,7 @@ import java.util.concurrent.atomic.AtomicInteger;
pendingInputFrameCount = new AtomicInteger();
availableInputFrameCount = new AtomicInteger();
textureTransformMatrix = new float[16];
inputSurfaceTexture = new SurfaceTexture(textureId);
inputSurfaceTexture = new SurfaceTexture(inputExternalTexId);
inputSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> {
checkState(pendingInputFrameCount.getAndDecrement() > 0);
@ -235,10 +263,15 @@ import java.util.concurrent.atomic.AtomicInteger;
try {
inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
frameProcessor.setTextureTransformMatrix(textureTransformMatrix);
frameProcessor.updateProgramAndDraw(textureId);
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
GlUtil.focusFramebuffer(
eglDisplay, eglContext, eglSurface, frameBuffer, outputWidth, outputHeight);
externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix);
externalCopyFrameProcessor.updateProgramAndDraw(inputExternalTexId, presentationTimeNs);
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
transformationFrameProcessor.updateProgramAndDraw(intermediateTexId, presentationTimeNs);
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
@ -260,8 +293,10 @@ import java.util.concurrent.atomic.AtomicInteger;
/** Releases all resources. */
public void release() {
frameProcessor.release();
GlUtil.deleteTexture(textureId);
externalCopyFrameProcessor.release();
transformationFrameProcessor.release();
GlUtil.deleteTexture(inputExternalTexId);
GlUtil.deleteTexture(intermediateTexId);
GlUtil.destroyEglContext(eglDisplay, eglContext);
inputSurfaceTexture.release();
inputSurface.release();

150
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
View File

@ -15,98 +15,13 @@
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.graphics.Matrix;
import android.opengl.GLES20;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Manages a GLSL shader program for applying a transformation matrix to a frame. */
/* package */ class GlFrameProcessor {
/** Manages a GLSL shader program for processing a frame. */
/* package */ interface GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
/**
* Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful
* for converting to the 4x4 column-major format commonly used in OpenGL.
*/
private static float[] getGlMatrixArray(Matrix matrix) {
float[] matrix3x3Array = new float[9];
matrix.getValues(matrix3x3Array);
float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array);
// Transpose from row-major to column-major representations.
float[] transposedMatrix4x4Array = new float[16];
android.opengl.Matrix.transposeM(
transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0);
return transposedMatrix4x4Array;
}
/**
* Returns a 4x4 matrix array containing the 3x3 matrix array's contents.
*
* <p>The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected
* to be in 3 dimensions. The output will have the third row/column's values be an identity
* matrix's values, so that vertex transformations using this matrix will not affect the z axis.
* <br>
* Input format: [a, b, c, d, e, f, g, h, i] <br>
* Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i]
*/
private static float[] getMatrix4x4Array(float[] matrix3x3Array) {
float[] matrix4x4Array = new float[16];
matrix4x4Array[10] = 1;
for (int inputRow = 0; inputRow < 3; inputRow++) {
for (int inputColumn = 0; inputColumn < 3; inputColumn++) {
int outputRow = (inputRow == 2) ? 3 : inputRow;
int outputColumn = (inputColumn == 2) ? 3 : inputColumn;
matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn];
}
}
return matrix4x4Array;
}
private static final String VERTEX_SHADER_TRANSFORMATION_PATH =
"shaders/vertex_shader_transformation.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external.glsl";
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
private final Context context;
private final Matrix transformationMatrix;
private final boolean enableExperimentalHdrEditing;
private @MonotonicNonNull GlProgram glProgram;
/**
* Creates a new instance.
*
* @param context A {@link Context}.
* @param transformationMatrix The transformation matrix to apply to each frame.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
*/
public GlFrameProcessor(
Context context, Matrix transformationMatrix, boolean enableExperimentalHdrEditing) {
this.context = context;
this.transformationMatrix = transformationMatrix;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
}
// TODO(b/214975934): Add getOutputDimensions(inputWidth, inputHeight) and move output dimension
// calculations out of the VideoTranscodingSamplePipeline into the frame processors.
/**
* Does any initialization necessary such as loading and compiling a GLSL shader programs.
@ -114,43 +29,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>This method may only be called after creating the OpenGL context and focusing a render
* target.
*/
public void initialize() throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes
// and uniforms expected in the code.
String vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TRANSFORMATION_ES3_PATH
: VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
enableExperimentalHdrEditing
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setBufferAttribute(
"aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
if (enableExperimentalHdrEditing) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix);
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrixArray);
}
/**
* Sets the texture transform matrix for converting an external surface texture's coordinates to
* sampling locations.
*
* @param textureTransformMatrix The external surface texture's {@link
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
*/
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
checkStateNotNull(glProgram);
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
}
void initialize() throws IOException;
/**
* Updates the shader program's vertex attributes and uniforms, binds them, and draws.
@ -159,23 +38,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* focussing the correct render target before calling this method.
*
* @param inputTexId The identifier of an OpenGL texture that the fragment shader can sample from.
* @param presentationTimeNs The presentation timestamp of the current frame, in nanoseconds.
*/
// TODO(b/214975934): Also pass presentationTimeNs.
public void updateProgramAndDraw(int inputTexId) {
checkStateNotNull(glProgram);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
glProgram.use();
glProgram.bindAttributesAndUniforms();
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
}
void updateProgramAndDraw(int inputTexId, long presentationTimeNs);
/** Releases all resources. */
public void release() {
if (glProgram != null) {
glProgram.delete();
}
}
void release();
}

126
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationFrameProcessor.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.graphics.Matrix;
import android.opengl.GLES20;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Applies a transformation matrix in the vertex shader. */
/* package */ class TransformationFrameProcessor implements GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
private static final String VERTEX_SHADER_TRANSFORMATION_PATH =
"shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl";
/**
* Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful
* for converting to the 4x4 column-major format commonly used in OpenGL.
*/
private static float[] getGlMatrixArray(Matrix matrix) {
float[] matrix3x3Array = new float[9];
matrix.getValues(matrix3x3Array);
float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array);
// Transpose from row-major to column-major representations.
float[] transposedMatrix4x4Array = new float[16];
android.opengl.Matrix.transposeM(
transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0);
return transposedMatrix4x4Array;
}
/**
* Returns a 4x4 matrix array containing the 3x3 matrix array's contents.
*
* <p>The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected
* to be in 3 dimensions. The output will have the third row/column's values be an identity
* matrix's values, so that vertex transformations using this matrix will not affect the z axis.
* <br>
* Input format: [a, b, c, d, e, f, g, h, i] <br>
* Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i]
*/
private static float[] getMatrix4x4Array(float[] matrix3x3Array) {
float[] matrix4x4Array = new float[16];
matrix4x4Array[10] = 1;
for (int inputRow = 0; inputRow < 3; inputRow++) {
for (int inputColumn = 0; inputColumn < 3; inputColumn++) {
int outputRow = (inputRow == 2) ? 3 : inputRow;
int outputColumn = (inputColumn == 2) ? 3 : inputColumn;
matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn];
}
}
return matrix4x4Array;
}
private final Context context;
private final Matrix transformationMatrix;
private @MonotonicNonNull GlProgram glProgram;
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param transformationMatrix The transformation matrix to apply to each frame.
*/
public TransformationFrameProcessor(Context context, Matrix transformationMatrix) {
this.context = context;
this.transformationMatrix = transformationMatrix;
}
@Override
public void initialize() throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms
// expected in the code.
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setBufferAttribute(
"aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setFloatsUniform("uTransformationMatrix", getGlMatrixArray(transformationMatrix));
}
@Override
public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) {
checkStateNotNull(glProgram);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
glProgram.use();
glProgram.bindAttributesAndUniforms();
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
}
@Override
public void release() {
if (glProgram != null) {
glProgram.delete();
}
}
}

2
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -166,7 +166,7 @@ import org.checkerframework.dataflow.qual.Pure;
encoderSupportedFormat.width,
encoderSupportedFormat.height,
inputFormat.pixelWidthHeightRatio,
transformationMatrix,
new TransformationFrameProcessor(context, transformationMatrix),
/* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing,
debugViewProvider);