loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move inputTexId parameter to GlFrameProcessor#initialize().

Browse Source 
This parameter will not change between frames in the near
future.

PiperOrigin-RevId: 433765986
This commit is contained in:
hschlueter 2022-03-10 16:57:58 +00:00 committed by Ian Baker
parent b165f2c098
commit 0e98c044da
5 changed files with 27 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationFrameProcessorTest.java
View File

@ -94,10 +94,10 @@ public final class TransformationFrameProcessorTest {
Matrix identityMatrix = new Matrix(); Matrix identityMatrix = new Matrix();
transformationFrameProcessor = transformationFrameProcessor =
new TransformationFrameProcessor(getApplicationContext(), identityMatrix); new TransformationFrameProcessor(getApplicationContext(), identityMatrix);
transformationFrameProcessor.initialize(); transformationFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
transformationFrameProcessor.updateProgramAndDraw(inputTexId, /* presentationTimeNs= */ 0); transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -117,11 +117,11 @@ public final class TransformationFrameProcessorTest {
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
transformationFrameProcessor = transformationFrameProcessor =
new TransformationFrameProcessor(getApplicationContext(), translateRightMatrix); new TransformationFrameProcessor(getApplicationContext(), translateRightMatrix);
transformationFrameProcessor.initialize(); transformationFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap = Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
transformationFrameProcessor.updateProgramAndDraw(inputTexId, /* presentationTimeNs= */ 0); transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -141,11 +141,11 @@ public final class TransformationFrameProcessorTest {
scaleNarrowMatrix.postScale(.5f, 1.2f); scaleNarrowMatrix.postScale(.5f, 1.2f);
transformationFrameProcessor = transformationFrameProcessor =
new TransformationFrameProcessor(getApplicationContext(), scaleNarrowMatrix); new TransformationFrameProcessor(getApplicationContext(), scaleNarrowMatrix);
transformationFrameProcessor.initialize(); transformationFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap = Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING); BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
transformationFrameProcessor.updateProgramAndDraw(inputTexId, /* presentationTimeNs= */ 0); transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -168,10 +168,10 @@ public final class TransformationFrameProcessorTest {
rotate90Matrix.postRotate(/* degrees= */ 90); rotate90Matrix.postRotate(/* degrees= */ 90);
transformationFrameProcessor = transformationFrameProcessor =
new TransformationFrameProcessor(getApplicationContext(), rotate90Matrix); new TransformationFrameProcessor(getApplicationContext(), rotate90Matrix);
transformationFrameProcessor.initialize(); transformationFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
transformationFrameProcessor.updateProgramAndDraw(inputTexId, /* presentationTimeNs= */ 0); transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);

6
libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java
View File

@ -58,7 +58,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void initialize() throws IOException { public void initialize(int inputTexId) throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms
// expected in the code. // expected in the code.
String vertexShaderFilePath = String vertexShaderFilePath =
@ -70,6 +70,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH; : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute( glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
@ -94,9 +95,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) { public void updateProgramAndDraw(long presentationTimeNs) {
checkStateNotNull(glProgram); checkStateNotNull(glProgram);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
glProgram.use(); glProgram.use();
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);

18
libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java
View File

@ -183,10 +183,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
int inputExternalTexId = GlUtil.createExternalTexture(); int inputExternalTexId = GlUtil.createExternalTexture();
externalCopyFrameProcessor.initialize(inputExternalTexId);
int intermediateTexId = GlUtil.createTexture(outputWidth, outputHeight); int intermediateTexId = GlUtil.createTexture(outputWidth, outputHeight);
int frameBuffer = GlUtil.createFboForTexture(intermediateTexId); int frameBuffer = GlUtil.createFboForTexture(intermediateTexId);
externalCopyFrameProcessor.initialize(); transformationFrameProcessor.initialize(intermediateTexId);
transformationFrameProcessor.initialize();
return new FrameEditor( return new FrameEditor(
singleThreadExecutorService, singleThreadExecutorService,
@ -196,7 +196,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
externalCopyFrameProcessor, externalCopyFrameProcessor,
transformationFrameProcessor, transformationFrameProcessor,
inputExternalTexId, inputExternalTexId,
intermediateTexId,
frameBuffer, frameBuffer,
outputWidth, outputWidth,
outputHeight, outputHeight,
@ -228,11 +227,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final float[] textureTransformMatrix; private final float[] textureTransformMatrix;
/** /**
* Identifier of the texture where the output of the {@link ExternalCopyFrameProcessor} is written * Identifier of a framebuffer object associated with the intermediate texture that the output of
* to and the {@link TransformationFrameProcessor} reads its input from. * the {@link ExternalCopyFrameProcessor} is written to and the {@link
* TransformationFrameProcessor} reads its input from.
*/ */
private final int intermediateTexId;
/** Identifier of a framebuffer object associated with the intermediate texture. */
private final int frameBuffer; private final int frameBuffer;
private final int outputWidth; private final int outputWidth;
@ -255,7 +253,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
ExternalCopyFrameProcessor externalCopyFrameProcessor, ExternalCopyFrameProcessor externalCopyFrameProcessor,
GlFrameProcessor transformationFrameProcessor, GlFrameProcessor transformationFrameProcessor,
int inputExternalTexId, int inputExternalTexId,
int intermediateTexId,
int frameBuffer, int frameBuffer,
int outputWidth, int outputWidth,
int outputHeight, int outputHeight,
@ -269,7 +266,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
this.externalCopyFrameProcessor = externalCopyFrameProcessor; this.externalCopyFrameProcessor = externalCopyFrameProcessor;
this.transformationFrameProcessor = transformationFrameProcessor; this.transformationFrameProcessor = transformationFrameProcessor;
this.inputExternalTexId = inputExternalTexId; this.inputExternalTexId = inputExternalTexId;
this.intermediateTexId = intermediateTexId;
this.frameBuffer = frameBuffer; this.frameBuffer = frameBuffer;
this.outputWidth = outputWidth; this.outputWidth = outputWidth;
this.outputHeight = outputHeight; this.outputHeight = outputHeight;
@ -402,10 +398,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
eglDisplay, eglContext, eglSurface, frameBuffer, outputWidth, outputHeight); eglDisplay, eglContext, eglSurface, frameBuffer, outputWidth, outputHeight);
externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix); externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix);
externalCopyFrameProcessor.updateProgramAndDraw(inputExternalTexId, presentationTimeNs); externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs);
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
transformationFrameProcessor.updateProgramAndDraw(intermediateTexId, presentationTimeNs); transformationFrameProcessor.updateProgramAndDraw(presentationTimeNs);
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface); EGL14.eglSwapBuffers(eglDisplay, eglSurface);

11
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
View File

@ -28,19 +28,20 @@ import java.io.IOException;
* *
* <p>This method may only be called after creating the OpenGL context and focusing a render * <p>This method may only be called after creating the OpenGL context and focusing a render
* target. * target.
*
* @param inputTexId The identifier of an OpenGL texture that the fragment shader can sample from.
*/ */
void initialize() throws IOException; void initialize(int inputTexId) throws IOException;
/** /**
* Updates the shader program's vertex attributes and uniforms, binds them, and draws. * Updates the shader program's vertex attributes and uniforms, binds them, and draws.
* *
* <p>The frame processor must be {@link #initialize() initialized}. The caller is responsible for * <p>The frame processor must be {@link #initialize(int) initialized}. The caller is responsible
* focussing the correct render target before calling this method. * for focussing the correct render target before calling this method.
* *
* @param inputTexId The identifier of an OpenGL texture that the fragment shader can sample from.
* @param presentationTimeNs The presentation timestamp of the current frame, in nanoseconds. * @param presentationTimeNs The presentation timestamp of the current frame, in nanoseconds.
*/ */
void updateProgramAndDraw(int inputTexId, long presentationTimeNs); void updateProgramAndDraw(long presentationTimeNs);
/** Releases all resources. */ /** Releases all resources. */
void release(); void release();

6
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationFrameProcessor.java
View File

@ -93,10 +93,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void initialize() throws IOException { public void initialize(int inputTexId) throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms
// expected in the code. // expected in the code.
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute( glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
@ -106,9 +107,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) { public void updateProgramAndDraw(long presentationTimeNs) {
checkStateNotNull(glProgram); checkStateNotNull(glProgram);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
glProgram.use(); glProgram.use();
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);