loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move program initialization to texture processor constructor.

Browse Source 
Once the more advanced GlTextureProcessor interface exists,
it will be possible to change the output size of a GlTextureProcessor
between frames. To keep the re-configuration based on the frame sizes
minimal, things indepedent of the frame size, such as the GlProgram,
can be initialized in the constructor.

PiperOrigin-RevId: 451997584
This commit is contained in:
hschlueter 2022-05-31 09:36:18 +00:00 committed by Marc Baechinger
parent 5cdac6575e
commit 87ab96d352
14 changed files with 267 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java
View File

@ -35,7 +35,6 @@ import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.SingleFrameGlTextureProcessor;
import java.io.IOException;
import java.util.Locale;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A {@link SingleFrameGlTextureProcessor} that overlays a bitmap with a logo and timer on each
@ -57,16 +56,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Paint paint;
private final Bitmap overlayBitmap;
private final Bitmap logoBitmap;
private final Canvas overlayCanvas;
private final GlProgram glProgram;
private float bitmapScaleX;
private float bitmapScaleY;
private int bitmapTexId;
private @MonotonicNonNull Size outputSize;
private @MonotonicNonNull Bitmap logoBitmap;
private @MonotonicNonNull GlProgram glProgram;
public BitmapOverlayProcessor() {
/**
* Creates a new instance.
*
* @throws IOException If a problem occurs while reading shader files.
*/
public BitmapOverlayProcessor(Context context) throws IOException {
paint = new Paint();
paint.setTextSize(64);
paint.setAntiAlias(true);
@ -75,19 +78,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
overlayBitmap =
Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888);
overlayCanvas = new Canvas(overlayBitmap);
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException {
if (inputWidth > inputHeight) {
bitmapScaleX = inputWidth / (float) inputHeight;
bitmapScaleY = 1f;
} else {
bitmapScaleX = 1f;
bitmapScaleY = inputHeight / (float) inputWidth;
}
outputSize = new Size(inputWidth, inputHeight);
try {
logoBitmap =
@ -106,19 +96,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1);
}
@Override
public Size configure(int inputWidth, int inputHeight) {
if (inputWidth > inputHeight) {
bitmapScaleX = inputWidth / (float) inputHeight;
bitmapScaleY = 1f;
} else {
bitmapScaleX = 1f;
bitmapScaleY = inputHeight / (float) inputWidth;
}
glProgram.setFloatUniform("uScaleX", bitmapScaleX);
glProgram.setFloatUniform("uScaleY", bitmapScaleY);
return new Size(inputWidth, inputHeight);
}
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
}
@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use();
@ -137,6 +135,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
flipBitmapVertically(overlayBitmap));
GlUtil.checkGlError();
glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);

42
demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java
View File

@ -16,7 +16,6 @@
package androidx.media3.demo.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.opengl.GLES20;
@ -26,7 +25,6 @@ import androidx.media3.common.util.GlUtil;
import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.SingleFrameGlTextureProcessor;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are
@ -41,14 +39,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final String FRAGMENT_SHADER_PATH = "fragment_shader_vignette_es2.glsl";
private static final float DIMMING_PERIOD_US = 5_600_000f;
private float centerX;
private float centerY;
private float minInnerRadius;
private float deltaInnerRadius;
private float outerRadius;
private @MonotonicNonNull Size outputSize;
private @MonotonicNonNull GlProgram glProgram;
private final GlProgram glProgram;
private final float minInnerRadius;
private final float deltaInnerRadius;
/**
* Creates a new instance.
@ -61,29 +54,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*
* <p>The parameters are given in normalized texture coordinates from 0 to 1.
*
* @param context The {@link Context}.
* @param centerX The x-coordinate of the center of the effect.
* @param centerY The y-coordinate of the center of the effect.
* @param minInnerRadius The lower bound of the radius that is unaffected by the effect.
* @param maxInnerRadius The upper bound of the radius that is unaffected by the effect.
* @param outerRadius The radius after which all pixels are black.
* @throws IOException If a problem occurs while reading shader files.
*/
public PeriodicVignetteProcessor(
float centerX, float centerY, float minInnerRadius, float maxInnerRadius, float outerRadius) {
Context context,
float centerX,
float centerY,
float minInnerRadius,
float maxInnerRadius,
float outerRadius)
throws IOException {
checkArgument(minInnerRadius <= maxInnerRadius);
checkArgument(maxInnerRadius <= outerRadius);
this.centerX = centerX;
this.centerY = centerY;
this.minInnerRadius = minInnerRadius;
this.deltaInnerRadius = maxInnerRadius - minInnerRadius;
this.outerRadius = outerRadius;
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException {
outputSize = new Size(inputWidth, inputHeight);
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uCenter", new float[] {centerX, centerY});
glProgram.setFloatsUniform("uOuterRadius", new float[] {outerRadius});
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
@ -94,14 +85,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
public Size configure(int inputWidth, int inputHeight) {
return new Size(inputWidth, inputHeight);
}
@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use();
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US;
float innerRadius =
minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta));

9
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -19,6 +19,7 @@ import static android.Manifest.permission.READ_EXTERNAL_STORAGE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.net.Uri;
@ -274,12 +275,13 @@ public final class TransformerActivity extends AppCompatActivity {
try {
Class<?> clazz = Class.forName("androidx.media3.demo.transformer.MediaPipeProcessor");
Constructor<?> constructor =
clazz.getConstructor(String.class, String.class, String.class);
clazz.getConstructor(Context.class, String.class, String.class, String.class);
effects.add(
() -> {
(Context context) -> {
try {
return (SingleFrameGlTextureProcessor)
constructor.newInstance(
context,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb",
/* inputStreamName= */ "input_video",
/* outputStreamName= */ "output_video");
@ -294,8 +296,9 @@ public final class TransformerActivity extends AppCompatActivity {
}
if (selectedEffects[2]) {
effects.add(
() ->
(Context context) ->
new PeriodicVignetteProcessor(
context,
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat(

38
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
View File

@ -63,49 +63,36 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl";
private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl";
private final String graphName;
private final String inputStreamName;
private final String outputStreamName;
private final ConditionVariable frameProcessorConditionVariable;
private final FrameProcessor frameProcessor;
private final GlProgram glProgram;
private @MonotonicNonNull FrameProcessor frameProcessor;
private int inputWidth;
private int inputHeight;
private int inputTexId;
private @MonotonicNonNull GlProgram glProgram;
private @MonotonicNonNull TextureFrame outputFrame;
private @MonotonicNonNull RuntimeException frameProcessorPendingError;
/**
* Creates a new texture processor that wraps a MediaPipe graph.
*
* @param context The {@link Context}.
* @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph.
* @throws IOException If a problem occurs while reading shader files or initializing MediaPipe
* resources.
*/
public MediaPipeProcessor(String graphName, String inputStreamName, String outputStreamName) {
checkState(LOADER.isAvailable());
this.graphName = graphName;
this.inputStreamName = inputStreamName;
this.outputStreamName = outputStreamName;
frameProcessorConditionVariable = new ConditionVariable();
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
public MediaPipeProcessor(
Context context, String graphName, String inputStreamName, String outputStreamName)
throws IOException {
this.inputTexId = inputTexId;
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME);
checkState(LOADER.isAvailable());
frameProcessorConditionVariable = new ConditionVariable();
AndroidAssetUtil.initializeNativeAssetManager(context);
EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor =
new FrameProcessor(
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
// Unblock drawFrame when there is an output frame or an error.
frameProcessor.setConsumer(
frame -> {
@ -117,15 +104,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
frameProcessorPendingError = error;
frameProcessorConditionVariable.open();
});
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME);
}
@Override
public Size getOutputSize() {
public Size configure(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
return new Size(inputWidth, inputHeight);
}
@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
frameProcessorConditionVariable.close();
// Pass the input frame to MediaPipe.

9
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
View File

@ -122,7 +122,7 @@ public final class FrameProcessorChainTest {
throws FrameProcessingException {
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
for (Size element : textureProcessorOutputSizes) {
effects.add(() -> new FakeTextureProcessor(element));
effects.add((Context context) -> new FakeTextureProcessor(element));
}
return FrameProcessorChain.create(
getApplicationContext(),
@ -144,15 +144,12 @@ public final class FrameProcessorChainTest {
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) {}
@Override
public Size getOutputSize() {
public Size configure(int inputWidth, int inputHeight) {
return outputSize;
}
@Override
public void drawFrame(long presentationTimeNs) {}
public void drawFrame(int inputTexId, long presentationTimeNs) {}
@Override
public void release() {}

43
libraries/transformer/src/androidTest/java/androidx/media3/transformer/MatrixTransformationProcessorPixelTest.java
View File

@ -19,6 +19,7 @@ import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_A
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.opengl.EGLContext;
@ -56,9 +57,10 @@ public final class MatrixTransformationProcessorPixelTest {
GlUtil.glAssertionsEnabled = true;
}
private final Context context = getApplicationContext();
private final EGLDisplay eglDisplay = GlUtil.createEglDisplay();
private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay);
private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor;
private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationFrameProcessor;
private int inputTexId;
private int outputTexId;
private int width;
@ -80,8 +82,8 @@ public final class MatrixTransformationProcessorPixelTest {
@After
public void release() {
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
if (matrixTransformationFrameProcessor != null) {
matrixTransformationFrameProcessor.release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
@ -90,12 +92,12 @@ public final class MatrixTransformationProcessorPixelTest {
public void drawFrame_noEdits_producesExpectedOutput() throws Exception {
String testId = "drawFrame_noEdits";
Matrix identityMatrix = new Matrix();
matrixTransformationProcessor =
new MatrixTransformationProcessor((long presentationTimeUs) -> identityMatrix);
matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height);
matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(context, (long presentationTimeUs) -> identityMatrix);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0);
matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -113,12 +115,13 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_translateRight";
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
matrixTransformationProcessor =
new MatrixTransformationProcessor((long presentationTimeUs) -> translateRightMatrix);
matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height);
matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0);
matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -136,12 +139,13 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_scaleNarrow";
Matrix scaleNarrowMatrix = new Matrix();
scaleNarrowMatrix.postScale(.5f, 1.2f);
matrixTransformationProcessor =
new MatrixTransformationProcessor((long presentationTimeUs) -> scaleNarrowMatrix);
matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height);
matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH);
matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0);
matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -159,12 +163,13 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_rotate90";
Matrix rotate90Matrix = new Matrix();
rotate90Matrix.postRotate(/* degrees= */ 90);
matrixTransformationProcessor =
new MatrixTransformationProcessor((long presentationTimeUs) -> rotate90Matrix);
matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height);
matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH);
matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0);
matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);

74
libraries/transformer/src/androidTest/java/androidx/media3/transformer/PresentationPixelTest.java
View File

@ -20,6 +20,7 @@ import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_A
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.graphics.Bitmap;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
@ -67,6 +68,7 @@ public final class PresentationPixelTest {
GlUtil.glAssertionsEnabled = true;
}
private final Context context = getApplicationContext();
private final EGLDisplay eglDisplay = GlUtil.createEglDisplay();
private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay);
private @MonotonicNonNull SingleFrameGlTextureProcessor presentationTextureProcessor;
@ -97,14 +99,12 @@ public final class PresentationPixelTest {
@Test
public void drawFrame_noEdits_producesExpectedOutput() throws Exception {
String testId = "drawFrame_noEdits";
presentationTextureProcessor = new Presentation.Builder().build().toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
presentationTextureProcessor = new Presentation.Builder().build().toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
@ -125,14 +125,12 @@ public final class PresentationPixelTest {
new Presentation.Builder()
.setCrop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f)
.build()
.toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
.toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_SMALLER_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
@ -153,14 +151,12 @@ public final class PresentationPixelTest {
new Presentation.Builder()
.setCrop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f)
.build()
.toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
.toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
@ -182,15 +178,13 @@ public final class PresentationPixelTest {
new Presentation.Builder()
.setAspectRatio(1f, Presentation.LAYOUT_SCALE_TO_FIT)
.build()
.toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
.toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_NARROW_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
@ -212,15 +206,13 @@ public final class PresentationPixelTest {
new Presentation.Builder()
.setAspectRatio(2f, Presentation.LAYOUT_SCALE_TO_FIT)
.build()
.toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
.toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_WIDE_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
@ -242,15 +234,13 @@ public final class PresentationPixelTest {
new Presentation.Builder()
.setAspectRatio(1f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP)
.build()
.toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
.toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_NARROW_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
@ -272,15 +262,13 @@ public final class PresentationPixelTest {
new Presentation.Builder()
.setAspectRatio(2f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP)
.build()
.toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
.toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_WIDE_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
@ -302,15 +290,13 @@ public final class PresentationPixelTest {
new Presentation.Builder()
.setAspectRatio(1f, Presentation.LAYOUT_STRETCH_TO_FIT)
.build()
.toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
.toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ASPECT_RATIO_STRETCH_TO_FIT_NARROW_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
@ -332,15 +318,13 @@ public final class PresentationPixelTest {
new Presentation.Builder()
.setAspectRatio(2f, Presentation.LAYOUT_STRETCH_TO_FIT)
.build()
.toGlTextureProcessor();
presentationTextureProcessor.initialize(
getApplicationContext(), inputTexId, inputWidth, inputHeight);
Size outputSize = presentationTextureProcessor.getOutputSize();
.toGlTextureProcessor(context);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ASPECT_RATIO_STRETCH_TO_FIT_WIDE_PNG_ASSET_PATH);
presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0);
presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());

34
libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java
View File

@ -24,7 +24,6 @@ import android.util.Size;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Copies frames from an external texture and applies color transformations for HDR if needed. */
/* package */ class ExternalTextureProcessor implements SingleFrameGlTextureProcessor {
@ -49,22 +48,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
1.683f, -0.652f, 0.0f,
};
private final boolean enableExperimentalHdrEditing;
private final GlProgram glProgram;
private @MonotonicNonNull Size size;
private @MonotonicNonNull GlProgram glProgram;
public ExternalTextureProcessor(boolean enableExperimentalHdrEditing) {
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
/**
* Creates a new instance.
*
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @throws IOException If a problem occurs while reading shader files.
*/
public ExternalTextureProcessor(Context context, boolean enableExperimentalHdrEditing)
throws IOException {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
size = new Size(inputWidth, inputHeight);
String vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH
@ -74,7 +67,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition",
@ -87,8 +79,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public Size getOutputSize() {
return checkStateNotNull(size);
public Size configure(int inputWidth, int inputHeight) {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
return new Size(inputWidth, inputHeight);
}
/**
@ -104,10 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);

74
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
View File

@ -171,22 +171,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
ExternalTextureProcessor externalTextureProcessor =
new ExternalTextureProcessor(enableExperimentalHdrEditing);
new ExternalTextureProcessor(context, enableExperimentalHdrEditing);
ImmutableList<SingleFrameGlTextureProcessor> textureProcessors =
getTextureProcessors(externalTextureProcessor, pixelWidthHeightRatio, effects);
getTextureProcessors(context, externalTextureProcessor, pixelWidthHeightRatio, effects);
// Initialize texture processors.
int inputExternalTexId = GlUtil.createExternalTexture();
externalTextureProcessor.initialize(context, inputExternalTexId, inputWidth, inputHeight);
int[] framebuffers = new int[textureProcessors.size() - 1];
Size inputSize = externalTextureProcessor.getOutputSize();
Size outputSize = externalTextureProcessor.configure(inputWidth, inputHeight);
ImmutableList.Builder<TextureInfo> intermediateTextures = new ImmutableList.Builder<>();
for (int i = 1; i < textureProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight());
framebuffers[i - 1] = GlUtil.createFboForTexture(inputTexId);
int texId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight());
int fboId = GlUtil.createFboForTexture(texId);
intermediateTextures.add(
new TextureInfo(texId, fboId, outputSize.getWidth(), outputSize.getHeight()));
SingleFrameGlTextureProcessor textureProcessor = textureProcessors.get(i);
textureProcessor.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight());
inputSize = textureProcessor.getOutputSize();
outputSize = textureProcessor.configure(outputSize.getWidth(), outputSize.getHeight());
}
return new FrameProcessorChain(
eglDisplay,
@ -194,16 +193,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
singleThreadExecutorService,
inputExternalTexId,
streamOffsetUs,
framebuffers,
intermediateTextures.build(),
textureProcessors,
outputSize,
listener,
enableExperimentalHdrEditing);
}
private static ImmutableList<SingleFrameGlTextureProcessor> getTextureProcessors(
Context context,
ExternalTextureProcessor externalTextureProcessor,
float pixelWidthHeightRatio,
List<GlEffect> effects) {
List<GlEffect> effects)
throws IOException {
ImmutableList.Builder<SingleFrameGlTextureProcessor> textureProcessors =
new ImmutableList.Builder<SingleFrameGlTextureProcessor>().add(externalTextureProcessor);
@ -233,15 +235,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build();
if (!matrixTransformations.isEmpty()) {
textureProcessors.add(new MatrixTransformationProcessor(matrixTransformations));
textureProcessors.add(new MatrixTransformationProcessor(context, matrixTransformations));
matrixTransformationListBuilder = new ImmutableList.Builder<>();
}
textureProcessors.add(effect.toGlTextureProcessor());
textureProcessors.add(effect.toGlTextureProcessor(context));
}
ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build();
if (!matrixTransformations.isEmpty()) {
textureProcessors.add(new MatrixTransformationProcessor(matrixTransformations));
textureProcessors.add(new MatrixTransformationProcessor(context, matrixTransformations));
}
return textureProcessors.build();
@ -265,11 +267,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final ConcurrentLinkedQueue<Future<?>> futures;
/** Number of frames {@linkplain #registerInputFrame() registered} but not fully processed. */
private final AtomicInteger pendingFrameCount;
/** Wraps the {@link #inputSurfaceTexture}. */
private final Surface inputSurface;
/** Associated with an OpenGL external texture. */
private final SurfaceTexture inputSurfaceTexture;
/** Identifier of the OpenGL texture associated with the input {@link SurfaceTexture}. */
private final int inputExternalTexId;
/** Transformation matrix associated with the {@link #inputSurfaceTexture}. */
private final float[] textureTransformMatrix;
@ -278,12 +281,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors} at indices >= 1.
*/
private final ImmutableList<SingleFrameGlTextureProcessor> textureProcessors;
/**
* Identifiers of a framebuffer object associated with the intermediate textures that receive
* output from the previous {@link SingleFrameGlTextureProcessor}, and provide input for the
* following {@link SingleFrameGlTextureProcessor}.
* {@link TextureInfo} instances describing the intermediate textures that receive output from the
* previous {@link SingleFrameGlTextureProcessor}, and provide input for the following {@link
* SingleFrameGlTextureProcessor}.
*/
private final int[] framebuffers;
private final ImmutableList<TextureInfo> intermediateTextures;
/** The last texture processor's output {@link Size}. */
private final Size recommendedOutputSize;
private final Listener listener;
@ -318,8 +324,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ExecutorService singleThreadExecutorService,
int inputExternalTexId,
long streamOffsetUs,
int[] framebuffers,
ImmutableList<TextureInfo> intermediateTextures,
ImmutableList<SingleFrameGlTextureProcessor> textureProcessors,
Size recommendedOutputSize,
Listener listener,
boolean enableExperimentalHdrEditing) {
checkState(!textureProcessors.isEmpty());
@ -327,9 +334,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.singleThreadExecutorService = singleThreadExecutorService;
this.inputExternalTexId = inputExternalTexId;
this.streamOffsetUs = streamOffsetUs;
this.framebuffers = framebuffers;
this.intermediateTextures = intermediateTextures;
this.textureProcessors = textureProcessors;
this.recommendedOutputSize = recommendedOutputSize;
this.listener = listener;
this.stopProcessing = new AtomicBoolean();
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
@ -350,7 +359,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* SurfaceView) output surface}.
*/
public Size getOutputSize() {
return getLast(textureProcessors).getOutputSize();
return recommendedOutputSize;
}
/**
@ -493,37 +502,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
((ExternalTextureProcessor) textureProcessors.get(0))
.setTextureTransformMatrix(textureTransformMatrix);
int inputTexId = inputExternalTexId;
for (int i = 0; i < textureProcessors.size() - 1; i++) {
if (stopProcessing.get()) {
return;
}
Size intermediateSize = textureProcessors.get(i).getOutputSize();
TextureInfo outputTexture = intermediateTextures.get(i);
GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
outputEglSurface,
framebuffers[i],
intermediateSize.getWidth(),
intermediateSize.getHeight());
outputTexture.fboId,
outputTexture.width,
outputTexture.height);
clearOutputFrame();
textureProcessors.get(i).drawFrame(presentationTimeUs);
textureProcessors.get(i).drawFrame(inputTexId, presentationTimeUs);
inputTexId = outputTexture.texId;
}
GlUtil.focusEglSurface(eglDisplay, eglContext, outputEglSurface, outputWidth, outputHeight);
clearOutputFrame();
getLast(textureProcessors).drawFrame(presentationTimeUs);
getLast(textureProcessors).drawFrame(inputTexId, presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, inputFrameTimeNs);
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
if (debugSurfaceViewWrapper != null) {
long framePresentationTimeUs = presentationTimeUs;
long finalPresentationTimeUs = presentationTimeUs;
int finalInputTexId = inputTexId;
debugSurfaceViewWrapper.maybeRenderToSurfaceView(
() -> {
clearOutputFrame();
try {
getLast(textureProcessors).drawFrame(framePresentationTimeUs);
getLast(textureProcessors).drawFrame(finalInputTexId, finalPresentationTimeUs);
} catch (FrameProcessingException e) {
Log.d(TAG, "Error rendering to debug preview", e);
}

8
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java
View File

@ -15,19 +15,21 @@
*/
package androidx.media3.transformer;
import android.content.Context;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
/**
* Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation.
*
* <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlTextureProcessor() converted} to a {@link SingleFrameGlTextureProcessor} which applies the
* effect.
* #toGlTextureProcessor(Context) converted} to a {@link SingleFrameGlTextureProcessor} which
* applies the effect.
*/
@UnstableApi
public interface GlEffect {
/** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */
// TODO(b/227625423): use GlTextureProcessor here once this interface exists.
SingleFrameGlTextureProcessor toGlTextureProcessor();
SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) throws IOException;
}

6
libraries/transformer/src/main/java/androidx/media3/transformer/GlMatrixTransformation.java
View File

@ -15,9 +15,11 @@
*/
package androidx.media3.transformer;
import android.content.Context;
import android.opengl.Matrix;
import android.util.Size;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
/**
* Specifies a 4x4 transformation {@link Matrix} to apply in the vertex shader for each frame.
@ -49,7 +51,7 @@ public interface GlMatrixTransformation extends GlEffect {
float[] getGlMatrixArray(long presentationTimeUs);
@Override
default SingleFrameGlTextureProcessor toGlTextureProcessor() {
return new MatrixTransformationProcessor(this);
default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) throws IOException {
return new MatrixTransformationProcessor(context, this);
}
}

45
libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java
View File

@ -17,7 +17,6 @@ package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.opengl.GLES20;
@ -29,7 +28,6 @@ import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.util.Arrays;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into
@ -84,37 +82,45 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*/
private ImmutableList<float[]> visiblePolygon;
private @MonotonicNonNull Size outputSize;
private @MonotonicNonNull GlProgram glProgram;
private final GlProgram glProgram;
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation
* matrix to use for each frame.
* @throws IOException If a problem occurs while reading shader files.
*/
public MatrixTransformationProcessor(MatrixTransformation matrixTransformation) {
this(ImmutableList.of(matrixTransformation));
public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation)
throws IOException {
this(context, ImmutableList.of(matrixTransformation));
}
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation
* matrix to use for each frame.
* @throws IOException If a problem occurs while reading shader files.
*/
public MatrixTransformationProcessor(GlMatrixTransformation matrixTransformation) {
this(ImmutableList.of(matrixTransformation));
public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation)
throws IOException {
this(context, ImmutableList.of(matrixTransformation));
}
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* @throws IOException If a problem occurs while reading shader files.
*/
public MatrixTransformationProcessor(
ImmutableList<GlMatrixTransformation> matrixTransformations) {
Context context, ImmutableList<GlMatrixTransformation> matrixTransformations)
throws IOException {
this.matrixTransformations = matrixTransformations;
transformationMatrixCache = new float[matrixTransformations.size()][16];
@ -122,38 +128,33 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
tempResultMatrix = new float[16];
Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
visiblePolygon = NDC_SQUARE;
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH);
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException {
public Size configure(int inputWidth, int inputHeight) {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
outputSize = new Size(inputWidth, inputHeight);
Size outputSize = new Size(inputWidth, inputHeight);
for (int i = 0; i < matrixTransformations.size(); i++) {
outputSize =
matrixTransformations.get(i).configure(outputSize.getWidth(), outputSize.getHeight());
}
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
return outputSize;
}
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
}
@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
updateCompositeTransformationMatrixAndVisiblePolygon(presentationTimeUs);
if (visiblePolygon.size() < 3) {
return; // Need at least three visible vertices for a triangle.
}
try {
checkStateNotNull(glProgram).use();
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrix);
glProgram.setBufferAttribute(
"aFramePosition",
@ -170,10 +171,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void release() {
if (glProgram != null) {
glProgram.delete();
}
}
/**
* Updates {@link #compositeTransformationMatrix} and {@link #visiblePolygon} based on the given

39
libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java
View File

@ -15,10 +15,8 @@
*/
package androidx.media3.transformer;
import android.content.Context;
import android.util.Size;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
/**
* Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels
@ -27,11 +25,13 @@ import java.io.IOException;
* <p>Methods must be called in the following order:
*
* <ol>
* <li>The constructor, for implementation-specific arguments.
* <li>{@link #initialize(Context, int, int, int)}, to set up graphics initialization.
* <li>{@link #drawFrame(long)}, to process one frame.
* <li>{@link #configure(int, int)}, to configure the frame processor based on the input
* dimensions.
* <li>{@link #drawFrame(int, long)}, to process one frame.
* <li>{@link #release()}, upon conclusion of processing.
* </ol>
*
* <p>All methods in this class must be called on the thread that owns the OpenGL context.
*/
@UnstableApi
// TODO(b/227625423): Add GlTextureProcessor interface for async texture processors and make this an
@ -39,42 +39,31 @@ import java.io.IOException;
public interface SingleFrameGlTextureProcessor {
/**
* Performs all initialization that requires OpenGL, such as, loading and compiling a GLSL shader
* program.
* Configures the texture processor based on the input dimensions.
*
* <p>This method may only be called if there is a current OpenGL context.
* <p>This method can be called multiple times.
*
* @param context The {@link Context}.
* @param inputTexId Identifier of a 2D OpenGL texture.
* @param inputWidth The input width, in pixels.
* @param inputHeight The input height, in pixels.
* @throws IOException If an error occurs while reading resources.
* @return The output {@link Size} of frames processed through {@link #drawFrame(int, long)}.
*/
void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException;
/**
* Returns the output {@link Size} of frames processed through {@link #drawFrame(long)}.
*
* <p>This method may only be called after the texture processor has been {@link
* #initialize(Context, int, int, int) initialized}.
*/
Size getOutputSize();
Size configure(int inputWidth, int inputHeight);
/**
* Draws one frame.
*
* <p>This method may only be called after the texture processor has been {@link
* #initialize(Context, int, int, int) initialized}. The caller is responsible for focussing the
* correct render target before calling this method.
* <p>This method may only be called after the texture processor has been {@link #configure(int,
* int) configured}. The caller is responsible for focussing the correct render target before
* calling this method.
*
* <p>A minimal implementation should tell OpenGL to use its shader program, bind the shader
* program's vertex attributes and uniforms, and issue a drawing command.
*
* @param inputTexId Identifier of a 2D OpenGL texture containing the input frame.
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
* @throws FrameProcessingException If an error occurs while processing or drawing the frame.
*/
void drawFrame(long presentationTimeUs) throws FrameProcessingException;
void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException;
/** Releases all resources. */
void release();

46
libraries/transformer/src/main/java/androidx/media3/transformer/TextureInfo.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import androidx.media3.common.util.UnstableApi;
/** Contains information describing an OpenGL texture. */
@UnstableApi
/* package */ final class TextureInfo {
/** The OpenGL texture identifier. */
public final int texId;
/** Identifier of a framebuffer object associated with the texture. */
public final int fboId;
/** The width of the texture, in pixels. */
public final int width;
/** The height of the texture, in pixels. */
public final int height;
/**
* Creates a new instance.
*
* @param texId The OpenGL texture identifier.
* @param fboId Identifier of a framebuffer object associated with the texture.
* @param width The width of the texture, in pixels.
* @param height The height of the texture, in pixels.
*/
public TextureInfo(int texId, int fboId, int width, int height) {
this.texId = texId;
this.fboId = fboId;
this.width = width;
this.height = height;
}
}