diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java index 85a8d59b6d..67dff8aa95 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java @@ -35,7 +35,6 @@ import androidx.media3.transformer.FrameProcessingException; import androidx.media3.transformer.SingleFrameGlTextureProcessor; import java.io.IOException; import java.util.Locale; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A {@link SingleFrameGlTextureProcessor} that overlays a bitmap with a logo and timer on each @@ -57,16 +56,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Paint paint; private final Bitmap overlayBitmap; + private final Bitmap logoBitmap; private final Canvas overlayCanvas; + private final GlProgram glProgram; private float bitmapScaleX; private float bitmapScaleY; private int bitmapTexId; - private @MonotonicNonNull Size outputSize; - private @MonotonicNonNull Bitmap logoBitmap; - private @MonotonicNonNull GlProgram glProgram; - public BitmapOverlayProcessor() { + /** + * Creates a new instance. + * + * @throws IOException If a problem occurs while reading shader files. + */ + public BitmapOverlayProcessor(Context context) throws IOException { paint = new Paint(); paint.setTextSize(64); paint.setAntiAlias(true); @@ -75,19 +78,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; overlayBitmap = Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888); overlayCanvas = new Canvas(overlayBitmap); - } - - @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) - throws IOException { - if (inputWidth > inputHeight) { - bitmapScaleX = inputWidth / (float) inputHeight; - bitmapScaleY = 1f; - } else { - bitmapScaleX = 1f; - bitmapScaleY = inputHeight / (float) inputWidth; - } - outputSize = new Size(inputWidth, inputHeight); try { logoBitmap = @@ -106,19 +96,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0); glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1); + } + + @Override + public Size configure(int inputWidth, int inputHeight) { + if (inputWidth > inputHeight) { + bitmapScaleX = inputWidth / (float) inputHeight; + bitmapScaleY = 1f; + } else { + bitmapScaleX = 1f; + bitmapScaleY = inputHeight / (float) inputWidth; + } + glProgram.setFloatUniform("uScaleX", bitmapScaleX); glProgram.setFloatUniform("uScaleY", bitmapScaleY); + + return new Size(inputWidth, inputHeight); } @Override - public Size getOutputSize() { - return checkStateNotNull(outputSize); - } - - @Override - public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { try { checkStateNotNull(glProgram).use(); @@ -137,6 +135,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; flipBitmapVertically(overlayBitmap)); GlUtil.checkGlError(); + glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0); glProgram.bindAttributesAndUniforms(); // The four-vertex triangle strip forms a quad. GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java index 74c1a31294..42aec157e0 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java @@ -16,7 +16,6 @@ package androidx.media3.demo.transformer; import static androidx.media3.common.util.Assertions.checkArgument; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.GLES20; @@ -26,7 +25,6 @@ import androidx.media3.common.util.GlUtil; import androidx.media3.transformer.FrameProcessingException; import androidx.media3.transformer.SingleFrameGlTextureProcessor; import java.io.IOException; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are @@ -41,14 +39,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final String FRAGMENT_SHADER_PATH = "fragment_shader_vignette_es2.glsl"; private static final float DIMMING_PERIOD_US = 5_600_000f; - private float centerX; - private float centerY; - private float minInnerRadius; - private float deltaInnerRadius; - private float outerRadius; - - private @MonotonicNonNull Size outputSize; - private @MonotonicNonNull GlProgram glProgram; + private final GlProgram glProgram; + private final float minInnerRadius; + private final float deltaInnerRadius; /** * Creates a new instance. @@ -61,29 +54,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * *

The parameters are given in normalized texture coordinates from 0 to 1. * + * @param context The {@link Context}. * @param centerX The x-coordinate of the center of the effect. * @param centerY The y-coordinate of the center of the effect. * @param minInnerRadius The lower bound of the radius that is unaffected by the effect. * @param maxInnerRadius The upper bound of the radius that is unaffected by the effect. * @param outerRadius The radius after which all pixels are black. + * @throws IOException If a problem occurs while reading shader files. */ public PeriodicVignetteProcessor( - float centerX, float centerY, float minInnerRadius, float maxInnerRadius, float outerRadius) { + Context context, + float centerX, + float centerY, + float minInnerRadius, + float maxInnerRadius, + float outerRadius) + throws IOException { checkArgument(minInnerRadius <= maxInnerRadius); checkArgument(maxInnerRadius <= outerRadius); - this.centerX = centerX; - this.centerY = centerY; this.minInnerRadius = minInnerRadius; this.deltaInnerRadius = maxInnerRadius - minInnerRadius; - this.outerRadius = outerRadius; - } - - @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) - throws IOException { - outputSize = new Size(inputWidth, inputHeight); glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); glProgram.setFloatsUniform("uCenter", new float[] {centerX, centerY}); glProgram.setFloatsUniform("uOuterRadius", new float[] {outerRadius}); // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. @@ -94,14 +85,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public Size getOutputSize() { - return checkStateNotNull(outputSize); + public Size configure(int inputWidth, int inputHeight) { + return new Size(inputWidth, inputHeight); } @Override - public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { try { - checkStateNotNull(glProgram).use(); + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; float innerRadius = minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 594459e315..178cfc0098 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -19,6 +19,7 @@ import static android.Manifest.permission.READ_EXTERNAL_STORAGE; import static androidx.media3.common.util.Assertions.checkNotNull; import android.app.Activity; +import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; import android.net.Uri; @@ -274,12 +275,13 @@ public final class TransformerActivity extends AppCompatActivity { try { Class clazz = Class.forName("androidx.media3.demo.transformer.MediaPipeProcessor"); Constructor constructor = - clazz.getConstructor(String.class, String.class, String.class); + clazz.getConstructor(Context.class, String.class, String.class, String.class); effects.add( - () -> { + (Context context) -> { try { return (SingleFrameGlTextureProcessor) constructor.newInstance( + context, /* graphName= */ "edge_detector_mediapipe_graph.binarypb", /* inputStreamName= */ "input_video", /* outputStreamName= */ "output_video"); @@ -294,8 +296,9 @@ public final class TransformerActivity extends AppCompatActivity { } if (selectedEffects[2]) { effects.add( - () -> + (Context context) -> new PeriodicVignetteProcessor( + context, bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), /* minInnerRadius= */ bundle.getFloat( diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java index 8860a2ccc9..65a8666634 100644 --- a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java @@ -63,49 +63,36 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl"; private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl"; - private final String graphName; - private final String inputStreamName; - private final String outputStreamName; private final ConditionVariable frameProcessorConditionVariable; + private final FrameProcessor frameProcessor; + private final GlProgram glProgram; - private @MonotonicNonNull FrameProcessor frameProcessor; private int inputWidth; private int inputHeight; - private int inputTexId; - private @MonotonicNonNull GlProgram glProgram; private @MonotonicNonNull TextureFrame outputFrame; private @MonotonicNonNull RuntimeException frameProcessorPendingError; /** * Creates a new texture processor that wraps a MediaPipe graph. * + * @param context The {@link Context}. * @param graphName Name of a MediaPipe graph asset to load. * @param inputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph. + * @throws IOException If a problem occurs while reading shader files or initializing MediaPipe + * resources. */ - public MediaPipeProcessor(String graphName, String inputStreamName, String outputStreamName) { - checkState(LOADER.isAvailable()); - this.graphName = graphName; - this.inputStreamName = inputStreamName; - this.outputStreamName = outputStreamName; - frameProcessorConditionVariable = new ConditionVariable(); - } - - @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) + public MediaPipeProcessor( + Context context, String graphName, String inputStreamName, String outputStreamName) throws IOException { - this.inputTexId = inputTexId; - this.inputWidth = inputWidth; - this.inputHeight = inputHeight; - glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME); + checkState(LOADER.isAvailable()); + frameProcessorConditionVariable = new ConditionVariable(); AndroidAssetUtil.initializeNativeAssetManager(context); - EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); frameProcessor = new FrameProcessor( context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName); - // Unblock drawFrame when there is an output frame or an error. frameProcessor.setConsumer( frame -> { @@ -117,15 +104,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; frameProcessorPendingError = error; frameProcessorConditionVariable.open(); }); + glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME); } @Override - public Size getOutputSize() { + public Size configure(int inputWidth, int inputHeight) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; return new Size(inputWidth, inputHeight); } @Override - public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { frameProcessorConditionVariable.close(); // Pass the input frame to MediaPipe. diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java index 05920f2058..92a0a5395b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java @@ -122,7 +122,7 @@ public final class FrameProcessorChainTest { throws FrameProcessingException { ImmutableList.Builder effects = new ImmutableList.Builder<>(); for (Size element : textureProcessorOutputSizes) { - effects.add(() -> new FakeTextureProcessor(element)); + effects.add((Context context) -> new FakeTextureProcessor(element)); } return FrameProcessorChain.create( getApplicationContext(), @@ -144,15 +144,12 @@ public final class FrameProcessorChainTest { } @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) {} - - @Override - public Size getOutputSize() { + public Size configure(int inputWidth, int inputHeight) { return outputSize; } @Override - public void drawFrame(long presentationTimeNs) {} + public void drawFrame(int inputTexId, long presentationTimeNs) {} @Override public void release() {} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/MatrixTransformationProcessorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/MatrixTransformationProcessorPixelTest.java index 9a5648274d..f25c09a16b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/MatrixTransformationProcessorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/MatrixTransformationProcessorPixelTest.java @@ -19,6 +19,7 @@ import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_A import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; +import android.content.Context; import android.graphics.Bitmap; import android.graphics.Matrix; import android.opengl.EGLContext; @@ -56,9 +57,10 @@ public final class MatrixTransformationProcessorPixelTest { GlUtil.glAssertionsEnabled = true; } + private final Context context = getApplicationContext(); private final EGLDisplay eglDisplay = GlUtil.createEglDisplay(); private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay); - private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor; + private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationFrameProcessor; private int inputTexId; private int outputTexId; private int width; @@ -80,8 +82,8 @@ public final class MatrixTransformationProcessorPixelTest { @After public void release() { - if (matrixTransformationProcessor != null) { - matrixTransformationProcessor.release(); + if (matrixTransformationFrameProcessor != null) { + matrixTransformationFrameProcessor.release(); } GlUtil.destroyEglContext(eglDisplay, eglContext); } @@ -90,12 +92,12 @@ public final class MatrixTransformationProcessorPixelTest { public void drawFrame_noEdits_producesExpectedOutput() throws Exception { String testId = "drawFrame_noEdits"; Matrix identityMatrix = new Matrix(); - matrixTransformationProcessor = - new MatrixTransformationProcessor((long presentationTimeUs) -> identityMatrix); - matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height); + matrixTransformationFrameProcessor = + new MatrixTransformationProcessor(context, (long presentationTimeUs) -> identityMatrix); + matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); - matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0); + matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -113,12 +115,13 @@ public final class MatrixTransformationProcessorPixelTest { String testId = "drawFrame_translateRight"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - matrixTransformationProcessor = - new MatrixTransformationProcessor((long presentationTimeUs) -> translateRightMatrix); - matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height); + matrixTransformationFrameProcessor = + new MatrixTransformationProcessor( + context, /* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix); + matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); - matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0); + matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -136,12 +139,13 @@ public final class MatrixTransformationProcessorPixelTest { String testId = "drawFrame_scaleNarrow"; Matrix scaleNarrowMatrix = new Matrix(); scaleNarrowMatrix.postScale(.5f, 1.2f); - matrixTransformationProcessor = - new MatrixTransformationProcessor((long presentationTimeUs) -> scaleNarrowMatrix); - matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height); + matrixTransformationFrameProcessor = + new MatrixTransformationProcessor( + context, /* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix); + matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH); - matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0); + matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -159,12 +163,13 @@ public final class MatrixTransformationProcessorPixelTest { String testId = "drawFrame_rotate90"; Matrix rotate90Matrix = new Matrix(); rotate90Matrix.postRotate(/* degrees= */ 90); - matrixTransformationProcessor = - new MatrixTransformationProcessor((long presentationTimeUs) -> rotate90Matrix); - matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height); + matrixTransformationFrameProcessor = + new MatrixTransformationProcessor( + context, /* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix); + matrixTransformationFrameProcessor.configure(width, height); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH); - matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0); + matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/PresentationPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/PresentationPixelTest.java index 6e34118d04..e4404c06c5 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/PresentationPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/PresentationPixelTest.java @@ -20,6 +20,7 @@ import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_A import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; +import android.content.Context; import android.graphics.Bitmap; import android.opengl.EGLContext; import android.opengl.EGLDisplay; @@ -67,6 +68,7 @@ public final class PresentationPixelTest { GlUtil.glAssertionsEnabled = true; } + private final Context context = getApplicationContext(); private final EGLDisplay eglDisplay = GlUtil.createEglDisplay(); private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay); private @MonotonicNonNull SingleFrameGlTextureProcessor presentationTextureProcessor; @@ -97,14 +99,12 @@ public final class PresentationPixelTest { @Test public void drawFrame_noEdits_producesExpectedOutput() throws Exception { String testId = "drawFrame_noEdits"; - presentationTextureProcessor = new Presentation.Builder().build().toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + presentationTextureProcessor = new Presentation.Builder().build().toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); @@ -125,14 +125,12 @@ public final class PresentationPixelTest { new Presentation.Builder() .setCrop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f) .build() - .toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + .toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_SMALLER_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); @@ -153,14 +151,12 @@ public final class PresentationPixelTest { new Presentation.Builder() .setCrop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f) .build() - .toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + .toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); @@ -182,15 +178,13 @@ public final class PresentationPixelTest { new Presentation.Builder() .setAspectRatio(1f, Presentation.LAYOUT_SCALE_TO_FIT) .build() - .toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + .toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_NARROW_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); @@ -212,15 +206,13 @@ public final class PresentationPixelTest { new Presentation.Builder() .setAspectRatio(2f, Presentation.LAYOUT_SCALE_TO_FIT) .build() - .toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + .toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_WIDE_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); @@ -242,15 +234,13 @@ public final class PresentationPixelTest { new Presentation.Builder() .setAspectRatio(1f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP) .build() - .toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + .toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_NARROW_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); @@ -272,15 +262,13 @@ public final class PresentationPixelTest { new Presentation.Builder() .setAspectRatio(2f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP) .build() - .toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + .toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_WIDE_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); @@ -302,15 +290,13 @@ public final class PresentationPixelTest { new Presentation.Builder() .setAspectRatio(1f, Presentation.LAYOUT_STRETCH_TO_FIT) .build() - .toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + .toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ASPECT_RATIO_STRETCH_TO_FIT_NARROW_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); @@ -332,15 +318,13 @@ public final class PresentationPixelTest { new Presentation.Builder() .setAspectRatio(2f, Presentation.LAYOUT_STRETCH_TO_FIT) .build() - .toGlTextureProcessor(); - presentationTextureProcessor.initialize( - getApplicationContext(), inputTexId, inputWidth, inputHeight); - Size outputSize = presentationTextureProcessor.getOutputSize(); + .toGlTextureProcessor(context); + Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ASPECT_RATIO_STRETCH_TO_FIT_WIDE_PNG_ASSET_PATH); - presentationTextureProcessor.drawFrame(/* presentationTimeUs= */ 0); + presentationTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( outputSize.getWidth(), outputSize.getHeight()); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java index 8000d738ee..2d68dd63d9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java @@ -24,7 +24,6 @@ import android.util.Size; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import java.io.IOException; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** Copies frames from an external texture and applies color transformations for HDR if needed. */ /* package */ class ExternalTextureProcessor implements SingleFrameGlTextureProcessor { @@ -49,22 +48,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; 1.683f, -0.652f, 0.0f, }; - private final boolean enableExperimentalHdrEditing; + private final GlProgram glProgram; - private @MonotonicNonNull Size size; - private @MonotonicNonNull GlProgram glProgram; - - public ExternalTextureProcessor(boolean enableExperimentalHdrEditing) { - this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; - } - - @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) + /** + * Creates a new instance. + * + * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. + * @throws IOException If a problem occurs while reading shader files. + */ + public ExternalTextureProcessor(Context context, boolean enableExperimentalHdrEditing) throws IOException { - checkArgument(inputWidth > 0, "inputWidth must be positive"); - checkArgument(inputHeight > 0, "inputHeight must be positive"); - - size = new Size(inputWidth, inputHeight); String vertexShaderFilePath = enableExperimentalHdrEditing ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH @@ -74,7 +67,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. glProgram.setBufferAttribute( "aFramePosition", @@ -87,8 +79,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public Size getOutputSize() { - return checkStateNotNull(size); + public Size configure(int inputWidth, int inputHeight) { + checkArgument(inputWidth > 0, "inputWidth must be positive"); + checkArgument(inputHeight > 0, "inputHeight must be positive"); + + return new Size(inputWidth, inputHeight); } /** @@ -104,10 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { checkStateNotNull(glProgram); try { glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); glProgram.bindAttributesAndUniforms(); // The four-vertex triangle strip forms a quad. GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index a1688077c7..9ea96e3a9b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -171,22 +171,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } ExternalTextureProcessor externalTextureProcessor = - new ExternalTextureProcessor(enableExperimentalHdrEditing); + new ExternalTextureProcessor(context, enableExperimentalHdrEditing); ImmutableList textureProcessors = - getTextureProcessors(externalTextureProcessor, pixelWidthHeightRatio, effects); + getTextureProcessors(context, externalTextureProcessor, pixelWidthHeightRatio, effects); // Initialize texture processors. int inputExternalTexId = GlUtil.createExternalTexture(); - externalTextureProcessor.initialize(context, inputExternalTexId, inputWidth, inputHeight); - - int[] framebuffers = new int[textureProcessors.size() - 1]; - Size inputSize = externalTextureProcessor.getOutputSize(); + Size outputSize = externalTextureProcessor.configure(inputWidth, inputHeight); + ImmutableList.Builder intermediateTextures = new ImmutableList.Builder<>(); for (int i = 1; i < textureProcessors.size(); i++) { - int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight()); - framebuffers[i - 1] = GlUtil.createFboForTexture(inputTexId); + int texId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight()); + int fboId = GlUtil.createFboForTexture(texId); + intermediateTextures.add( + new TextureInfo(texId, fboId, outputSize.getWidth(), outputSize.getHeight())); SingleFrameGlTextureProcessor textureProcessor = textureProcessors.get(i); - textureProcessor.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight()); - inputSize = textureProcessor.getOutputSize(); + outputSize = textureProcessor.configure(outputSize.getWidth(), outputSize.getHeight()); } return new FrameProcessorChain( eglDisplay, @@ -194,16 +193,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; singleThreadExecutorService, inputExternalTexId, streamOffsetUs, - framebuffers, + intermediateTextures.build(), textureProcessors, + outputSize, listener, enableExperimentalHdrEditing); } private static ImmutableList getTextureProcessors( + Context context, ExternalTextureProcessor externalTextureProcessor, float pixelWidthHeightRatio, - List effects) { + List effects) + throws IOException { ImmutableList.Builder textureProcessors = new ImmutableList.Builder().add(externalTextureProcessor); @@ -233,15 +235,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); if (!matrixTransformations.isEmpty()) { - textureProcessors.add(new MatrixTransformationProcessor(matrixTransformations)); + textureProcessors.add(new MatrixTransformationProcessor(context, matrixTransformations)); matrixTransformationListBuilder = new ImmutableList.Builder<>(); } - textureProcessors.add(effect.toGlTextureProcessor()); + textureProcessors.add(effect.toGlTextureProcessor(context)); } ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); if (!matrixTransformations.isEmpty()) { - textureProcessors.add(new MatrixTransformationProcessor(matrixTransformations)); + textureProcessors.add(new MatrixTransformationProcessor(context, matrixTransformations)); } return textureProcessors.build(); @@ -265,11 +267,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final ConcurrentLinkedQueue> futures; /** Number of frames {@linkplain #registerInputFrame() registered} but not fully processed. */ private final AtomicInteger pendingFrameCount; - /** Wraps the {@link #inputSurfaceTexture}. */ private final Surface inputSurface; /** Associated with an OpenGL external texture. */ private final SurfaceTexture inputSurfaceTexture; + /** Identifier of the OpenGL texture associated with the input {@link SurfaceTexture}. */ + private final int inputExternalTexId; /** Transformation matrix associated with the {@link #inputSurfaceTexture}. */ private final float[] textureTransformMatrix; @@ -278,12 +281,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors} at indices >= 1. */ private final ImmutableList textureProcessors; + /** - * Identifiers of a framebuffer object associated with the intermediate textures that receive - * output from the previous {@link SingleFrameGlTextureProcessor}, and provide input for the - * following {@link SingleFrameGlTextureProcessor}. + * {@link TextureInfo} instances describing the intermediate textures that receive output from the + * previous {@link SingleFrameGlTextureProcessor}, and provide input for the following {@link + * SingleFrameGlTextureProcessor}. */ - private final int[] framebuffers; + private final ImmutableList intermediateTextures; + /** The last texture processor's output {@link Size}. */ + private final Size recommendedOutputSize; private final Listener listener; @@ -318,8 +324,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ExecutorService singleThreadExecutorService, int inputExternalTexId, long streamOffsetUs, - int[] framebuffers, + ImmutableList intermediateTextures, ImmutableList textureProcessors, + Size recommendedOutputSize, Listener listener, boolean enableExperimentalHdrEditing) { checkState(!textureProcessors.isEmpty()); @@ -327,9 +334,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.singleThreadExecutorService = singleThreadExecutorService; + this.inputExternalTexId = inputExternalTexId; this.streamOffsetUs = streamOffsetUs; - this.framebuffers = framebuffers; + this.intermediateTextures = intermediateTextures; this.textureProcessors = textureProcessors; + this.recommendedOutputSize = recommendedOutputSize; this.listener = listener; this.stopProcessing = new AtomicBoolean(); this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; @@ -350,7 +359,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * SurfaceView) output surface}. */ public Size getOutputSize() { - return getLast(textureProcessors).getOutputSize(); + return recommendedOutputSize; } /** @@ -493,37 +502,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); ((ExternalTextureProcessor) textureProcessors.get(0)) .setTextureTransformMatrix(textureTransformMatrix); + int inputTexId = inputExternalTexId; for (int i = 0; i < textureProcessors.size() - 1; i++) { if (stopProcessing.get()) { return; } - Size intermediateSize = textureProcessors.get(i).getOutputSize(); + TextureInfo outputTexture = intermediateTextures.get(i); GlUtil.focusFramebuffer( eglDisplay, eglContext, outputEglSurface, - framebuffers[i], - intermediateSize.getWidth(), - intermediateSize.getHeight()); + outputTexture.fboId, + outputTexture.width, + outputTexture.height); clearOutputFrame(); - textureProcessors.get(i).drawFrame(presentationTimeUs); + textureProcessors.get(i).drawFrame(inputTexId, presentationTimeUs); + inputTexId = outputTexture.texId; } GlUtil.focusEglSurface(eglDisplay, eglContext, outputEglSurface, outputWidth, outputHeight); clearOutputFrame(); - getLast(textureProcessors).drawFrame(presentationTimeUs); + getLast(textureProcessors).drawFrame(inputTexId, presentationTimeUs); EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, inputFrameTimeNs); EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); if (debugSurfaceViewWrapper != null) { - long framePresentationTimeUs = presentationTimeUs; + long finalPresentationTimeUs = presentationTimeUs; + int finalInputTexId = inputTexId; debugSurfaceViewWrapper.maybeRenderToSurfaceView( () -> { clearOutputFrame(); try { - getLast(textureProcessors).drawFrame(framePresentationTimeUs); + getLast(textureProcessors).drawFrame(finalInputTexId, finalPresentationTimeUs); } catch (FrameProcessingException e) { Log.d(TAG, "Error rendering to debug preview", e); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java index 854446d6a1..fa64cb780d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java @@ -15,19 +15,21 @@ */ package androidx.media3.transformer; +import android.content.Context; import androidx.media3.common.util.UnstableApi; +import java.io.IOException; /** * Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation. * *

Implementations contain information specifying the effect and can be {@linkplain - * #toGlTextureProcessor() converted} to a {@link SingleFrameGlTextureProcessor} which applies the - * effect. + * #toGlTextureProcessor(Context) converted} to a {@link SingleFrameGlTextureProcessor} which + * applies the effect. */ @UnstableApi public interface GlEffect { /** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */ // TODO(b/227625423): use GlTextureProcessor here once this interface exists. - SingleFrameGlTextureProcessor toGlTextureProcessor(); + SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) throws IOException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlMatrixTransformation.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlMatrixTransformation.java index c6d8c37104..24bcbbf2c4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlMatrixTransformation.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlMatrixTransformation.java @@ -15,9 +15,11 @@ */ package androidx.media3.transformer; +import android.content.Context; import android.opengl.Matrix; import android.util.Size; import androidx.media3.common.util.UnstableApi; +import java.io.IOException; /** * Specifies a 4x4 transformation {@link Matrix} to apply in the vertex shader for each frame. @@ -49,7 +51,7 @@ public interface GlMatrixTransformation extends GlEffect { float[] getGlMatrixArray(long presentationTimeUs); @Override - default SingleFrameGlTextureProcessor toGlTextureProcessor() { - return new MatrixTransformationProcessor(this); + default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) throws IOException { + return new MatrixTransformationProcessor(context, this); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java index e464b82245..c8d214fdf4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java @@ -17,7 +17,6 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.GLES20; @@ -29,7 +28,6 @@ import androidx.media3.common.util.UnstableApi; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.Arrays; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into @@ -84,37 +82,45 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; */ private ImmutableList visiblePolygon; - private @MonotonicNonNull Size outputSize; - private @MonotonicNonNull GlProgram glProgram; + private final GlProgram glProgram; /** * Creates a new instance. * + * @param context The {@link Context}. * @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation * matrix to use for each frame. + * @throws IOException If a problem occurs while reading shader files. */ - public MatrixTransformationProcessor(MatrixTransformation matrixTransformation) { - this(ImmutableList.of(matrixTransformation)); + public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation) + throws IOException { + this(context, ImmutableList.of(matrixTransformation)); } /** * Creates a new instance. * + * @param context The {@link Context}. * @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation * matrix to use for each frame. + * @throws IOException If a problem occurs while reading shader files. */ - public MatrixTransformationProcessor(GlMatrixTransformation matrixTransformation) { - this(ImmutableList.of(matrixTransformation)); + public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation) + throws IOException { + this(context, ImmutableList.of(matrixTransformation)); } /** * Creates a new instance. * + * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * apply to each frame in order. + * @throws IOException If a problem occurs while reading shader files. */ public MatrixTransformationProcessor( - ImmutableList matrixTransformations) { + Context context, ImmutableList matrixTransformations) + throws IOException { this.matrixTransformations = matrixTransformations; transformationMatrixCache = new float[matrixTransformations.size()][16]; @@ -122,38 +128,33 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; tempResultMatrix = new float[16]; Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); visiblePolygon = NDC_SQUARE; + glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); } @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) - throws IOException { + public Size configure(int inputWidth, int inputHeight) { checkArgument(inputWidth > 0, "inputWidth must be positive"); checkArgument(inputHeight > 0, "inputHeight must be positive"); - outputSize = new Size(inputWidth, inputHeight); + Size outputSize = new Size(inputWidth, inputHeight); for (int i = 0; i < matrixTransformations.size(); i++) { outputSize = matrixTransformations.get(i).configure(outputSize.getWidth(), outputSize.getHeight()); } - glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + return outputSize; } @Override - public Size getOutputSize() { - return checkStateNotNull(outputSize); - } - - @Override - public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { updateCompositeTransformationMatrixAndVisiblePolygon(presentationTimeUs); if (visiblePolygon.size() < 3) { return; // Need at least three visible vertices for a triangle. } try { - checkStateNotNull(glProgram).use(); + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrix); glProgram.setBufferAttribute( "aFramePosition", @@ -170,9 +171,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void release() { - if (glProgram != null) { - glProgram.delete(); - } + glProgram.delete(); } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java index 3e0546f7d2..873ebfe130 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java @@ -15,10 +15,8 @@ */ package androidx.media3.transformer; -import android.content.Context; import android.util.Size; import androidx.media3.common.util.UnstableApi; -import java.io.IOException; /** * Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels @@ -27,11 +25,13 @@ import java.io.IOException; *

Methods must be called in the following order: * *

    - *
  1. The constructor, for implementation-specific arguments. - *
  2. {@link #initialize(Context, int, int, int)}, to set up graphics initialization. - *
  3. {@link #drawFrame(long)}, to process one frame. + *
  4. {@link #configure(int, int)}, to configure the frame processor based on the input + * dimensions. + *
  5. {@link #drawFrame(int, long)}, to process one frame. *
  6. {@link #release()}, upon conclusion of processing. *
+ * + *

All methods in this class must be called on the thread that owns the OpenGL context. */ @UnstableApi // TODO(b/227625423): Add GlTextureProcessor interface for async texture processors and make this an @@ -39,42 +39,31 @@ import java.io.IOException; public interface SingleFrameGlTextureProcessor { /** - * Performs all initialization that requires OpenGL, such as, loading and compiling a GLSL shader - * program. + * Configures the texture processor based on the input dimensions. * - *

This method may only be called if there is a current OpenGL context. + *

This method can be called multiple times. * - * @param context The {@link Context}. - * @param inputTexId Identifier of a 2D OpenGL texture. * @param inputWidth The input width, in pixels. * @param inputHeight The input height, in pixels. - * @throws IOException If an error occurs while reading resources. + * @return The output {@link Size} of frames processed through {@link #drawFrame(int, long)}. */ - void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) - throws IOException; - - /** - * Returns the output {@link Size} of frames processed through {@link #drawFrame(long)}. - * - *

This method may only be called after the texture processor has been {@link - * #initialize(Context, int, int, int) initialized}. - */ - Size getOutputSize(); + Size configure(int inputWidth, int inputHeight); /** * Draws one frame. * - *

This method may only be called after the texture processor has been {@link - * #initialize(Context, int, int, int) initialized}. The caller is responsible for focussing the - * correct render target before calling this method. + *

This method may only be called after the texture processor has been {@link #configure(int, + * int) configured}. The caller is responsible for focussing the correct render target before + * calling this method. * *

A minimal implementation should tell OpenGL to use its shader program, bind the shader * program's vertex attributes and uniforms, and issue a drawing command. * + * @param inputTexId Identifier of a 2D OpenGL texture containing the input frame. * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. * @throws FrameProcessingException If an error occurs while processing or drawing the frame. */ - void drawFrame(long presentationTimeUs) throws FrameProcessingException; + void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException; /** Releases all resources. */ void release(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TextureInfo.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TextureInfo.java new file mode 100644 index 0000000000..f81f99d2c0 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TextureInfo.java @@ -0,0 +1,46 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import androidx.media3.common.util.UnstableApi; + +/** Contains information describing an OpenGL texture. */ +@UnstableApi +/* package */ final class TextureInfo { + /** The OpenGL texture identifier. */ + public final int texId; + /** Identifier of a framebuffer object associated with the texture. */ + public final int fboId; + /** The width of the texture, in pixels. */ + public final int width; + /** The height of the texture, in pixels. */ + public final int height; + + /** + * Creates a new instance. + * + * @param texId The OpenGL texture identifier. + * @param fboId Identifier of a framebuffer object associated with the texture. + * @param width The width of the texture, in pixels. + * @param height The height of the texture, in pixels. + */ + public TextureInfo(int texId, int fboId, int width, int height) { + this.texId = texId; + this.fboId = fboId; + this.width = width; + this.height = height; + } +}