diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index f544ebf064..a6157ce66f 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -15,12 +15,19 @@ */ package androidx.media3.common; +import static java.lang.annotation.ElementType.TYPE_USE; + import android.content.Context; import android.graphics.Bitmap; import android.opengl.EGLExt; import android.view.Surface; +import androidx.annotation.IntDef; import androidx.annotation.Nullable; import androidx.media3.common.util.UnstableApi; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.util.List; import java.util.concurrent.Executor; @@ -39,6 +46,24 @@ import java.util.concurrent.Executor; @UnstableApi public interface VideoFrameProcessor { // TODO(b/243036513): Allow effects to be replaced. + /** + * Specifies how the input frames are made available to the {@link VideoFrameProcessor}. One of + * {@link #INPUT_TYPE_SURFACE}, {@link #INPUT_TYPE_BITMAP} or {@link #INPUT_TYPE_TEXID}. + */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef({INPUT_TYPE_SURFACE, INPUT_TYPE_BITMAP, INPUT_TYPE_TEXID}) + public @interface InputType {} + /** Input frames come from a {@link #getInputSurface surface}. */ + public static final int INPUT_TYPE_SURFACE = 1; + /** Input frames come from a {@link Bitmap}. */ + public static final int INPUT_TYPE_BITMAP = 2; + /** + * Input frames come from a {@linkplain android.opengl.GLES10#GL_TEXTURE_2D traditional GLES + * texture}. + */ + public static final int INPUT_TYPE_TEXID = 3; /** A factory for {@link VideoFrameProcessor} instances. */ interface Factory { @@ -53,10 +78,7 @@ public interface VideoFrameProcessor { * @param debugViewProvider A {@link DebugViewProvider}. * @param inputColorInfo The {@link ColorInfo} for input frames. * @param outputColorInfo The {@link ColorInfo} for output frames. - * @param isInputTextureExternal Whether the input frames are produced externally (e.g. from a - * video) or not (e.g. from a {@link Bitmap}). See the - * SurfaceTexture docs for more information on external textures. + * @param inputType The {@link InputType}. * @param releaseFramesAutomatically If {@code true}, the instance will render output frames to * the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as * {@link VideoFrameProcessor} is done processing them. If {@code false}, the {@link @@ -74,7 +96,7 @@ public interface VideoFrameProcessor { DebugViewProvider debugViewProvider, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - boolean isInputTextureExternal, + @InputType int inputType, boolean releaseFramesAutomatically, Executor executor, Listener listener) @@ -129,10 +151,10 @@ public interface VideoFrameProcessor { long DROP_OUTPUT_FRAME = -2; /** - * Provides an input {@link Bitmap} to the {@code VideoFrameProcessor}. + * Provides an input {@link Bitmap} to the {@link VideoFrameProcessor}. * - *
This method should only be used for when the {@code VideoFrameProcessor}'s {@code - * isInputTextureExternal} parameter is set to {@code false}. + *
This method must only be called when the {@link VideoFrameProcessor} is {@linkplain + * Factory#create created} with {@link #INPUT_TYPE_BITMAP}. * *
Can be called on any thread. * @@ -146,11 +168,11 @@ public interface VideoFrameProcessor { void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate); /** - * Returns the input {@link Surface}, where {@code VideoFrameProcessor} consumes input frames + * Returns the input {@link Surface}, where {@link VideoFrameProcessor} consumes input frames * from. * - *
This method should only be used for when the {@code VideoFrameProcessor}'s {@code - * isInputTextureExternal} parameter is set to {@code true}. + *
This method must only be called when the {@link VideoFrameProcessor} is {@linkplain + * Factory#create created} with {@link #INPUT_TYPE_SURFACE}. * *
Can be called on any thread. */ @@ -175,8 +197,8 @@ public interface VideoFrameProcessor { * *
Must be called before rendering a frame to the input surface. * - *
This method should only be used for when the {@code VideoFrameProcessor}'s {@code - * isInputTextureExternal} parameter is set to {@code true}. + *
This method must only be called when the {@link VideoFrameProcessor} is {@linkplain + * Factory#create created} with {@link #INPUT_TYPE_SURFACE}. * *
Can be called on any thread. * @@ -189,8 +211,8 @@ public interface VideoFrameProcessor { * Returns the number of input frames that have been {@linkplain #registerInputFrame() registered} * but not processed off the {@linkplain #getInputSurface() input surface} yet. * - *
This method should only be used for when the {@code VideoFrameProcessor}'s {@code - * isInputTextureExternal} parameter is set to {@code true}. + *
This method must only be called when the {@link VideoFrameProcessor} is {@linkplain + * Factory#create created} with {@link #INPUT_TYPE_SURFACE}. * *
Can be called on any thread. */ @@ -249,8 +271,8 @@ public interface VideoFrameProcessor { *
All the frames that are {@linkplain #registerInputFrame() registered} prior to calling this * method are no longer considered to be registered when this method returns. * - *
This method should only be used for when the {@code VideoFrameProcessor}'s {@code - * isInputTextureExternal} parameter is set to {@code true}. + *
This method must only be called when the {@link VideoFrameProcessor} is {@linkplain + * Factory#create created} with {@link #INPUT_TYPE_SURFACE}. * *
{@link Listener} methods invoked prior to calling this method should be ignored.
*/
diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java
index f527da2d45..718ec4b6b6 100644
--- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java
+++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java
@@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
+import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
import static com.google.common.truth.Truth.assertThat;
@@ -124,7 +125,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
return new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build())
- .setIsInputTextureExternal(false)
+ .setInputType(INPUT_TYPE_BITMAP)
.setOnOutputFrameAvailableListener(
unused -> checkNotNull(framesProduced).incrementAndGet());
}
diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java
index ade54bd5ec..b98f93deb1 100644
--- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java
+++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java
@@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
+import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
@@ -110,7 +111,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
public void noEffects_withImageInput_matchesGoldenFile() throws Exception {
String testId = "noEffects_withImageInput_matchesGoldenFile";
videoFrameProcessorTestRunner =
- getDefaultFrameProcessorTestRunnerBuilder(testId).setIsInputTextureExternal(false).build();
+ getDefaultFrameProcessorTestRunnerBuilder(testId).setInputType(INPUT_TYPE_BITMAP).build();
Bitmap originalBitmap = readBitmap(IMAGE_PNG_ASSET_PATH);
Bitmap expectedBitmap = readBitmap(IMAGE_TO_VIDEO_PNG_ASSET_PATH);
@@ -129,7 +130,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
String testId = "wrappedCrop_withImageInput_matchesGoldenFile";
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
- .setIsInputTextureExternal(false)
+ .setInputType(INPUT_TYPE_BITMAP)
.setEffects(
new GlEffectWrapper(
new Crop(
diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java
index 66b6d39de0..1c9443e281 100644
--- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java
+++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java
@@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
+import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
@@ -300,7 +301,7 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest {
DebugViewProvider.NONE,
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
- /* isInputTextureExternal= */ true,
+ INPUT_TYPE_SURFACE,
releaseFramesAutomatically,
MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
index 979c69bf69..1b4a5918d6 100644
--- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
@@ -182,7 +182,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
- boolean isInputTextureExternal,
+ @InputType int inputType,
boolean releaseFramesAutomatically,
Executor listenerExecutor,
Listener listener)
@@ -221,7 +221,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
inputColorInfo,
outputColorInfo,
enableColorTransfers,
- isInputTextureExternal,
+ inputType,
releaseFramesAutomatically,
singleThreadExecutorService,
listenerExecutor,
@@ -259,7 +259,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private DefaultVideoFrameProcessor(
EGLDisplay eglDisplay,
EGLContext eglContext,
- boolean isInputTextureExternal,
+ @InputType int inputType,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
ImmutableList This method should only be used for when the {@link VideoFrameProcessor}'s {@code
- * isInputTextureExternal} parameter is set to {@code true}.
+ * This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
+ * Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
*
* @param width The default width for input buffers, in pixels.
* @param height The default height for input buffers, in pixels.
@@ -443,7 +449,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
- boolean isInputTextureExternal,
+ @InputType int inputType,
boolean releaseFramesAutomatically,
ExecutorService singleThreadExecutorService,
Executor executor,
@@ -487,7 +493,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
inputColorInfo,
outputColorInfo,
enableColorTransfers,
- isInputTextureExternal,
+ inputType,
releaseFramesAutomatically,
executor,
listener,
@@ -502,7 +508,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return new DefaultVideoFrameProcessor(
eglDisplay,
eglContext,
- isInputTextureExternal,
+ inputType,
videoFrameProcessingTaskExecutor,
shaderPrograms,
releaseFramesAutomatically);
@@ -528,7 +534,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
- boolean isInputTextureExternal,
+ @InputType int inputType,
boolean releaseFramesAutomatically,
Executor executor,
Listener listener,
@@ -570,7 +576,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromInputTexture) {
DefaultShaderProgram defaultShaderProgram;
if (sampleFromInputTexture) {
- if (isInputTextureExternal) {
+ if (inputType == INPUT_TYPE_SURFACE) {
defaultShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context,
@@ -614,7 +620,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
outputColorInfo,
enableColorTransfers,
sampleFromInputTexture,
- isInputTextureExternal,
+ inputType,
releaseFramesAutomatically,
executor,
listener,
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
index cbd5b9bc86..4ae6ae502b 100644
--- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
@@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
+import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
@@ -72,7 +73,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final EGLContext eglContext;
private final DebugViewProvider debugViewProvider;
private final boolean sampleFromInputTexture;
- private final boolean isInputTextureExternal;
+ private final @VideoFrameProcessor.InputType int inputType;
private final ColorInfo inputColorInfo;
private final ColorInfo outputColorInfo;
private final boolean enableColorTransfers;
@@ -115,7 +116,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean sampleFromInputTexture,
- boolean isInputTextureExternal,
+ @VideoFrameProcessor.InputType int inputType,
boolean releaseFramesAutomatically,
Executor videoFrameProcessorListenerExecutor,
VideoFrameProcessor.Listener videoFrameProcessorListener,
@@ -128,7 +129,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglContext = eglContext;
this.debugViewProvider = debugViewProvider;
this.sampleFromInputTexture = sampleFromInputTexture;
- this.isInputTextureExternal = isInputTextureExternal;
+ this.inputType = inputType;
this.inputColorInfo = inputColorInfo;
this.outputColorInfo = outputColorInfo;
this.enableColorTransfers = enableColorTransfers;
@@ -456,7 +457,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ImmutableList The default value is {@code true}.
+ * The default value is {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}.
*/
@CanIgnoreReturnValue
- public Builder setIsInputTextureExternal(boolean isInputTextureExternal) {
- this.isInputTextureExternal = isInputTextureExternal;
+ public Builder setInputType(@VideoFrameProcessor.InputType int inputType) {
+ this.inputType = inputType;
return this;
}
@@ -225,7 +226,7 @@ public final class VideoFrameProcessorTestRunner {
pixelWidthHeightRatio,
inputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : inputColorInfo,
outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo,
- isInputTextureExternal,
+ inputType,
onOutputFrameAvailableListener);
}
}
@@ -257,7 +258,7 @@ public final class VideoFrameProcessorTestRunner {
float pixelWidthHeightRatio,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
- boolean isInputTextureExternal,
+ @VideoFrameProcessor.InputType int inputType,
OnOutputFrameAvailableListener onOutputFrameAvailableListener)
throws VideoFrameProcessingException {
this.testId = testId;
@@ -274,7 +275,7 @@ public final class VideoFrameProcessorTestRunner {
DebugViewProvider.NONE,
inputColorInfo,
outputColorInfo,
- isInputTextureExternal,
+ inputType,
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
index 7842a5a0c2..b48c85d6df 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
@@ -17,6 +17,8 @@
package androidx.media3.transformer;
import static androidx.media3.common.ColorInfo.isTransferHdr;
+import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP;
+import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing;
import static androidx.media3.transformer.TransformationRequest.HDR_MODE_KEEP_HDR;
@@ -128,6 +130,9 @@ import org.checkerframework.dataflow.qual.Pure;
if (presentation != null) {
effectsWithPresentation.add(presentation);
}
+ @VideoFrameProcessor.InputType
+ int inputType =
+ MimeTypes.isVideo(firstInputFormat.sampleMimeType) ? INPUT_TYPE_SURFACE : INPUT_TYPE_BITMAP;
try {
videoFrameProcessor =
videoFrameProcessorFactory.create(
@@ -136,7 +141,7 @@ import org.checkerframework.dataflow.qual.Pure;
debugViewProvider,
videoFrameProcessorInputColor,
videoFrameProcessorOutputColor,
- MimeTypes.isVideo(firstInputFormat.sampleMimeType),
+ inputType,
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {