loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change isInputTextureExternal boolean parameter to inputType intDef

Browse Source 
PiperOrigin-RevId: 525690361
This commit is contained in:
tofunmi 2023-04-20 10:40:48 +01:00 committed by Rohit Singh
parent 3bc0172188
commit 0f8fa232e7
9 changed files with 95 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java
View File

@ -15,12 +15,19 @@
*/
package androidx.media3.common;
import static java.lang.annotation.ElementType.TYPE_USE;
import android.content.Context;
import android.graphics.Bitmap;
import android.opengl.EGLExt;
import android.view.Surface;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.util.UnstableApi;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.List;
import java.util.concurrent.Executor;
@ -39,6 +46,24 @@ import java.util.concurrent.Executor;
@UnstableApi
public interface VideoFrameProcessor {
// TODO(b/243036513): Allow effects to be replaced.
/**
* Specifies how the input frames are made available to the {@link VideoFrameProcessor}. One of
* {@link #INPUT_TYPE_SURFACE}, {@link #INPUT_TYPE_BITMAP} or {@link #INPUT_TYPE_TEXID}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({INPUT_TYPE_SURFACE, INPUT_TYPE_BITMAP, INPUT_TYPE_TEXID})
public @interface InputType {}
/** Input frames come from a {@link #getInputSurface surface}. */
public static final int INPUT_TYPE_SURFACE = 1;
/** Input frames come from a {@link Bitmap}. */
public static final int INPUT_TYPE_BITMAP = 2;
/**
* Input frames come from a {@linkplain android.opengl.GLES10#GL_TEXTURE_2D traditional GLES
* texture}.
*/
public static final int INPUT_TYPE_TEXID = 3;
/** A factory for {@link VideoFrameProcessor} instances. */
interface Factory {
@ -53,10 +78,7 @@ public interface VideoFrameProcessor {
* @param debugViewProvider A {@link DebugViewProvider}.
* @param inputColorInfo The {@link ColorInfo} for input frames.
* @param outputColorInfo The {@link ColorInfo} for output frames.
* @param isInputTextureExternal Whether the input frames are produced externally (e.g. from a
* video) or not (e.g. from a {@link Bitmap}). See <a
* href="https://source.android.com/docs/core/graphics/arch-st#ext_texture">the
* SurfaceTexture docs</a> for more information on external textures.
* @param inputType The {@link InputType}.
* @param releaseFramesAutomatically If {@code true}, the instance will render output frames to
* the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as
* {@link VideoFrameProcessor} is done processing them. If {@code false}, the {@link
@ -74,7 +96,7 @@ public interface VideoFrameProcessor {
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean isInputTextureExternal,
@InputType int inputType,
boolean releaseFramesAutomatically,
Executor executor,
Listener listener)
@ -129,10 +151,10 @@ public interface VideoFrameProcessor {
long DROP_OUTPUT_FRAME = -2;
/**
* Provides an input {@link Bitmap} to the {@code VideoFrameProcessor}.
* Provides an input {@link Bitmap} to the {@link VideoFrameProcessor}.
*
* <p>This method should only be used for when the {@code VideoFrameProcessor}'s {@code
* isInputTextureExternal} parameter is set to {@code false}.
* <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
* Factory#create created} with {@link #INPUT_TYPE_BITMAP}.
*
* <p>Can be called on any thread.
*
@ -146,11 +168,11 @@ public interface VideoFrameProcessor {
void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate);
/**
* Returns the input {@link Surface}, where {@code VideoFrameProcessor} consumes input frames
* Returns the input {@link Surface}, where {@link VideoFrameProcessor} consumes input frames
* from.
*
* <p>This method should only be used for when the {@code VideoFrameProcessor}'s {@code
* isInputTextureExternal} parameter is set to {@code true}.
* <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
* Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
*
* <p>Can be called on any thread.
*/
@ -175,8 +197,8 @@ public interface VideoFrameProcessor {
*
* <p>Must be called before rendering a frame to the input surface.
*
* <p>This method should only be used for when the {@code VideoFrameProcessor}'s {@code
* isInputTextureExternal} parameter is set to {@code true}.
* <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
* Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
*
* <p>Can be called on any thread.
*
@ -189,8 +211,8 @@ public interface VideoFrameProcessor {
* Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
* but not processed off the {@linkplain #getInputSurface() input surface} yet.
*
* <p>This method should only be used for when the {@code VideoFrameProcessor}'s {@code
* isInputTextureExternal} parameter is set to {@code true}.
* <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
* Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
*
* <p>Can be called on any thread.
*/
@ -249,8 +271,8 @@ public interface VideoFrameProcessor {
* <p>All the frames that are {@linkplain #registerInputFrame() registered} prior to calling this
* method are no longer considered to be registered when this method returns.
*
* <p>This method should only be used for when the {@code VideoFrameProcessor}'s {@code
* isInputTextureExternal} parameter is set to {@code true}.
* <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
* Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
*
* <p>{@link Listener} methods invoked prior to calling this method should be ignored.
*/

3
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
import static com.google.common.truth.Truth.assertThat;
@ -124,7 +125,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
return new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build())
.setIsInputTextureExternal(false)
.setInputType(INPUT_TYPE_BITMAP)
.setOnOutputFrameAvailableListener(
unused -> checkNotNull(framesProduced).incrementAndGet());
}

5
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
@ -110,7 +111,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
public void noEffects_withImageInput_matchesGoldenFile() throws Exception {
String testId = "noEffects_withImageInput_matchesGoldenFile";
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId).setIsInputTextureExternal(false).build();
getDefaultFrameProcessorTestRunnerBuilder(testId).setInputType(INPUT_TYPE_BITMAP).build();
Bitmap originalBitmap = readBitmap(IMAGE_PNG_ASSET_PATH);
Bitmap expectedBitmap = readBitmap(IMAGE_TO_VIDEO_PNG_ASSET_PATH);
@ -129,7 +130,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
String testId = "wrappedCrop_withImageInput_matchesGoldenFile";
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setIsInputTextureExternal(false)
.setInputType(INPUT_TYPE_BITMAP)
.setEffects(
new GlEffectWrapper(
new Crop(

3
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
@ -300,7 +301,7 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest {
DebugViewProvider.NONE,
/* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* isInputTextureExternal= */ true,
INPUT_TYPE_SURFACE,
releaseFramesAutomatically,
MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {

48
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -182,7 +182,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean isInputTextureExternal,
@InputType int inputType,
boolean releaseFramesAutomatically,
Executor listenerExecutor,
Listener listener)
@ -221,7 +221,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
inputColorInfo,
outputColorInfo,
enableColorTransfers,
isInputTextureExternal,
inputType,
releaseFramesAutomatically,
singleThreadExecutorService,
listenerExecutor,
@ -259,7 +259,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private DefaultVideoFrameProcessor(
EGLDisplay eglDisplay,
EGLContext eglContext,
boolean isInputTextureExternal,
@InputType int inputType,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
ImmutableList<GlShaderProgram> shaderPrograms,
boolean releaseFramesAutomatically)
@ -275,16 +275,22 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
GlShaderProgram inputShaderProgram = shaderPrograms.get(0);
if (isInputTextureExternal) {
checkState(inputShaderProgram instanceof ExternalShaderProgram);
inputExternalTextureManager =
new ExternalTextureManager(
(ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor);
inputShaderProgram.setInputListener(inputExternalTextureManager);
} else {
inputBitmapTextureManager =
new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor);
inputShaderProgram.setInputListener(inputBitmapTextureManager);
switch (inputType) {
case VideoFrameProcessor.INPUT_TYPE_SURFACE:
checkState(inputShaderProgram instanceof ExternalShaderProgram);
inputExternalTextureManager =
new ExternalTextureManager(
(ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor);
inputShaderProgram.setInputListener(inputExternalTextureManager);
break;
case VideoFrameProcessor.INPUT_TYPE_BITMAP:
inputBitmapTextureManager =
new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor);
inputShaderProgram.setInputListener(inputBitmapTextureManager);
break;
case VideoFrameProcessor.INPUT_TYPE_TEXID: // fall through
default:
throw new VideoFrameProcessingException("Input type not supported yet");
}
finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(shaderPrograms);
@ -306,8 +312,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* call this method after instantiation to ensure that buffers are handled at full resolution. See
* {@link SurfaceTexture#setDefaultBufferSize(int, int)} for more information.
*
* <p>This method should only be used for when the {@link VideoFrameProcessor}'s {@code
* isInputTextureExternal} parameter is set to {@code true}.
* <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
* Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
*
* @param width The default width for input buffers, in pixels.
* @param height The default height for input buffers, in pixels.
@ -443,7 +449,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean isInputTextureExternal,
@InputType int inputType,
boolean releaseFramesAutomatically,
ExecutorService singleThreadExecutorService,
Executor executor,
@ -487,7 +493,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
inputColorInfo,
outputColorInfo,
enableColorTransfers,
isInputTextureExternal,
inputType,
releaseFramesAutomatically,
executor,
listener,
@ -502,7 +508,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return new DefaultVideoFrameProcessor(
eglDisplay,
eglContext,
isInputTextureExternal,
inputType,
videoFrameProcessingTaskExecutor,
shaderPrograms,
releaseFramesAutomatically);
@ -528,7 +534,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean isInputTextureExternal,
@InputType int inputType,
boolean releaseFramesAutomatically,
Executor executor,
Listener listener,
@ -570,7 +576,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromInputTexture) {
DefaultShaderProgram defaultShaderProgram;
if (sampleFromInputTexture) {
if (isInputTextureExternal) {
if (inputType == INPUT_TYPE_SURFACE) {
defaultShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context,
@ -614,7 +620,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
outputColorInfo,
enableColorTransfers,
sampleFromInputTexture,
isInputTextureExternal,
inputType,
releaseFramesAutomatically,
executor,
listener,

9
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
@ -72,7 +73,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final EGLContext eglContext;
private final DebugViewProvider debugViewProvider;
private final boolean sampleFromInputTexture;
private final boolean isInputTextureExternal;
private final @VideoFrameProcessor.InputType int inputType;
private final ColorInfo inputColorInfo;
private final ColorInfo outputColorInfo;
private final boolean enableColorTransfers;
@ -115,7 +116,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean sampleFromInputTexture,
boolean isInputTextureExternal,
@VideoFrameProcessor.InputType int inputType,
boolean releaseFramesAutomatically,
Executor videoFrameProcessorListenerExecutor,
VideoFrameProcessor.Listener videoFrameProcessorListener,
@ -128,7 +129,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglContext = eglContext;
this.debugViewProvider = debugViewProvider;
this.sampleFromInputTexture = sampleFromInputTexture;
this.isInputTextureExternal = isInputTextureExternal;
this.inputType = inputType;
this.inputColorInfo = inputColorInfo;
this.outputColorInfo = outputColorInfo;
this.enableColorTransfers = enableColorTransfers;
@ -456,7 +457,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ImmutableList<GlMatrixTransformation> expandedMatrixTransformations =
matrixTransformationListBuilder.build();
if (sampleFromInputTexture) {
if (isInputTextureExternal) {
if (inputType == INPUT_TYPE_SURFACE) {
defaultShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context,

3
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -16,6 +16,7 @@
package androidx.media3.exoplayer.video;
import static android.view.Display.DEFAULT_DISPLAY;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
@ -2014,7 +2015,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
DebugViewProvider.NONE,
inputColorInfo,
outputColorInfo,
/* isInputTextureExternal= */ true,
INPUT_TYPE_SURFACE,
/* releaseFramesAutomatically= */ false,
/* executor= */ handler::post,
new VideoFrameProcessor.Listener() {

17
libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.test.utils;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapFromRgba8888Image;
@ -65,13 +66,13 @@ public final class VideoFrameProcessorTestRunner {
private float pixelWidthHeightRatio;
private @MonotonicNonNull ColorInfo inputColorInfo;
private @MonotonicNonNull ColorInfo outputColorInfo;
private boolean isInputTextureExternal;
private @VideoFrameProcessor.InputType int inputType;
private OnOutputFrameAvailableListener onOutputFrameAvailableListener;
/** Creates a new instance with default values. */
public Builder() {
pixelWidthHeightRatio = DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO;
isInputTextureExternal = true;
inputType = INPUT_TYPE_SURFACE;
onOutputFrameAvailableListener = unused -> {};
}
@ -191,11 +192,11 @@ public final class VideoFrameProcessorTestRunner {
* Sets whether input comes from an external texture. See {@link
* VideoFrameProcessor.Factory#create}.
*
* <p>The default value is {@code true}.
* <p>The default value is {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}.
*/
@CanIgnoreReturnValue
public Builder setIsInputTextureExternal(boolean isInputTextureExternal) {
this.isInputTextureExternal = isInputTextureExternal;
public Builder setInputType(@VideoFrameProcessor.InputType int inputType) {
this.inputType = inputType;
return this;
}
@ -225,7 +226,7 @@ public final class VideoFrameProcessorTestRunner {
pixelWidthHeightRatio,
inputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : inputColorInfo,
outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo,
isInputTextureExternal,
inputType,
onOutputFrameAvailableListener);
}
}
@ -257,7 +258,7 @@ public final class VideoFrameProcessorTestRunner {
float pixelWidthHeightRatio,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean isInputTextureExternal,
@VideoFrameProcessor.InputType int inputType,
OnOutputFrameAvailableListener onOutputFrameAvailableListener)
throws VideoFrameProcessingException {
this.testId = testId;
@ -274,7 +275,7 @@ public final class VideoFrameProcessorTestRunner {
DebugViewProvider.NONE,
inputColorInfo,
outputColorInfo,
isInputTextureExternal,
inputType,
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {

7
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
View File

@ -17,6 +17,8 @@
package androidx.media3.transformer;
import static androidx.media3.common.ColorInfo.isTransferHdr;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing;
import static androidx.media3.transformer.TransformationRequest.HDR_MODE_KEEP_HDR;
@ -128,6 +130,9 @@ import org.checkerframework.dataflow.qual.Pure;
if (presentation != null) {
effectsWithPresentation.add(presentation);
}
@VideoFrameProcessor.InputType
int inputType =
MimeTypes.isVideo(firstInputFormat.sampleMimeType) ? INPUT_TYPE_SURFACE : INPUT_TYPE_BITMAP;
try {
videoFrameProcessor =
videoFrameProcessorFactory.create(
@ -136,7 +141,7 @@ import org.checkerframework.dataflow.qual.Pure;
debugViewProvider,
videoFrameProcessorInputColor,
videoFrameProcessorOutputColor,
MimeTypes.isVideo(firstInputFormat.sampleMimeType),
inputType,
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {