loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Effects: Output to texture without surface in VFP.

Browse Source 
Allow the VideoFrameProcessor to output to a texture without an output surface.

Tested by updating texture output tests to no longer output to a surface.

PiperOrigin-RevId: 527244605
This commit is contained in:
huangdarwin 2023-04-26 14:09:33 +01:00 committed by Ian Baker
parent 889f435a49
commit 97272c139c
5 changed files with 95 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java
View File

@ -123,7 +123,8 @@ public interface VideoFrameProcessor {
void onOutputSizeChanged(int width, int height);
/**
* Called when an output frame with the given {@code presentationTimeUs} becomes available.
* Called when an output frame with the given {@code presentationTimeUs} becomes available for
* rendering.
*
* @param presentationTimeUs The presentation time of the frame, in microseconds.
*/

3
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -114,6 +114,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
*
* <p>If set, the {@link VideoFrameProcessor} will output to an OpenGL texture, accessible via
* {@link TextureOutputListener#onTextureRendered}. Otherwise, no texture will be rendered to.
*
* <p>If an {@linkplain #setOutputSurfaceInfo output surface} is set, the texture output will
* be be adjusted as needed, to match the output surface's output.
*/
@VisibleForTesting
@CanIgnoreReturnValue

152
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -48,12 +48,13 @@ import com.google.common.collect.ImmutableList;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Wrapper around a {@link DefaultShaderProgram} that writes to the provided output surface and if
* provided, the optional debug surface view or output texture.
* Wrapper around a {@link DefaultShaderProgram} that renders to the provided output surface or
* texture.
*
* <p>Also renders to a debug surface, if provided.
*
* <p>The wrapped {@link DefaultShaderProgram} applies the {@link GlMatrixTransformation} and {@link
* RgbMatrix} instances passed to the constructor, followed by any transformations needed to convert
@ -92,15 +93,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private InputListener inputListener;
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
@Nullable private SurfaceView debugSurfaceView;
private @MonotonicNonNull GlTextureInfo outputTexture;
@Nullable private GlTextureInfo outputTexture;
private boolean frameProcessingStarted;
private volatile boolean outputChanged;
private volatile boolean outputSurfaceInfoChanged;
@GuardedBy("this")
@Nullable
private SurfaceInfo outputSurfaceInfo;
/** Wraps the {@link Surface} in {@link #outputSurfaceInfo}. */
@GuardedBy("this")
@Nullable
private EGLSurface outputEglSurface;
@ -240,6 +242,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
try {
if (outputTexture != null) {
GlTextureInfo outputTexture = checkNotNull(this.outputTexture);
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
}
@ -270,7 +273,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
this.outputEglSurface = null;
}
outputChanged =
outputSurfaceInfoChanged =
this.outputSurfaceInfo == null
|| outputSurfaceInfo == null
|| this.outputSurfaceInfo.width != outputSurfaceInfo.width
@ -279,14 +282,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.outputSurfaceInfo = outputSurfaceInfo;
}
private void renderFrame(
private synchronized void renderFrame(
GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) {
try {
maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
if (textureOutputListener != null && defaultShaderProgram != null) {
if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
|| !ensureConfigured(inputTexture.width, inputTexture.height)) {
inputListener.onInputFrameProcessed(inputTexture);
return; // Drop frames when requested, or there is no output surface.
}
if (outputSurfaceInfo != null) {
renderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
}
if (textureOutputListener != null) {
renderFrameToOutputTexture(inputTexture, presentationTimeUs);
}
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
videoFrameProcessorListenerExecutor.execute(
() ->
@ -300,17 +309,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
inputListener.onInputFrameProcessed(inputTexture);
}
private synchronized void maybeRenderFrameToOutputSurface(
private synchronized void renderFrameToOutputSurface(
GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs)
throws VideoFrameProcessingException, GlUtil.GlException {
if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
|| !ensureConfigured(inputTexture.width, inputTexture.height)) {
return; // Drop frames when requested, or there is no output surface.
}
EGLSurface outputEglSurface = this.outputEglSurface;
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
DefaultShaderProgram defaultShaderProgram = this.defaultShaderProgram;
EGLSurface outputEglSurface = checkNotNull(this.outputEglSurface);
SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo);
DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram);
GlUtil.focusEglSurface(
eglDisplay,
@ -332,7 +336,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs)
throws GlUtil.GlException, VideoFrameProcessingException {
checkNotNull(outputTexture);
GlTextureInfo outputTexture = checkNotNull(this.outputTexture);
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame();
@ -345,12 +349,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*
* <p>Returns {@code false} if {@code outputSurfaceInfo} is unset.
*/
@EnsuresNonNullIf(
expression = {"outputSurfaceInfo", "outputEglSurface", "defaultShaderProgram"},
result = true)
private synchronized boolean ensureConfigured(int inputWidth, int inputHeight)
throws VideoFrameProcessingException, GlUtil.GlException {
// Clear extra or outdated resources.
boolean inputSizeChanged =
this.inputWidth != inputWidth
|| this.inputHeight != inputHeight
@ -370,20 +371,31 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSizeBeforeSurfaceTransformation.getHeight()));
}
}
checkNotNull(outputSizeBeforeSurfaceTransformation);
if (outputSurfaceInfo == null) {
GlUtil.destroyEglSurface(eglDisplay, outputEglSurface);
outputEglSurface = null;
}
if (outputSurfaceInfo == null && textureOutputListener == null) {
if (defaultShaderProgram != null) {
defaultShaderProgram.release();
defaultShaderProgram = null;
}
GlUtil.destroyEglSurface(eglDisplay, outputEglSurface);
outputEglSurface = null;
return false;
}
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
@Nullable EGLSurface outputEglSurface = this.outputEglSurface;
if (outputEglSurface == null) {
int outputWidth =
outputSurfaceInfo == null
? outputSizeBeforeSurfaceTransformation.getWidth()
: outputSurfaceInfo.width;
int outputHeight =
outputSurfaceInfo == null
? outputSizeBeforeSurfaceTransformation.getHeight()
: outputSurfaceInfo.height;
// Allocate or update resources.
if (outputSurfaceInfo != null && outputEglSurface == null) {
outputEglSurface =
glObjectsProvider.createEglSurface(
eglDisplay,
@ -391,67 +403,58 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputColorInfo.colorTransfer,
// Frames are only released automatically when outputting to an encoder.
/* isEncoderInputSurface= */ releaseFramesAutomatically);
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(
outputSurfaceInfo.width, outputSurfaceInfo.height);
if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
debugSurfaceViewWrapper =
new SurfaceViewWrapper(
eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer);
}
this.debugSurfaceView = debugSurfaceView;
}
if (defaultShaderProgram != null && (outputChanged || inputSizeChanged)) {
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
debugSurfaceViewWrapper =
new SurfaceViewWrapper(
eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer);
}
this.debugSurfaceView = debugSurfaceView;
if (textureOutputListener != null) {
int outputTexId =
GlUtil.createTexture(
outputWidth,
outputHeight,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
outputTexture =
glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
}
if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) {
defaultShaderProgram.release();
defaultShaderProgram = null;
outputChanged = false;
outputSurfaceInfoChanged = false;
}
if (defaultShaderProgram == null) {
DefaultShaderProgram defaultShaderProgram =
createDefaultShaderProgramForOutputSurface(outputSurfaceInfo);
if (textureOutputListener != null) {
configureOutputTexture(
checkNotNull(outputSizeBeforeSurfaceTransformation).getWidth(),
checkNotNull(outputSizeBeforeSurfaceTransformation).getHeight());
}
this.defaultShaderProgram = defaultShaderProgram;
defaultShaderProgram =
createDefaultShaderProgram(
outputSurfaceInfo == null ? 0 : outputSurfaceInfo.orientationDegrees,
outputWidth,
outputHeight);
}
this.outputSurfaceInfo = outputSurfaceInfo;
this.outputEglSurface = outputEglSurface;
return true;
}
private void configureOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
if (outputTexture != null) {
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
}
int outputTexId =
GlUtil.createTexture(
outputWidth,
outputHeight,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
outputTexture =
glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
}
private DefaultShaderProgram createDefaultShaderProgramForOutputSurface(
SurfaceInfo outputSurfaceInfo) throws VideoFrameProcessingException {
private synchronized DefaultShaderProgram createDefaultShaderProgram(
int outputOrientationDegrees, int outputWidth, int outputHeight)
throws VideoFrameProcessingException {
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
new ImmutableList.Builder<GlMatrixTransformation>().addAll(matrixTransformations);
if (outputSurfaceInfo.orientationDegrees != 0) {
if (outputOrientationDegrees != 0) {
matrixTransformationListBuilder.add(
new ScaleAndRotateTransformation.Builder()
.setRotationDegrees(outputSurfaceInfo.orientationDegrees)
.setRotationDegrees(outputOrientationDegrees)
.build());
}
matrixTransformationListBuilder.add(
Presentation.createForWidthAndHeight(
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
outputWidth, outputHeight, Presentation.LAYOUT_SCALE_TO_FIT));
DefaultShaderProgram defaultShaderProgram;
ImmutableList<GlMatrixTransformation> expandedMatrixTransformations =
@ -488,8 +491,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix);
Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
checkState(outputSize.getWidth() == outputSurfaceInfo.width);
checkState(outputSize.getHeight() == outputSurfaceInfo.height);
if (outputSurfaceInfo != null) {
SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo);
checkState(outputSize.getWidth() == outputSurfaceInfo.width);
checkState(outputSize.getHeight() == outputSurfaceInfo.height);
}
return defaultShaderProgram;
}

11
libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java
View File

@ -281,14 +281,17 @@ public final class VideoFrameProcessorTestRunner {
new VideoFrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
@Nullable
Surface outputSurface =
bitmapReader.getSurface(
width,
height,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(
outputColorInfo));
checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
if (outputSurface != null) {
checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
}
}
@Override
@ -368,7 +371,8 @@ public final class VideoFrameProcessorTestRunner {
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
public interface BitmapReader {
/** Returns the {@link VideoFrameProcessor} output {@link Surface}. */
/** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
@Nullable
Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents);
/** Returns the output {@link Bitmap}. */
@ -388,6 +392,7 @@ public final class VideoFrameProcessorTestRunner {
@Override
@SuppressLint("WrongConstant")
@Nullable
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
imageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);

12
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
View File

@ -31,7 +31,6 @@ import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
@ -52,6 +51,7 @@ import androidx.media3.transformer.EncoderUtil;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.junit.After;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -306,16 +306,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
private @MonotonicNonNull Bitmap outputBitmap;
@Override
@Nullable
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
int texId;
try {
texId = GlUtil.createExternalTexture();
} catch (GlUtil.GlException e) {
throw new RuntimeException(e);
}
SurfaceTexture surfaceTexture = new SurfaceTexture(texId);
return new Surface(surfaceTexture);
return null;
}
@Override