loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Effect: Use callback to release texture

Browse Source 
This allows us to avoid needing a reference to the VideoFrameProcessor, which
can be especially difficult if an App only has a reference to the
VideoFrameProcessor.Factory it passes into Transformer/ExoPlayer.

PiperOrigin-RevId: 533205983
This commit is contained in:
huangdarwin 2023-05-18 20:03:27 +01:00 committed by Ian Baker
parent 04106da932
commit 25fa2df2de
6 changed files with 63 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java
View File

@ -49,7 +49,7 @@ public interface VideoFrameProcessor {
/** A listener for frame processing events. */
@UnstableApi
public interface OnInputFrameProcessedListener {
interface OnInputFrameProcessedListener {
/** Called when the given input frame has been processed. */
void onInputFrameProcessed(int textureId) throws VideoFrameProcessingException;
@ -291,18 +291,6 @@ public interface VideoFrameProcessor {
*/
void renderOutputFrame(long renderTimeNs);
/**
* Releases resources associated with all output frames with presentation time less than or equal
* to {@code presentationTimeUs}.
*
* <p>Not needed for outputting to an {@linkplain #setOutputSurfaceInfo output surface}, but may
* be required for other outputs.
*
* @param presentationTimeUs The presentation time where all frames before and at this time should
* be released, in microseconds.
*/
void releaseOutputFrame(long presentationTimeUs);
/**
* Informs the {@code VideoFrameProcessor} that no further input frames should be accepted.
*

42
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -71,11 +71,24 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
/** Listener interface for texture output. */
@VisibleForTesting(otherwise = PACKAGE_PRIVATE)
public interface TextureOutputListener {
/** Called when a texture has been rendered to. */
void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs)
/**
* Called when a texture has been rendered to. {@code releaseOutputTextureCallback} must be
* called to release the {@link GlTextureInfo}.
*/
void onTextureRendered(
GlTextureInfo outputTexture,
long presentationTimeUs,
ReleaseOutputTextureCallback releaseOutputTextureCallback)
throws VideoFrameProcessingException;
}
/**
* Releases the output information stored for textures before and at {@code presentationTimeUs}.
*/
public interface ReleaseOutputTextureCallback {
void release(long presentationTimeUs);
}
/** A factory for {@link DefaultVideoFrameProcessor} instances. */
public static final class Factory implements VideoFrameProcessor.Factory {
@ -118,9 +131,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* Sets texture output settings.
*
* <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
* {@link TextureOutputListener#onTextureRendered}. Textures will stop being output when
* {@code textureOutputCapacity} is reached, until they're released via {@link
* #releaseOutputFrame}. Output textures must be released using {@link #releaseOutputFrame}.
* {@link TextureOutputListener#onTextureRendered}. Textures will stop being outputted when
* the number of output textures available reaches the {@code textureOutputCapacity}. To
* regain capacity, output textures must be released using {@link
* ReleaseOutputTextureCallback}.
*
* <p>If not set, there will be no texture output.
*
@ -455,21 +469,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
() -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs));
}
/**
* {@inheritDoc}
*
* <p>If a {@link TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set},
* this must be called to release the output information stored in the {@link GlTextureInfo}
* instances.
*/
@Override
public void releaseOutputFrame(long presentationTimeUs) {
// TODO(b/262694346): Add Compositor system tests exercising this code path after GL texture
// input is possible.
videoFrameProcessingTaskExecutor.submit(
() -> finalShaderProgramWrapper.releaseOutputFrame(presentationTimeUs));
}
@Override
public void signalEndOfInput() {
DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos();
@ -610,6 +609,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
outputColorInfo,
enableColorTransfers,
renderFramesAutomatically,
videoFrameProcessingTaskExecutor,
executor,
listener,
glObjectsProvider,
@ -661,6 +661,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean renderFramesAutomatically,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor executor,
Listener listener,
GlObjectsProvider glObjectsProvider,
@ -714,6 +715,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
outputColorInfo,
enableColorTransfers,
renderFramesAutomatically,
videoFrameProcessingTaskExecutor,
executor,
listener,
glObjectsProvider,

30
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -85,6 +85,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final ColorInfo outputColorInfo;
private final boolean enableColorTransfers;
private final boolean renderFramesAutomatically;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final Executor videoFrameProcessorListenerExecutor;
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
@ -126,6 +127,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean renderFramesAutomatically,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor videoFrameProcessorListenerExecutor,
VideoFrameProcessor.Listener videoFrameProcessorListener,
GlObjectsProvider glObjectsProvider,
@ -140,6 +142,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.outputColorInfo = outputColorInfo;
this.enableColorTransfers = enableColorTransfers;
this.renderFramesAutomatically = renderFramesAutomatically;
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
this.videoFrameProcessorListener = videoFrameProcessorListener;
this.glObjectsProvider = glObjectsProvider;
@ -224,6 +227,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
public void releaseOutputFrame(long presentationTimeUs) {
videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs));
}
private void releaseOutputFrameInternal(long presentationTimeUs) {
while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
&& checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) {
outputTexturePool.freeTexture();
@ -232,16 +239,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
public void renderOutputFrame(long renderTimeNs) {
frameProcessingStarted = true;
checkState(!renderFramesAutomatically);
Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
renderFrame(
/* inputTexture= */ oldestAvailableFrame.first,
/* presentationTimeUs= */ oldestAvailableFrame.second,
renderTimeNs);
}
@Override
public void flush() {
frameProcessingStarted = true;
@ -267,6 +264,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
public void renderOutputFrame(long renderTimeNs) {
frameProcessingStarted = true;
checkState(!renderFramesAutomatically);
Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
renderFrame(
/* inputTexture= */ oldestAvailableFrame.first,
/* presentationTimeUs= */ oldestAvailableFrame.second,
renderTimeNs);
}
/**
* Sets the output {@link SurfaceInfo}.
*
@ -369,7 +376,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// glFinish. Consider removing glFinish and requiring onTextureRendered to handle
// synchronization.
GLES20.glFinish();
checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs);
checkNotNull(textureOutputListener)
.onTextureRendered(outputTexture, presentationTimeUs, this::releaseOutputFrame);
}
/**

6
libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java
View File

@ -109,6 +109,8 @@ import java.util.Queue;
* <p>Throws {@link IllegalStateException} if {@code textureInfo} isn't in use.
*/
public void freeTexture(GlTextureInfo textureInfo) {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
checkState(inUseTextures.contains(textureInfo));
inUseTextures.remove(textureInfo);
freeTextures.add(textureInfo);
@ -120,6 +122,8 @@ import java.util.Queue;
* <p>Throws {@link IllegalStateException} if there's no textures in use to free.
*/
public void freeTexture() {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
checkState(!inUseTextures.isEmpty());
GlTextureInfo texture = inUseTextures.remove();
freeTextures.add(texture);
@ -127,6 +131,8 @@ import java.util.Queue;
/** Free all in-use textures. */
public void freeAllTextures() {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
freeTextures.addAll(inUseTextures);
inUseTextures.clear();
}

22
libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java
View File

@ -288,11 +288,7 @@ public final class VideoFrameProcessorTestRunner {
boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
@Nullable
Surface outputSurface =
bitmapReader.getSurface(
width,
height,
useHighPrecisionColorComponents,
checkNotNull(videoFrameProcessor)::releaseOutputFrame);
bitmapReader.getSurface(width, height, useHighPrecisionColorComponents);
if (outputSurface != null) {
checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
@ -407,18 +403,10 @@ public final class VideoFrameProcessorTestRunner {
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
public interface BitmapReader {
/** Wraps a callback for {@link VideoFrameProcessor#releaseOutputFrame}. */
interface ReleaseOutputFrameListener {
void releaseOutputFrame(long releaseTimeUs);
}
/** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
@Nullable
Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener);
Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents);
/** Returns the output {@link Bitmap}. */
Bitmap getBitmap();
@ -438,11 +426,7 @@ public final class VideoFrameProcessorTestRunner {
@Override
@SuppressLint("WrongConstant")
@Nullable
public Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener) {
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
imageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
return imageReader.getSurface();

27
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
View File

@ -507,14 +507,15 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setTextureOutput(
(outputTexture, presentationTimeUs) ->
(outputTexture, presentationTimeUs, releaseOutputTextureCallback) ->
inputTextureIntoVideoFrameProcessor(
testId,
consumersBitmapReader,
colorInfo,
effects,
outputTexture,
presentationTimeUs),
presentationTimeUs,
releaseOutputTextureCallback),
/* textureOutputCapacity= */ 1)
.build();
return new VideoFrameProcessorTestRunner.Builder()
@ -533,7 +534,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
ColorInfo colorInfo,
List<Effect> effects,
GlTextureInfo texture,
long presentationTimeUs)
long presentationTimeUs,
DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback)
throws VideoFrameProcessingException {
GlObjectsProvider contextSharingGlObjectsProvider =
new DefaultGlObjectsProvider(GlUtil.getCurrentContext());
@ -559,6 +561,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
} catch (InterruptedException e) {
throw new VideoFrameProcessingException(e);
}
releaseOutputTextureCallback.release(presentationTimeUs);
}
private VideoFrameProcessorTestRunner.Builder getSurfaceInputFrameProcessorTestRunnerBuilder(
@ -584,19 +587,12 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
private static final class TextureBitmapReader implements BitmapReader {
// TODO(b/239172735): This outputs an incorrect black output image on emulators.
private boolean useHighPrecisionColorComponents;
private @MonotonicNonNull ReleaseOutputFrameListener releaseOutputFrameListener;
private @MonotonicNonNull Bitmap outputBitmap;
@Nullable
@Override
public Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener releaseOutputFrameListener) {
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
this.releaseOutputFrameListener = releaseOutputFrameListener;
return null;
}
@ -605,7 +601,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
return checkStateNotNull(outputBitmap);
}
public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs)
public void readBitmapFromTexture(
GlTextureInfo outputTexture,
long presentationTimeUs,
DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback)
throws VideoFrameProcessingException {
try {
GlUtil.focusFramebufferUsingCurrentContext(
@ -613,12 +612,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
outputBitmap =
createBitmapFromCurrentGlFrameBuffer(
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}
checkNotNull(releaseOutputFrameListener).releaseOutputFrame(presentationTimeUs);
releaseOutputTextureCallback.release(presentationTimeUs);
}
private static Bitmap createBitmapFromCurrentGlFrameBuffer(