Remove the textureReleaseCallback from textureOutputListerner
PiperOrigin-RevId: 559817280
This commit is contained in:
parent
d5f8487fe2
commit
feae0245b9
@ -162,10 +162,10 @@ public class FrameDropTest {
|
|||||||
checkNotNull(
|
checkNotNull(
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setTextureOutput(
|
.setTextureOutput(
|
||||||
(outputTexture, presentationTimeUs, releaseOutputTextureCallback, token) -> {
|
(textureProducer, outputTexture, presentationTimeUs, token) -> {
|
||||||
checkNotNull(textureBitmapReader)
|
checkNotNull(textureBitmapReader)
|
||||||
.readBitmap(outputTexture, presentationTimeUs);
|
.readBitmap(outputTexture, presentationTimeUs);
|
||||||
releaseOutputTextureCallback.release(presentationTimeUs);
|
textureProducer.releaseOutputTexture(presentationTimeUs);
|
||||||
},
|
},
|
||||||
/* textureOutputCapacity= */ 1)
|
/* textureOutputCapacity= */ 1)
|
||||||
.build()
|
.build()
|
||||||
|
@ -70,8 +70,8 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
private static final int PRIMARY_INPUT_ID = 0;
|
private static final int PRIMARY_INPUT_ID = 0;
|
||||||
|
|
||||||
private final Context context;
|
private final Context context;
|
||||||
private final Listener listener;
|
private final VideoCompositor.Listener listener;
|
||||||
private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
|
private final GlTextureProducer.Listener textureOutputListener;
|
||||||
private final GlObjectsProvider glObjectsProvider;
|
private final GlObjectsProvider glObjectsProvider;
|
||||||
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
||||||
|
|
||||||
@ -101,8 +101,8 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
Context context,
|
Context context,
|
||||||
GlObjectsProvider glObjectsProvider,
|
GlObjectsProvider glObjectsProvider,
|
||||||
@Nullable ExecutorService executorService,
|
@Nullable ExecutorService executorService,
|
||||||
Listener listener,
|
VideoCompositor.Listener listener,
|
||||||
DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener,
|
GlTextureProducer.Listener textureOutputListener,
|
||||||
@IntRange(from = 1) int textureOutputCapacity) {
|
@IntRange(from = 1) int textureOutputCapacity) {
|
||||||
this.context = context;
|
this.context = context;
|
||||||
this.listener = listener;
|
this.listener = listener;
|
||||||
@ -129,8 +129,8 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*
|
*
|
||||||
* <p>The input source must be able to have at least two {@linkplain #queueInputTexture queued
|
* <p>The input source must be able to have at least two {@linkplain
|
||||||
* textures} before one texture is {@linkplain
|
* VideoCompositor#queueInputTexture queued textures} before one texture is {@linkplain
|
||||||
* DefaultVideoFrameProcessor.ReleaseOutputTextureCallback released}.
|
* DefaultVideoFrameProcessor.ReleaseOutputTextureCallback released}.
|
||||||
*
|
*
|
||||||
* <p>When composited, textures are drawn in the reverse order of their registration order, so
|
* <p>When composited, textures are drawn in the reverse order of their registration order, so
|
||||||
@ -173,14 +173,14 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
@Override
|
@Override
|
||||||
public synchronized void queueInputTexture(
|
public synchronized void queueInputTexture(
|
||||||
int inputId,
|
int inputId,
|
||||||
|
GlTextureProducer textureProducer,
|
||||||
GlTextureInfo inputTexture,
|
GlTextureInfo inputTexture,
|
||||||
long presentationTimeUs,
|
long presentationTimeUs) {
|
||||||
DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseTextureCallback) {
|
|
||||||
InputSource inputSource = inputSources.get(inputId);
|
InputSource inputSource = inputSources.get(inputId);
|
||||||
checkState(!inputSource.isInputEnded);
|
checkState(!inputSource.isInputEnded);
|
||||||
|
|
||||||
InputFrameInfo inputFrameInfo =
|
InputFrameInfo inputFrameInfo =
|
||||||
new InputFrameInfo(inputTexture, presentationTimeUs, releaseTextureCallback);
|
new InputFrameInfo(textureProducer, inputTexture, presentationTimeUs);
|
||||||
inputSource.frameInfos.add(inputFrameInfo);
|
inputSource.frameInfos.add(inputFrameInfo);
|
||||||
|
|
||||||
if (inputId == PRIMARY_INPUT_ID) {
|
if (inputId == PRIMARY_INPUT_ID) {
|
||||||
@ -202,6 +202,11 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void releaseOutputTexture(long presentationTimeUs) {
|
||||||
|
videoFrameProcessingTaskExecutor.submit(() -> releaseOutputTextureInternal(presentationTimeUs));
|
||||||
|
}
|
||||||
|
|
||||||
private synchronized void releaseExcessFramesInAllSecondaryStreams() {
|
private synchronized void releaseExcessFramesInAllSecondaryStreams() {
|
||||||
for (int i = 0; i < inputSources.size(); i++) {
|
for (int i = 0; i < inputSources.size(); i++) {
|
||||||
if (i == PRIMARY_INPUT_ID) {
|
if (i == PRIMARY_INPUT_ID) {
|
||||||
@ -248,7 +253,8 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
private synchronized void releaseFrames(InputSource inputSource, int numberOfFramesToRelease) {
|
private synchronized void releaseFrames(InputSource inputSource, int numberOfFramesToRelease) {
|
||||||
for (int i = 0; i < numberOfFramesToRelease; i++) {
|
for (int i = 0; i < numberOfFramesToRelease; i++) {
|
||||||
InputFrameInfo frameInfoToRelease = inputSource.frameInfos.remove();
|
InputFrameInfo frameInfoToRelease = inputSource.frameInfos.remove();
|
||||||
frameInfoToRelease.releaseCallback.release(frameInfoToRelease.presentationTimeUs);
|
frameInfoToRelease.textureProducer.releaseOutputTexture(
|
||||||
|
frameInfoToRelease.presentationTimeUs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -290,10 +296,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
long syncObject = GlUtil.createGlSyncFence();
|
long syncObject = GlUtil.createGlSyncFence();
|
||||||
syncObjects.add(syncObject);
|
syncObjects.add(syncObject);
|
||||||
textureOutputListener.onTextureRendered(
|
textureOutputListener.onTextureRendered(
|
||||||
outputTexture,
|
/* textureProducer= */ this, outputTexture, outputPresentationTimestampUs, syncObject);
|
||||||
/* presentationTimeUs= */ outputPresentationTimestampUs,
|
|
||||||
this::releaseOutputFrame,
|
|
||||||
syncObject);
|
|
||||||
|
|
||||||
InputSource primaryInputSource = inputSources.get(PRIMARY_INPUT_ID);
|
InputSource primaryInputSource = inputSources.get(PRIMARY_INPUT_ID);
|
||||||
releaseFrames(primaryInputSource, /* numberOfFramesToRelease= */ 1);
|
releaseFrames(primaryInputSource, /* numberOfFramesToRelease= */ 1);
|
||||||
@ -368,11 +371,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
return framesToCompositeList;
|
return framesToCompositeList;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void releaseOutputFrame(long presentationTimeUs) {
|
private synchronized void releaseOutputTextureInternal(long presentationTimeUs)
|
||||||
videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs));
|
|
||||||
}
|
|
||||||
|
|
||||||
private synchronized void releaseOutputFrameInternal(long presentationTimeUs)
|
|
||||||
throws VideoFrameProcessingException, GlUtil.GlException {
|
throws VideoFrameProcessingException, GlUtil.GlException {
|
||||||
while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
|
while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
|
||||||
&& outputTextureTimestamps.element() <= presentationTimeUs) {
|
&& outputTextureTimestamps.element() <= presentationTimeUs) {
|
||||||
@ -478,17 +477,15 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
|||||||
|
|
||||||
/** Holds information on a frame and how to release it. */
|
/** Holds information on a frame and how to release it. */
|
||||||
private static final class InputFrameInfo {
|
private static final class InputFrameInfo {
|
||||||
|
public final GlTextureProducer textureProducer;
|
||||||
public final GlTextureInfo texture;
|
public final GlTextureInfo texture;
|
||||||
public final long presentationTimeUs;
|
public final long presentationTimeUs;
|
||||||
public final DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseCallback;
|
|
||||||
|
|
||||||
public InputFrameInfo(
|
public InputFrameInfo(
|
||||||
GlTextureInfo texture,
|
GlTextureProducer textureProducer, GlTextureInfo texture, long presentationTimeUs) {
|
||||||
long presentationTimeUs,
|
this.textureProducer = textureProducer;
|
||||||
DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseCallback) {
|
|
||||||
this.texture = texture;
|
this.texture = texture;
|
||||||
this.presentationTimeUs = presentationTimeUs;
|
this.presentationTimeUs = presentationTimeUs;
|
||||||
this.releaseCallback = releaseCallback;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -44,7 +44,6 @@ import androidx.media3.common.DebugViewProvider;
|
|||||||
import androidx.media3.common.Effect;
|
import androidx.media3.common.Effect;
|
||||||
import androidx.media3.common.FrameInfo;
|
import androidx.media3.common.FrameInfo;
|
||||||
import androidx.media3.common.GlObjectsProvider;
|
import androidx.media3.common.GlObjectsProvider;
|
||||||
import androidx.media3.common.GlTextureInfo;
|
|
||||||
import androidx.media3.common.OnInputFrameProcessedListener;
|
import androidx.media3.common.OnInputFrameProcessedListener;
|
||||||
import androidx.media3.common.SurfaceInfo;
|
import androidx.media3.common.SurfaceInfo;
|
||||||
import androidx.media3.common.VideoFrameProcessingException;
|
import androidx.media3.common.VideoFrameProcessingException;
|
||||||
@ -74,28 +73,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
@UnstableApi
|
@UnstableApi
|
||||||
public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
|
|
||||||
/** Listener interface for texture output. */
|
|
||||||
@VisibleForTesting(otherwise = PACKAGE_PRIVATE)
|
|
||||||
public interface TextureOutputListener {
|
|
||||||
/**
|
|
||||||
* Called when a texture has been rendered to.
|
|
||||||
*
|
|
||||||
* @param outputTexture The texture that has been rendered.
|
|
||||||
* @param presentationTimeUs The presentation time of the texture.
|
|
||||||
* @param releaseOutputTextureCallback A {@link ReleaseOutputTextureCallback} that must be
|
|
||||||
* called to release the {@link GlTextureInfo}.
|
|
||||||
* @param syncObject A GL sync object that has been inserted into the GL command stream after
|
|
||||||
* the last write of the {@code outputTexture}. Value is 0 if and only if the {@link
|
|
||||||
* GLES30#glFenceSync} failed.
|
|
||||||
*/
|
|
||||||
void onTextureRendered(
|
|
||||||
GlTextureInfo outputTexture,
|
|
||||||
long presentationTimeUs,
|
|
||||||
ReleaseOutputTextureCallback releaseOutputTextureCallback,
|
|
||||||
long syncObject)
|
|
||||||
throws VideoFrameProcessingException, GlUtil.GlException;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Releases the output information stored for textures before and at {@code presentationTimeUs}.
|
* Releases the output information stored for textures before and at {@code presentationTimeUs}.
|
||||||
*/
|
*/
|
||||||
@ -112,7 +89,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
private boolean enableColorTransfers;
|
private boolean enableColorTransfers;
|
||||||
private @MonotonicNonNull GlObjectsProvider glObjectsProvider;
|
private @MonotonicNonNull GlObjectsProvider glObjectsProvider;
|
||||||
private @MonotonicNonNull ExecutorService executorService;
|
private @MonotonicNonNull ExecutorService executorService;
|
||||||
private @MonotonicNonNull TextureOutputListener textureOutputListener;
|
private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener;
|
||||||
private int textureOutputCapacity;
|
private int textureOutputCapacity;
|
||||||
|
|
||||||
/** Creates an instance. */
|
/** Creates an instance. */
|
||||||
@ -165,8 +142,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
* Sets texture output settings.
|
* Sets texture output settings.
|
||||||
*
|
*
|
||||||
* <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
|
* <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
|
||||||
* {@link TextureOutputListener#onTextureRendered}. Textures will stop being outputted when
|
* {@link GlTextureProducer.Listener#onTextureRendered}. Textures will stop being outputted
|
||||||
* the number of output textures available reaches the {@code textureOutputCapacity}. To
|
* when the number of output textures available reaches the {@code textureOutputCapacity}. To
|
||||||
* regain capacity, output textures must be released using {@link
|
* regain capacity, output textures must be released using {@link
|
||||||
* ReleaseOutputTextureCallback}.
|
* ReleaseOutputTextureCallback}.
|
||||||
*
|
*
|
||||||
@ -175,14 +152,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
*
|
*
|
||||||
* <p>If not set, there will be no texture output.
|
* <p>If not set, there will be no texture output.
|
||||||
*
|
*
|
||||||
* @param textureOutputListener The {@link TextureOutputListener}.
|
* @param textureOutputListener The {@link GlTextureProducer.Listener}.
|
||||||
* @param textureOutputCapacity The amount of output textures that may be allocated at a time
|
* @param textureOutputCapacity The amount of output textures that may be allocated at a time
|
||||||
* before texture output blocks. Must be greater than or equal to 1.
|
* before texture output blocks. Must be greater than or equal to 1.
|
||||||
*/
|
*/
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
@CanIgnoreReturnValue
|
@CanIgnoreReturnValue
|
||||||
public Builder setTextureOutput(
|
public Builder setTextureOutput(
|
||||||
TextureOutputListener textureOutputListener,
|
GlTextureProducer.Listener textureOutputListener,
|
||||||
@IntRange(from = 1) int textureOutputCapacity) {
|
@IntRange(from = 1) int textureOutputCapacity) {
|
||||||
this.textureOutputListener = textureOutputListener;
|
this.textureOutputListener = textureOutputListener;
|
||||||
checkArgument(textureOutputCapacity >= 1);
|
checkArgument(textureOutputCapacity >= 1);
|
||||||
@ -204,14 +181,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
private final boolean enableColorTransfers;
|
private final boolean enableColorTransfers;
|
||||||
private final GlObjectsProvider glObjectsProvider;
|
private final GlObjectsProvider glObjectsProvider;
|
||||||
@Nullable private final ExecutorService executorService;
|
@Nullable private final ExecutorService executorService;
|
||||||
@Nullable private final TextureOutputListener textureOutputListener;
|
@Nullable private final GlTextureProducer.Listener textureOutputListener;
|
||||||
private final int textureOutputCapacity;
|
private final int textureOutputCapacity;
|
||||||
|
|
||||||
private Factory(
|
private Factory(
|
||||||
boolean enableColorTransfers,
|
boolean enableColorTransfers,
|
||||||
GlObjectsProvider glObjectsProvider,
|
GlObjectsProvider glObjectsProvider,
|
||||||
@Nullable ExecutorService executorService,
|
@Nullable ExecutorService executorService,
|
||||||
@Nullable TextureOutputListener textureOutputListener,
|
@Nullable GlTextureProducer.Listener textureOutputListener,
|
||||||
int textureOutputCapacity) {
|
int textureOutputCapacity) {
|
||||||
this.enableColorTransfers = enableColorTransfers;
|
this.enableColorTransfers = enableColorTransfers;
|
||||||
this.glObjectsProvider = glObjectsProvider;
|
this.glObjectsProvider = glObjectsProvider;
|
||||||
@ -262,7 +239,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
ColorInfo outputColorInfo,
|
ColorInfo outputColorInfo,
|
||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
Executor listenerExecutor,
|
Executor listenerExecutor,
|
||||||
Listener listener)
|
VideoFrameProcessor.Listener listener)
|
||||||
throws VideoFrameProcessingException {
|
throws VideoFrameProcessingException {
|
||||||
// TODO(b/261188041) Add tests to verify the Listener is invoked on the given Executor.
|
// TODO(b/261188041) Add tests to verify the Listener is invoked on the given Executor.
|
||||||
|
|
||||||
@ -363,7 +340,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
EGLContext eglContext,
|
EGLContext eglContext,
|
||||||
InputSwitcher inputSwitcher,
|
InputSwitcher inputSwitcher,
|
||||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||||
Listener listener,
|
VideoFrameProcessor.Listener listener,
|
||||||
Executor listenerExecutor,
|
Executor listenerExecutor,
|
||||||
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
@ -626,9 +603,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||||
Executor videoFrameProcessorListenerExecutor,
|
Executor videoFrameProcessorListenerExecutor,
|
||||||
Listener listener,
|
VideoFrameProcessor.Listener listener,
|
||||||
GlObjectsProvider glObjectsProvider,
|
GlObjectsProvider glObjectsProvider,
|
||||||
@Nullable TextureOutputListener textureOutputListener,
|
@Nullable GlTextureProducer.Listener textureOutputListener,
|
||||||
int textureOutputCapacity)
|
int textureOutputCapacity)
|
||||||
throws GlUtil.GlException, VideoFrameProcessingException {
|
throws GlUtil.GlException, VideoFrameProcessingException {
|
||||||
EGLDisplay eglDisplay = GlUtil.getDefaultEglDisplay();
|
EGLDisplay eglDisplay = GlUtil.getDefaultEglDisplay();
|
||||||
@ -780,7 +757,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
List<GlShaderProgram> shaderPrograms,
|
List<GlShaderProgram> shaderPrograms,
|
||||||
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
||||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||||
Listener videoFrameProcessorListener,
|
VideoFrameProcessor.Listener videoFrameProcessorListener,
|
||||||
Executor videoFrameProcessorListenerExecutor) {
|
Executor videoFrameProcessorListenerExecutor) {
|
||||||
ArrayList<GlShaderProgram> shaderProgramsToChain = new ArrayList<>(shaderPrograms);
|
ArrayList<GlShaderProgram> shaderProgramsToChain = new ArrayList<>(shaderPrograms);
|
||||||
shaderProgramsToChain.add(finalShaderProgramWrapper);
|
shaderProgramsToChain.add(finalShaderProgramWrapper);
|
||||||
|
@ -65,7 +65,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
* <p>This wrapper is used for the final {@link DefaultShaderProgram} instance in the chain of
|
* <p>This wrapper is used for the final {@link DefaultShaderProgram} instance in the chain of
|
||||||
* {@link DefaultShaderProgram} instances used by {@link VideoFrameProcessor}.
|
* {@link DefaultShaderProgram} instances used by {@link VideoFrameProcessor}.
|
||||||
*/
|
*/
|
||||||
/* package */ final class FinalShaderProgramWrapper implements GlShaderProgram {
|
/* package */ final class FinalShaderProgramWrapper implements GlShaderProgram, GlTextureProducer {
|
||||||
|
|
||||||
interface OnInputStreamProcessedListener {
|
interface OnInputStreamProcessedListener {
|
||||||
void onInputStreamProcessed();
|
void onInputStreamProcessed();
|
||||||
@ -90,7 +90,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
private final TexturePool outputTexturePool;
|
private final TexturePool outputTexturePool;
|
||||||
private final LongArrayQueue outputTextureTimestamps; // Synchronized with outputTexturePool.
|
private final LongArrayQueue outputTextureTimestamps; // Synchronized with outputTexturePool.
|
||||||
private final LongArrayQueue syncObjects;
|
private final LongArrayQueue syncObjects;
|
||||||
@Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
|
@Nullable private final GlTextureProducer.Listener textureOutputListener;
|
||||||
|
|
||||||
private int inputWidth;
|
private int inputWidth;
|
||||||
private int inputHeight;
|
private int inputHeight;
|
||||||
@ -127,7 +127,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||||
Executor videoFrameProcessorListenerExecutor,
|
Executor videoFrameProcessorListenerExecutor,
|
||||||
VideoFrameProcessor.Listener videoFrameProcessorListener,
|
VideoFrameProcessor.Listener videoFrameProcessorListener,
|
||||||
@Nullable DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener,
|
@Nullable GlTextureProducer.Listener textureOutputListener,
|
||||||
int textureOutputCapacity) {
|
int textureOutputCapacity) {
|
||||||
this.context = context;
|
this.context = context;
|
||||||
this.matrixTransformations = new ArrayList<>();
|
this.matrixTransformations = new ArrayList<>();
|
||||||
@ -220,11 +220,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
throw new UnsupportedOperationException();
|
throw new UnsupportedOperationException();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void releaseOutputFrame(long presentationTimeUs) {
|
@Override
|
||||||
videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs));
|
public void releaseOutputTexture(long presentationTimeUs) {
|
||||||
|
videoFrameProcessingTaskExecutor.submit(() -> releaseOutputTextureInternal(presentationTimeUs));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void releaseOutputFrameInternal(long presentationTimeUs) throws GlUtil.GlException {
|
private void releaseOutputTextureInternal(long presentationTimeUs) throws GlUtil.GlException {
|
||||||
checkState(textureOutputListener != null);
|
checkState(textureOutputListener != null);
|
||||||
while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
|
while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
|
||||||
&& outputTextureTimestamps.element() <= presentationTimeUs) {
|
&& outputTextureTimestamps.element() <= presentationTimeUs) {
|
||||||
@ -390,7 +391,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
long syncObject = GlUtil.createGlSyncFence();
|
long syncObject = GlUtil.createGlSyncFence();
|
||||||
syncObjects.add(syncObject);
|
syncObjects.add(syncObject);
|
||||||
checkNotNull(textureOutputListener)
|
checkNotNull(textureOutputListener)
|
||||||
.onTextureRendered(outputTexture, presentationTimeUs, this::releaseOutputFrame, syncObject);
|
.onTextureRendered(
|
||||||
|
/* textureProducer= */ this, outputTexture, presentationTimeUs, syncObject);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -0,0 +1,51 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2023 The Android Open Source Project
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package androidx.media3.effect;
|
||||||
|
|
||||||
|
import android.opengl.GLES30;
|
||||||
|
import androidx.media3.common.GlTextureInfo;
|
||||||
|
import androidx.media3.common.VideoFrameProcessingException;
|
||||||
|
import androidx.media3.common.util.GlUtil;
|
||||||
|
import androidx.media3.common.util.UnstableApi;
|
||||||
|
|
||||||
|
/** A component that outputs {@linkplain GlTextureInfo OpenGL textures}. */
|
||||||
|
@UnstableApi
|
||||||
|
public interface GlTextureProducer {
|
||||||
|
|
||||||
|
/** Listener for texture output. */
|
||||||
|
interface Listener {
|
||||||
|
/**
|
||||||
|
* Called when a texture has been rendered to.
|
||||||
|
*
|
||||||
|
* @param textureProducer The {@link GlTextureProducer} that has rendered the texture.
|
||||||
|
* @param outputTexture The texture that has been rendered.
|
||||||
|
* @param presentationTimeUs The presentation time of the texture.
|
||||||
|
* @param syncObject A GL sync object that has been inserted into the GL command stream after
|
||||||
|
* the last write of the {@code outputTexture}. Value is 0 if and only if the {@link
|
||||||
|
* GLES30#glFenceSync} failed.
|
||||||
|
*/
|
||||||
|
void onTextureRendered(
|
||||||
|
GlTextureProducer textureProducer,
|
||||||
|
GlTextureInfo outputTexture,
|
||||||
|
long presentationTimeUs,
|
||||||
|
long syncObject)
|
||||||
|
throws VideoFrameProcessingException, GlUtil.GlException;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Releases the output texture at the given {@code presentationTimeUs}. */
|
||||||
|
void releaseOutputTexture(long presentationTimeUs);
|
||||||
|
}
|
@ -26,7 +26,7 @@ import androidx.media3.common.util.UnstableApi;
|
|||||||
* <p>Input and output are provided via OpenGL textures.
|
* <p>Input and output are provided via OpenGL textures.
|
||||||
*/
|
*/
|
||||||
@UnstableApi
|
@UnstableApi
|
||||||
public interface VideoCompositor {
|
public interface VideoCompositor extends GlTextureProducer {
|
||||||
|
|
||||||
/** Listener for errors. */
|
/** Listener for errors. */
|
||||||
interface Listener {
|
interface Listener {
|
||||||
@ -48,8 +48,7 @@ public interface VideoCompositor {
|
|||||||
int registerInputSource();
|
int registerInputSource();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Signals that no more frames will come from the upstream {@link
|
* Signals that no more frames will come from the upstream {@link GlTextureProducer.Listener}.
|
||||||
* DefaultVideoFrameProcessor.TextureOutputListener}.
|
|
||||||
*
|
*
|
||||||
* <p>Each input source must have a unique {@code inputId} returned from {@link
|
* <p>Each input source must have a unique {@code inputId} returned from {@link
|
||||||
* #registerInputSource}.
|
* #registerInputSource}.
|
||||||
@ -58,16 +57,16 @@ public interface VideoCompositor {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Queues an input texture to be composited, for example from an upstream {@link
|
* Queues an input texture to be composited, for example from an upstream {@link
|
||||||
* DefaultVideoFrameProcessor.TextureOutputListener}.
|
* GlTextureProducer.Listener}.
|
||||||
*
|
*
|
||||||
* <p>Each input source must have a unique {@code inputId} returned from {@link
|
* <p>Each input source must have a unique {@code inputId} returned from {@link
|
||||||
* #registerInputSource}.
|
* #registerInputSource}.
|
||||||
*/
|
*/
|
||||||
void queueInputTexture(
|
void queueInputTexture(
|
||||||
int inputId,
|
int inputId,
|
||||||
|
GlTextureProducer textureProducer,
|
||||||
GlTextureInfo inputTexture,
|
GlTextureInfo inputTexture,
|
||||||
long presentationTimeUs,
|
long presentationTimeUs);
|
||||||
DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseTextureCallback);
|
|
||||||
|
|
||||||
/** Releases all resources. */
|
/** Releases all resources. */
|
||||||
void release();
|
void release();
|
||||||
|
@ -647,9 +647,9 @@ public final class DefaultVideoCompositorPixelTest {
|
|||||||
compositorEnded.countDown();
|
compositorEnded.countDown();
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
/* textureOutputListener= */ (outputTexture,
|
/* textureOutputListener= */ (outputTextureProducer,
|
||||||
|
outputTexture,
|
||||||
presentationTimeUs,
|
presentationTimeUs,
|
||||||
releaseOutputTextureCallback,
|
|
||||||
syncObject) -> {
|
syncObject) -> {
|
||||||
if (!useSharedExecutor) {
|
if (!useSharedExecutor) {
|
||||||
GlUtil.awaitSyncObject(syncObject);
|
GlUtil.awaitSyncObject(syncObject);
|
||||||
@ -658,7 +658,7 @@ public final class DefaultVideoCompositorPixelTest {
|
|||||||
presentationTimeUs,
|
presentationTimeUs,
|
||||||
BitmapPixelTestUtil.createUnpremultipliedArgb8888BitmapFromFocusedGlFramebuffer(
|
BitmapPixelTestUtil.createUnpremultipliedArgb8888BitmapFromFocusedGlFramebuffer(
|
||||||
outputTexture.width, outputTexture.height));
|
outputTexture.width, outputTexture.height));
|
||||||
releaseOutputTextureCallback.release(presentationTimeUs);
|
outputTextureProducer.releaseOutputTexture(presentationTimeUs);
|
||||||
},
|
},
|
||||||
/* textureOutputCapacity= */ 1);
|
/* textureOutputCapacity= */ 1);
|
||||||
inputBitmapReaders = new ArrayList<>();
|
inputBitmapReaders = new ArrayList<>();
|
||||||
@ -791,15 +791,15 @@ public final class DefaultVideoCompositorPixelTest {
|
|||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setGlObjectsProvider(glObjectsProvider)
|
.setGlObjectsProvider(glObjectsProvider)
|
||||||
.setTextureOutput(
|
.setTextureOutput(
|
||||||
/* textureOutputListener= */ (outputTexture,
|
/* textureOutputListener= */ (outputTextureProducer,
|
||||||
|
outputTexture,
|
||||||
presentationTimeUs,
|
presentationTimeUs,
|
||||||
releaseOutputTextureCallback,
|
|
||||||
syncObject) -> {
|
syncObject) -> {
|
||||||
GlUtil.awaitSyncObject(syncObject);
|
GlUtil.awaitSyncObject(syncObject);
|
||||||
textureBitmapReader.readBitmapUnpremultipliedAlpha(
|
textureBitmapReader.readBitmapUnpremultipliedAlpha(
|
||||||
outputTexture, presentationTimeUs);
|
outputTexture, presentationTimeUs);
|
||||||
videoCompositor.queueInputTexture(
|
videoCompositor.queueInputTexture(
|
||||||
inputId, outputTexture, presentationTimeUs, releaseOutputTextureCallback);
|
inputId, outputTextureProducer, outputTexture, presentationTimeUs);
|
||||||
},
|
},
|
||||||
/* textureOutputCapacity= */ 2);
|
/* textureOutputCapacity= */ 2);
|
||||||
if (executorService != null) {
|
if (executorService != null) {
|
||||||
|
@ -138,12 +138,9 @@ public class DefaultVideoFrameProcessorMultipleTextureOutputPixelTest {
|
|||||||
VideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
VideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setTextureOutput(
|
.setTextureOutput(
|
||||||
(outputTexture,
|
(outputTextureProducer, outputTexture, presentationTimeUs, unusedSyncObject) -> {
|
||||||
presentationTimeUs,
|
|
||||||
releaseOutputTextureCallback,
|
|
||||||
unusedSyncObject) -> {
|
|
||||||
checkNotNull(textureBitmapReader).readBitmap(outputTexture, presentationTimeUs);
|
checkNotNull(textureBitmapReader).readBitmap(outputTexture, presentationTimeUs);
|
||||||
releaseOutputTextureCallback.release(presentationTimeUs);
|
outputTextureProducer.releaseOutputTexture(presentationTimeUs);
|
||||||
},
|
},
|
||||||
/* textureOutputCapacity= */ 1)
|
/* textureOutputCapacity= */ 1)
|
||||||
.build();
|
.build();
|
||||||
|
@ -42,6 +42,7 @@ import androidx.media3.common.util.GlUtil;
|
|||||||
import androidx.media3.effect.BitmapOverlay;
|
import androidx.media3.effect.BitmapOverlay;
|
||||||
import androidx.media3.effect.DefaultGlObjectsProvider;
|
import androidx.media3.effect.DefaultGlObjectsProvider;
|
||||||
import androidx.media3.effect.DefaultVideoFrameProcessor;
|
import androidx.media3.effect.DefaultVideoFrameProcessor;
|
||||||
|
import androidx.media3.effect.GlTextureProducer;
|
||||||
import androidx.media3.effect.OverlayEffect;
|
import androidx.media3.effect.OverlayEffect;
|
||||||
import androidx.media3.test.utils.BitmapPixelTestUtil;
|
import androidx.media3.test.utils.BitmapPixelTestUtil;
|
||||||
import androidx.media3.test.utils.TextureBitmapReader;
|
import androidx.media3.test.utils.TextureBitmapReader;
|
||||||
@ -496,15 +497,15 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
|||||||
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setTextureOutput(
|
.setTextureOutput(
|
||||||
(outputTexture, presentationTimeUs, releaseOutputTextureCallback, syncObject) ->
|
(outputTextureProducer, outputTexture, presentationTimeUs, syncObject) ->
|
||||||
inputTextureIntoVideoFrameProcessor(
|
inputTextureIntoVideoFrameProcessor(
|
||||||
testId,
|
testId,
|
||||||
consumersBitmapReader,
|
consumersBitmapReader,
|
||||||
colorInfo,
|
colorInfo,
|
||||||
effects,
|
effects,
|
||||||
outputTexture,
|
outputTexture,
|
||||||
|
outputTextureProducer,
|
||||||
presentationTimeUs,
|
presentationTimeUs,
|
||||||
releaseOutputTextureCallback,
|
|
||||||
syncObject),
|
syncObject),
|
||||||
/* textureOutputCapacity= */ 1)
|
/* textureOutputCapacity= */ 1)
|
||||||
.build();
|
.build();
|
||||||
@ -524,8 +525,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
|||||||
ColorInfo colorInfo,
|
ColorInfo colorInfo,
|
||||||
List<Effect> effects,
|
List<Effect> effects,
|
||||||
GlTextureInfo texture,
|
GlTextureInfo texture,
|
||||||
|
GlTextureProducer textureProducer,
|
||||||
long presentationTimeUs,
|
long presentationTimeUs,
|
||||||
DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback,
|
|
||||||
long syncObject)
|
long syncObject)
|
||||||
throws VideoFrameProcessingException, GlUtil.GlException {
|
throws VideoFrameProcessingException, GlUtil.GlException {
|
||||||
GlObjectsProvider contextSharingGlObjectsProvider =
|
GlObjectsProvider contextSharingGlObjectsProvider =
|
||||||
@ -533,12 +534,9 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
|||||||
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setTextureOutput(
|
.setTextureOutput(
|
||||||
(outputTexture,
|
(outputTextureProducer, outputTexture, presentationTimeUs1, unusedSyncObject) -> {
|
||||||
presentationTimeUs1,
|
|
||||||
releaseOutputTextureCallback1,
|
|
||||||
unusedSyncObject) -> {
|
|
||||||
bitmapReader.readBitmap(outputTexture, presentationTimeUs1);
|
bitmapReader.readBitmap(outputTexture, presentationTimeUs1);
|
||||||
releaseOutputTextureCallback1.release(presentationTimeUs1);
|
outputTextureProducer.releaseOutputTexture(presentationTimeUs1);
|
||||||
},
|
},
|
||||||
/* textureOutputCapacity= */ 1)
|
/* textureOutputCapacity= */ 1)
|
||||||
.setGlObjectsProvider(contextSharingGlObjectsProvider)
|
.setGlObjectsProvider(contextSharingGlObjectsProvider)
|
||||||
@ -560,7 +558,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
|||||||
throw VideoFrameProcessingException.from(e);
|
throw VideoFrameProcessingException.from(e);
|
||||||
}
|
}
|
||||||
videoFrameProcessorTestRunner.endFrameProcessing(VIDEO_FRAME_PROCESSING_WAIT_MS / 2);
|
videoFrameProcessorTestRunner.endFrameProcessing(VIDEO_FRAME_PROCESSING_WAIT_MS / 2);
|
||||||
releaseOutputTextureCallback.release(presentationTimeUs);
|
textureProducer.releaseOutputTexture(presentationTimeUs);
|
||||||
}
|
}
|
||||||
|
|
||||||
private VideoFrameProcessorTestRunner.Builder getSurfaceInputFrameProcessorTestRunnerBuilder(
|
private VideoFrameProcessorTestRunner.Builder getSurfaceInputFrameProcessorTestRunnerBuilder(
|
||||||
@ -569,12 +567,9 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
|||||||
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setTextureOutput(
|
.setTextureOutput(
|
||||||
(outputTexture,
|
(outputTextureProducer, outputTexture, presentationTimeUs, unusedSyncObject) -> {
|
||||||
presentationTimeUs,
|
|
||||||
releaseOutputTextureCallback,
|
|
||||||
unusedSyncObject) -> {
|
|
||||||
textureBitmapReader.readBitmap(outputTexture, presentationTimeUs);
|
textureBitmapReader.readBitmap(outputTexture, presentationTimeUs);
|
||||||
releaseOutputTextureCallback.release(presentationTimeUs);
|
outputTextureProducer.releaseOutputTexture(presentationTimeUs);
|
||||||
},
|
},
|
||||||
/* textureOutputCapacity= */ 1)
|
/* textureOutputCapacity= */ 1)
|
||||||
.build();
|
.build();
|
||||||
|
Loading…
x
Reference in New Issue
Block a user