loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Split GlTextureProcessor.Listener into input/output/error listener.

Browse Source 
This simplifies ChainingGlTextureProcessor as it now only connects a
consuming and a producing GlTextureProcessor rather than a previous,
current, and next GlTextureProcessor.

Also use default no-op implementations of the listeners in
SingleFrameGlTextureProcessor and MediaPipeProcessor to avoid
null-checks.

PiperOrigin-RevId: 466301642
(cherry picked from commit 7dc05edbab233dcd631a4a78ad65f1c30a1e9734)
This commit is contained in:
Googler 2022-08-09 10:05:14 +00:00 committed by microkatz
parent 37274c91de
commit cdd47ff703
7 changed files with 183 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
View File

@ -31,7 +31,6 @@ import com.google.mediapipe.framework.AppTextureFrame;
import com.google.mediapipe.framework.TextureFrame; import com.google.mediapipe.framework.TextureFrame;
import com.google.mediapipe.glutil.EglManager; import com.google.mediapipe.glutil.EglManager;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Runs a MediaPipe graph on input frames. */ /** Runs a MediaPipe graph on input frames. */
/* package */ final class MediaPipeProcessor implements GlTextureProcessor { /* package */ final class MediaPipeProcessor implements GlTextureProcessor {
@ -55,10 +54,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
private final FrameProcessor frameProcessor; private final FrameProcessor frameProcessor;
private volatile GlTextureProcessor.@MonotonicNonNull Listener listener;
private volatile boolean acceptedFrame;
private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames; private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
private InputListener inputListener;
private OutputListener outputListener;
private ErrorListener errorListener;
private boolean acceptedFrame;
/** /**
* Creates a new texture processor that wraps a MediaPipe graph. * Creates a new texture processor that wraps a MediaPipe graph.
* *
@ -78,11 +80,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
checkState(LOADER.isAvailable()); checkState(LOADER.isAvailable());
// TODO(b/227624622): Confirm whether MediaPipeProcessor could support HDR colors. // TODO(b/227624622): Confirm whether MediaPipeProcessor could support HDR colors.
checkArgument(!useHdr, "MediaPipeProcessor does not support HDR colors."); checkArgument(!useHdr, "MediaPipeProcessor does not support HDR colors.");
inputListener = new InputListener() {};
outputListener = new OutputListener() {};
errorListener = (frameProcessingException) -> {};
EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor = frameProcessor =
new FrameProcessor( new FrameProcessor(
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName); context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
outputFrames = new ConcurrentHashMap<>(); outputFrames = new ConcurrentHashMap<>();
// OnWillAddFrameListener is called on the same thread as frameProcessor.onNewFrame(...), so no
// synchronization is needed for acceptedFrame.
frameProcessor.setOnWillAddFrameListener((long timestamp) -> acceptedFrame = true);
}
@Override
public void setInputListener(InputListener inputListener) {
this.inputListener = inputListener;
}
@Override
public void setOutputListener(OutputListener outputListener) {
this.outputListener = outputListener;
frameProcessor.setConsumer( frameProcessor.setConsumer(
frame -> { frame -> {
TextureInfo texture = TextureInfo texture =
@ -92,22 +110,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
frame.getWidth(), frame.getWidth(),
frame.getHeight()); frame.getHeight());
outputFrames.put(texture, frame); outputFrames.put(texture, frame);
if (listener != null) { outputListener.onOutputFrameAvailable(texture, frame.getTimestamp());
listener.onOutputFrameAvailable(texture, frame.getTimestamp());
}
}); });
frameProcessor.setAsynchronousErrorListener(
error -> {
if (listener != null) {
listener.onFrameProcessingError(new FrameProcessingException(error));
}
});
frameProcessor.setOnWillAddFrameListener((long timestamp) -> acceptedFrame = true);
} }
@Override @Override
public void setListener(GlTextureProcessor.Listener listener) { public void setErrorListener(ErrorListener errorListener) {
this.listener = listener; this.errorListener = errorListener;
frameProcessor.setAsynchronousErrorListener(
error -> errorListener.onFrameProcessingError(new FrameProcessingException(error)));
} }
@Override @Override
@ -123,13 +134,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
appTextureFrame.waitUntilReleasedWithGpuSync(); appTextureFrame.waitUntilReleasedWithGpuSync();
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
if (listener != null) { errorListener.onFrameProcessingError(new FrameProcessingException(e));
listener.onFrameProcessingError(new FrameProcessingException(e));
}
}
if (listener != null) {
listener.onInputFrameProcessed(inputTexture);
} }
inputListener.onInputFrameProcessed(inputTexture);
return acceptedFrame; return acceptedFrame;
} }
@ -146,8 +153,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public final void signalEndOfCurrentInputStream() { public final void signalEndOfCurrentInputStream() {
frameProcessor.waitUntilIdle(); frameProcessor.waitUntilIdle();
if (listener != null) { outputListener.onCurrentOutputStreamEnded();
listener.onCurrentOutputStreamEnded();
}
} }
} }

81
libraries/effect/src/main/java/androidx/media3/effect/ChainingGlTextureProcessorListener.java
View File

@ -16,92 +16,75 @@
package androidx.media3.effect; package androidx.media3.effect;
import android.util.Pair; import android.util.Pair;
import androidx.annotation.Nullable; import androidx.media3.effect.GlTextureProcessor.InputListener;
import androidx.media3.common.FrameProcessingException; import androidx.media3.effect.GlTextureProcessor.OutputListener;
import androidx.media3.common.FrameProcessor;
import java.util.ArrayDeque; import java.util.ArrayDeque;
import java.util.Queue; import java.util.Queue;
/** /**
* A {@link GlTextureProcessor.Listener} that connects the {@link GlTextureProcessor} it is * Connects a producing and a consuming {@link GlTextureProcessor} instance.
* {@linkplain GlTextureProcessor#setListener(GlTextureProcessor.Listener) set} on to a previous and *
* next {@link GlTextureProcessor}. * <p>This listener should be set as {@link InputListener} on the consuming {@link
* GlTextureProcessor} and as {@link OutputListener} on the producing {@link GlTextureProcessor}.
*/ */
/* package */ final class ChainingGlTextureProcessorListener /* package */ final class ChainingGlTextureProcessorListener
implements GlTextureProcessor.Listener { implements GlTextureProcessor.InputListener, GlTextureProcessor.OutputListener {
@Nullable private final GlTextureProcessor previousGlTextureProcessor; private final GlTextureProcessor producingGlTextureProcessor;
@Nullable private final GlTextureProcessor nextGlTextureProcessor; private final GlTextureProcessor consumingGlTextureProcessor;
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor; private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
private final FrameProcessor.Listener frameProcessorListener; private final Queue<Pair<TextureInfo, Long>> availableFrames;
private final Queue<Pair<TextureInfo, Long>> pendingFrames;
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param previousGlTextureProcessor The {@link GlTextureProcessor} that comes before the {@link * @param producingGlTextureProcessor The {@link GlTextureProcessor} for which this listener will
* GlTextureProcessor} this listener is set on or {@code null} if not applicable. * be set as {@link OutputListener}.
* @param nextGlTextureProcessor The {@link GlTextureProcessor} that comes after the {@link * @param consumingGlTextureProcessor The {@link GlTextureProcessor} for which this listener will
* GlTextureProcessor} this listener is set on or {@code null} if not applicable. * be set as {@link InputListener}.
* @param frameProcessingTaskExecutor The {@link FrameProcessingTaskExecutor} that is used for * @param frameProcessingTaskExecutor The {@link FrameProcessingTaskExecutor} that is used for
* OpenGL calls. All calls to the previous/next {@link GlTextureProcessor} will be executed by * OpenGL calls. All calls to the producing/consuming {@link GlTextureProcessor} will be
* the {@link FrameProcessingTaskExecutor}. The caller is responsible for releasing the {@link * executed by the {@link FrameProcessingTaskExecutor}. The caller is responsible for
* FrameProcessingTaskExecutor}. * releasing the {@link FrameProcessingTaskExecutor}.
* @param frameProcessorListener The {@link FrameProcessor.Listener} to forward exceptions to.
*/ */
public ChainingGlTextureProcessorListener( public ChainingGlTextureProcessorListener(
@Nullable GlTextureProcessor previousGlTextureProcessor, GlTextureProcessor producingGlTextureProcessor,
@Nullable GlTextureProcessor nextGlTextureProcessor, GlTextureProcessor consumingGlTextureProcessor,
FrameProcessingTaskExecutor frameProcessingTaskExecutor, FrameProcessingTaskExecutor frameProcessingTaskExecutor) {
FrameProcessor.Listener frameProcessorListener) { this.producingGlTextureProcessor = producingGlTextureProcessor;
this.previousGlTextureProcessor = previousGlTextureProcessor; this.consumingGlTextureProcessor = consumingGlTextureProcessor;
this.nextGlTextureProcessor = nextGlTextureProcessor;
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
this.frameProcessorListener = frameProcessorListener; availableFrames = new ArrayDeque<>();
pendingFrames = new ArrayDeque<>();
} }
@Override @Override
public void onInputFrameProcessed(TextureInfo inputTexture) { public void onInputFrameProcessed(TextureInfo inputTexture) {
if (previousGlTextureProcessor != null) {
GlTextureProcessor nonNullPreviousGlTextureProcessor = previousGlTextureProcessor;
frameProcessingTaskExecutor.submit( frameProcessingTaskExecutor.submit(
() -> nonNullPreviousGlTextureProcessor.releaseOutputFrame(inputTexture)); () -> producingGlTextureProcessor.releaseOutputFrame(inputTexture));
}
} }
@Override @Override
public void onOutputFrameAvailable(TextureInfo outputTexture, long presentationTimeUs) { public void onOutputFrameAvailable(TextureInfo outputTexture, long presentationTimeUs) {
if (nextGlTextureProcessor != null) {
GlTextureProcessor nonNullNextGlTextureProcessor = nextGlTextureProcessor;
frameProcessingTaskExecutor.submit( frameProcessingTaskExecutor.submit(
() -> { () -> {
pendingFrames.add(new Pair<>(outputTexture, presentationTimeUs)); availableFrames.add(new Pair<>(outputTexture, presentationTimeUs));
processFrameNowOrLater(nonNullNextGlTextureProcessor); processFrameNowOrLater();
}); });
} }
}
private void processFrameNowOrLater(GlTextureProcessor nextGlTextureProcessor) { private void processFrameNowOrLater() {
Pair<TextureInfo, Long> pendingFrame = pendingFrames.element(); Pair<TextureInfo, Long> pendingFrame = availableFrames.element();
TextureInfo outputTexture = pendingFrame.first; TextureInfo outputTexture = pendingFrame.first;
long presentationTimeUs = pendingFrame.second; long presentationTimeUs = pendingFrame.second;
if (nextGlTextureProcessor.maybeQueueInputFrame(outputTexture, presentationTimeUs)) { if (consumingGlTextureProcessor.maybeQueueInputFrame(outputTexture, presentationTimeUs)) {
pendingFrames.remove(); availableFrames.remove();
} else { } else {
frameProcessingTaskExecutor.submit(() -> processFrameNowOrLater(nextGlTextureProcessor)); frameProcessingTaskExecutor.submit(this::processFrameNowOrLater);
} }
} }
@Override @Override
public void onCurrentOutputStreamEnded() { public void onCurrentOutputStreamEnded() {
if (nextGlTextureProcessor != null) { frameProcessingTaskExecutor.submit(consumingGlTextureProcessor::signalEndOfCurrentInputStream);
frameProcessingTaskExecutor.submit(nextGlTextureProcessor::signalEndOfCurrentInputStream);
}
}
@Override
public void onFrameProcessingError(FrameProcessingException e) {
frameProcessorListener.onFrameProcessingError(e);
} }
} }

34
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
View File

@ -78,7 +78,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private int inputHeight; private int inputHeight;
@Nullable private MatrixTransformationProcessor matrixTransformationProcessor; @Nullable private MatrixTransformationProcessor matrixTransformationProcessor;
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper; @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
private @MonotonicNonNull Listener listener; private InputListener inputListener;
private @MonotonicNonNull Pair<Integer, Integer> outputSizeBeforeSurfaceTransformation; private @MonotonicNonNull Pair<Integer, Integer> outputSizeBeforeSurfaceTransformation;
@Nullable private SurfaceView debugSurfaceView; @Nullable private SurfaceView debugSurfaceView;
@ -113,18 +113,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
textureTransformMatrix = new float[16]; textureTransformMatrix = new float[16];
Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0); Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
streamOffsetUsQueue = new ArrayDeque<>(); streamOffsetUsQueue = new ArrayDeque<>();
inputListener = new InputListener() {};
} }
/**
* {@inheritDoc}
*
* <p>The {@code FinalMatrixTransformationProcessorWrapper} will only call {@link
* Listener#onInputFrameProcessed(TextureInfo)}. Other events are handled via the {@link
* FrameProcessor.Listener} passed to the constructor.
*/
@Override @Override
public void setListener(Listener listener) { public void setInputListener(InputListener inputListener) {
this.listener = listener; this.inputListener = inputListener;
}
@Override
public void setOutputListener(OutputListener outputListener) {
// The FrameProcessor.Listener passed to the constructor is used for output-related events.
throw new UnsupportedOperationException();
}
@Override
public void setErrorListener(ErrorListener errorListener) {
// The FrameProcessor.Listener passed to the constructor is used for errors.
throw new UnsupportedOperationException();
} }
@Override @Override
@ -174,9 +180,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Log.d(TAG, "Error rendering to debug preview", e); Log.d(TAG, "Error rendering to debug preview", e);
} }
} }
if (listener != null) { inputListener.onInputFrameProcessed(inputTexture);
listener.onInputFrameProcessed(inputTexture);
}
return true; return true;
} }
@ -278,8 +282,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public void releaseOutputFrame(TextureInfo outputTexture) { public void releaseOutputFrame(TextureInfo outputTexture) {
throw new UnsupportedOperationException( // The final texture processor writes to a surface so there is no texture to release.
"The final texture processor writes to a surface so there is no texture to release"); throw new UnsupportedOperationException();
} }
@Override @Override

27
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
View File

@ -219,26 +219,21 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
private static void chainTextureProcessorsWithListeners( private static void chainTextureProcessorsWithListeners(
ImmutableList<GlTextureProcessor> textureProcessors, ImmutableList<GlTextureProcessor> textureProcessors,
FrameProcessingTaskExecutor frameProcessingTaskExecutor, FrameProcessingTaskExecutor frameProcessingTaskExecutor,
FrameProcessor.Listener listener) { FrameProcessor.Listener frameProcessorListener) {
for (int i = 0; i < textureProcessors.size(); i++) { for (int i = 0; i < textureProcessors.size() - 1; i++) {
@Nullable GlTextureProcessor producingGlTextureProcessor = textureProcessors.get(i);
GlTextureProcessor previousGlTextureProcessor = GlTextureProcessor consumingGlTextureProcessor = textureProcessors.get(i + 1);
i - 1 >= 0 ? textureProcessors.get(i - 1) : null; ChainingGlTextureProcessorListener chainingGlTextureProcessorListener =
@Nullable
GlTextureProcessor nextGlTextureProcessor =
i + 1 < textureProcessors.size() ? textureProcessors.get(i + 1) : null;
textureProcessors
.get(i)
.setListener(
new ChainingGlTextureProcessorListener( new ChainingGlTextureProcessorListener(
previousGlTextureProcessor, producingGlTextureProcessor,
nextGlTextureProcessor, consumingGlTextureProcessor,
frameProcessingTaskExecutor, frameProcessingTaskExecutor);
listener)); producingGlTextureProcessor.setOutputListener(chainingGlTextureProcessorListener);
producingGlTextureProcessor.setErrorListener(frameProcessorListener::onFrameProcessingError);
consumingGlTextureProcessor.setInputListener(chainingGlTextureProcessorListener);
} }
} }
private static final String TAG = "GlEffectsFrameProcessor";
private static final String THREAD_NAME = "Transformer:GlEffectsFrameProcessor"; private static final String THREAD_NAME = "Transformer:GlEffectsFrameProcessor";
private static final long RELEASE_WAIT_TIME_MS = 100; private static final long RELEASE_WAIT_TIME_MS = 100;

50
libraries/effect/src/main/java/androidx/media3/effect/GlTextureProcessor.java
View File

@ -23,14 +23,14 @@ import androidx.media3.common.util.UnstableApi;
* *
* <p>The {@code GlTextureProcessor} consumes input frames it accepts via {@link * <p>The {@code GlTextureProcessor} consumes input frames it accepts via {@link
* #maybeQueueInputFrame(TextureInfo, long)} and surrenders each texture back to the caller via its * #maybeQueueInputFrame(TextureInfo, long)} and surrenders each texture back to the caller via its
* {@linkplain Listener#onInputFrameProcessed(TextureInfo) listener} once the texture's contents * {@linkplain InputListener#onInputFrameProcessed(TextureInfo) listener} once the texture's
* have been processed. * contents have been processed.
* *
* <p>The {@code GlTextureProcessor} produces output frames asynchronously and notifies its owner * <p>The {@code GlTextureProcessor} produces output frames asynchronously and notifies its owner
* when they are available via its {@linkplain Listener#onOutputFrameAvailable(TextureInfo, long) * when they are available via its {@linkplain OutputListener#onOutputFrameAvailable(TextureInfo,
* listener}. The {@code GlTextureProcessor} instance's owner must surrender the texture back to the * long) listener}. The {@code GlTextureProcessor} instance's owner must surrender the texture back
* {@code GlTextureProcessor} via {@link #releaseOutputFrame(TextureInfo)} when it has finished * to the {@code GlTextureProcessor} via {@link #releaseOutputFrame(TextureInfo)} when it has
* processing it. * finished processing it.
* *
* <p>{@code GlTextureProcessor} implementations can choose to produce output frames before * <p>{@code GlTextureProcessor} implementations can choose to produce output frames before
* receiving input frames or process several input frames before producing an output frame. However, * receiving input frames or process several input frames before producing an output frame. However,
@ -46,19 +46,26 @@ import androidx.media3.common.util.UnstableApi;
public interface GlTextureProcessor { public interface GlTextureProcessor {
/** /**
* Listener for frame processing events. * Listener for input-related frame processing events.
* *
* <p>This listener can be called from any thread. * <p>This listener can be called from any thread.
*/ */
interface Listener { interface InputListener {
/** /**
* Called when the {@link GlTextureProcessor} has processed an input frame. * Called when the {@link GlTextureProcessor} has processed an input frame.
* *
* @param inputTexture The {@link TextureInfo} that was used to {@linkplain * @param inputTexture The {@link TextureInfo} that was used to {@linkplain
* #maybeQueueInputFrame(TextureInfo, long) queue} the input frame. * #maybeQueueInputFrame(TextureInfo, long) queue} the input frame.
*/ */
void onInputFrameProcessed(TextureInfo inputTexture); default void onInputFrameProcessed(TextureInfo inputTexture) {}
}
/**
* Listener for output-related frame processing events.
*
* <p>This listener can be called from any thread.
*/
interface OutputListener {
/** /**
* Called when the {@link GlTextureProcessor} has produced an output frame. * Called when the {@link GlTextureProcessor} has produced an output frame.
* *
@ -71,14 +78,21 @@ public interface GlTextureProcessor {
* frame. * frame.
* @param presentationTimeUs The presentation timestamp of the output frame, in microseconds. * @param presentationTimeUs The presentation timestamp of the output frame, in microseconds.
*/ */
void onOutputFrameAvailable(TextureInfo outputTexture, long presentationTimeUs); default void onOutputFrameAvailable(TextureInfo outputTexture, long presentationTimeUs) {}
/** /**
* Called when the {@link GlTextureProcessor} will not produce further output frames belonging * Called when the {@link GlTextureProcessor} will not produce further output frames belonging
* to the current output stream. * to the current output stream.
*/ */
void onCurrentOutputStreamEnded(); default void onCurrentOutputStreamEnded() {}
}
/**
* Listener for frame processing errors.
*
* <p>This listener can be called from any thread.
*/
interface ErrorListener {
/** /**
* Called when an exception occurs during asynchronous frame processing. * Called when an exception occurs during asynchronous frame processing.
* *
@ -88,16 +102,22 @@ public interface GlTextureProcessor {
void onFrameProcessingError(FrameProcessingException e); void onFrameProcessingError(FrameProcessingException e);
} }
/** Sets the {@link Listener} for frame processing events. */ /** Sets the {@link InputListener}. */
void setListener(Listener listener); void setInputListener(InputListener inputListener);
/** Sets the {@link OutputListener}. */
void setOutputListener(OutputListener outputListener);
/** Sets the {@link ErrorListener}. */
void setErrorListener(ErrorListener errorListener);
/** /**
* Processes an input frame if possible. * Processes an input frame if possible.
* *
* <p>If this method returns {@code true} the input frame has been accepted. The {@code * <p>If this method returns {@code true} the input frame has been accepted. The {@code
* GlTextureProcessor} owns the accepted frame until it calls {@link * GlTextureProcessor} owns the accepted frame until it calls {@link
* Listener#onInputFrameProcessed(TextureInfo)}. The caller should not overwrite or release the * InputListener#onInputFrameProcessed(TextureInfo)}. The caller should not overwrite or release
* texture before the {@code GlTextureProcessor} has finished processing it. * the texture before the {@code GlTextureProcessor} has finished processing it.
* *
* <p>If this method returns {@code false}, the input frame could not be accepted and the caller * <p>If this method returns {@code false}, the input frame could not be accepted and the caller
* should decide whether to drop the frame or try again later. * should decide whether to drop the frame or try again later.

35
libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlTextureProcessor.java
View File

@ -36,7 +36,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@UnstableApi @UnstableApi
public abstract class SingleFrameGlTextureProcessor implements GlTextureProcessor { public abstract class SingleFrameGlTextureProcessor implements GlTextureProcessor {
private @MonotonicNonNull Listener listener; private InputListener inputListener;
private OutputListener outputListener;
private ErrorListener errorListener;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
private @MonotonicNonNull TextureInfo outputTexture; private @MonotonicNonNull TextureInfo outputTexture;
@ -51,6 +53,9 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
*/ */
public SingleFrameGlTextureProcessor(boolean useHdr) { public SingleFrameGlTextureProcessor(boolean useHdr) {
this.useHdr = useHdr; this.useHdr = useHdr;
inputListener = new InputListener() {};
outputListener = new OutputListener() {};
errorListener = (frameProcessingException) -> {};
} }
/** /**
@ -83,8 +88,18 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
throws FrameProcessingException; throws FrameProcessingException;
@Override @Override
public final void setListener(Listener listener) { public final void setInputListener(InputListener inputListener) {
this.listener = listener; this.inputListener = inputListener;
}
@Override
public final void setOutputListener(OutputListener outputListener) {
this.outputListener = outputListener;
}
@Override
public final void setErrorListener(ErrorListener errorListener) {
this.errorListener = errorListener;
} }
@Override @Override
@ -104,18 +119,14 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
outputTexture.fboId, outputTexture.width, outputTexture.height); outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame(); GlUtil.clearOutputFrame();
drawFrame(inputTexture.texId, presentationTimeUs); drawFrame(inputTexture.texId, presentationTimeUs);
if (listener != null) { inputListener.onInputFrameProcessed(inputTexture);
listener.onInputFrameProcessed(inputTexture); outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
listener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
}
} catch (FrameProcessingException | GlUtil.GlException | RuntimeException e) { } catch (FrameProcessingException | GlUtil.GlException | RuntimeException e) {
if (listener != null) { errorListener.onFrameProcessingError(
listener.onFrameProcessingError(
e instanceof FrameProcessingException e instanceof FrameProcessingException
? (FrameProcessingException) e ? (FrameProcessingException) e
: new FrameProcessingException(e)); : new FrameProcessingException(e));
} }
}
return true; return true;
} }
@ -144,9 +155,7 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
@Override @Override
public final void signalEndOfCurrentInputStream() { public final void signalEndOfCurrentInputStream() {
if (listener != null) { outputListener.onCurrentOutputStreamEnded();
listener.onCurrentOutputStreamEnded();
}
} }
@Override @Override

55
libraries/effect/src/test/java/androidx/media3/effect/ChainingGlTextureProcessorListenerTest.java
View File

@ -20,7 +20,6 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.FrameProcessor; import androidx.media3.common.FrameProcessor;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
@ -33,35 +32,23 @@ import org.junit.runner.RunWith;
public final class ChainingGlTextureProcessorListenerTest { public final class ChainingGlTextureProcessorListenerTest {
private static final long EXECUTOR_WAIT_TIME_MS = 100; private static final long EXECUTOR_WAIT_TIME_MS = 100;
private final FrameProcessor.Listener mockframeProcessorListener =
mock(FrameProcessor.Listener.class);
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor = private final FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor( new FrameProcessingTaskExecutor(
Util.newSingleThreadExecutor("Test"), mockframeProcessorListener); Util.newSingleThreadExecutor("Test"), mock(FrameProcessor.Listener.class));
private final GlTextureProcessor mockPreviousGlTextureProcessor = mock(GlTextureProcessor.class); private final GlTextureProcessor mockProducingGlTextureProcessor = mock(GlTextureProcessor.class);
private final FakeGlTextureProcessor fakeNextGlTextureProcessor = private final FakeGlTextureProcessor fakeConsumingGlTextureProcessor =
spy(new FakeGlTextureProcessor()); spy(new FakeGlTextureProcessor());
private final ChainingGlTextureProcessorListener chainingGlTextureProcessorListener = private final ChainingGlTextureProcessorListener chainingGlTextureProcessorListener =
new ChainingGlTextureProcessorListener( new ChainingGlTextureProcessorListener(
mockPreviousGlTextureProcessor, mockProducingGlTextureProcessor,
fakeNextGlTextureProcessor, fakeConsumingGlTextureProcessor,
frameProcessingTaskExecutor, frameProcessingTaskExecutor);
mockframeProcessorListener);
@After @After
public void release() throws InterruptedException { public void release() throws InterruptedException {
frameProcessingTaskExecutor.release(/* releaseTask= */ () -> {}, EXECUTOR_WAIT_TIME_MS); frameProcessingTaskExecutor.release(/* releaseTask= */ () -> {}, EXECUTOR_WAIT_TIME_MS);
} }
@Test
public void onFrameProcessingError_callsListener() {
FrameProcessingException exception = new FrameProcessingException("message");
chainingGlTextureProcessorListener.onFrameProcessingError(exception);
verify(mockframeProcessorListener, times(1)).onFrameProcessingError(exception);
}
@Test @Test
public void onInputFrameProcessed_surrendersFrameToPreviousGlTextureProcessor() public void onInputFrameProcessed_surrendersFrameToPreviousGlTextureProcessor()
throws InterruptedException { throws InterruptedException {
@ -71,7 +58,7 @@ public final class ChainingGlTextureProcessorListenerTest {
chainingGlTextureProcessorListener.onInputFrameProcessed(texture); chainingGlTextureProcessorListener.onInputFrameProcessed(texture);
Thread.sleep(EXECUTOR_WAIT_TIME_MS); Thread.sleep(EXECUTOR_WAIT_TIME_MS);
verify(mockPreviousGlTextureProcessor, times(1)).releaseOutputFrame(texture); verify(mockProducingGlTextureProcessor, times(1)).releaseOutputFrame(texture);
} }
@Test @Test
@ -84,7 +71,8 @@ public final class ChainingGlTextureProcessorListenerTest {
chainingGlTextureProcessorListener.onOutputFrameAvailable(texture, presentationTimeUs); chainingGlTextureProcessorListener.onOutputFrameAvailable(texture, presentationTimeUs);
Thread.sleep(EXECUTOR_WAIT_TIME_MS); Thread.sleep(EXECUTOR_WAIT_TIME_MS);
verify(fakeNextGlTextureProcessor, times(1)).maybeQueueInputFrame(texture, presentationTimeUs); verify(fakeConsumingGlTextureProcessor, times(1))
.maybeQueueInputFrame(texture, presentationTimeUs);
} }
@Test @Test
@ -93,12 +81,13 @@ public final class ChainingGlTextureProcessorListenerTest {
TextureInfo texture = TextureInfo texture =
new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100);
long presentationTimeUs = 123; long presentationTimeUs = 123;
fakeNextGlTextureProcessor.rejectNextFrame(); fakeConsumingGlTextureProcessor.rejectNextFrame();
chainingGlTextureProcessorListener.onOutputFrameAvailable(texture, presentationTimeUs); chainingGlTextureProcessorListener.onOutputFrameAvailable(texture, presentationTimeUs);
Thread.sleep(EXECUTOR_WAIT_TIME_MS); Thread.sleep(EXECUTOR_WAIT_TIME_MS);
verify(fakeNextGlTextureProcessor, times(2)).maybeQueueInputFrame(texture, presentationTimeUs); verify(fakeConsumingGlTextureProcessor, times(2))
.maybeQueueInputFrame(texture, presentationTimeUs);
} }
@Test @Test
@ -110,7 +99,7 @@ public final class ChainingGlTextureProcessorListenerTest {
TextureInfo secondTexture = TextureInfo secondTexture =
new TextureInfo(/* texId= */ 2, /* fboId= */ 2, /* width= */ 100, /* height= */ 100); new TextureInfo(/* texId= */ 2, /* fboId= */ 2, /* width= */ 100, /* height= */ 100);
long secondPresentationTimeUs = 567; long secondPresentationTimeUs = 567;
fakeNextGlTextureProcessor.rejectNextFrame(); fakeConsumingGlTextureProcessor.rejectNextFrame();
chainingGlTextureProcessorListener.onOutputFrameAvailable( chainingGlTextureProcessorListener.onOutputFrameAvailable(
firstTexture, firstPresentationTimeUs); firstTexture, firstPresentationTimeUs);
@ -118,9 +107,9 @@ public final class ChainingGlTextureProcessorListenerTest {
secondTexture, secondPresentationTimeUs); secondTexture, secondPresentationTimeUs);
Thread.sleep(EXECUTOR_WAIT_TIME_MS); Thread.sleep(EXECUTOR_WAIT_TIME_MS);
verify(fakeNextGlTextureProcessor, times(2)) verify(fakeConsumingGlTextureProcessor, times(2))
.maybeQueueInputFrame(firstTexture, firstPresentationTimeUs); .maybeQueueInputFrame(firstTexture, firstPresentationTimeUs);
verify(fakeNextGlTextureProcessor, times(1)) verify(fakeConsumingGlTextureProcessor, times(1))
.maybeQueueInputFrame(secondTexture, secondPresentationTimeUs); .maybeQueueInputFrame(secondTexture, secondPresentationTimeUs);
} }
@ -130,7 +119,7 @@ public final class ChainingGlTextureProcessorListenerTest {
chainingGlTextureProcessorListener.onCurrentOutputStreamEnded(); chainingGlTextureProcessorListener.onCurrentOutputStreamEnded();
Thread.sleep(EXECUTOR_WAIT_TIME_MS); Thread.sleep(EXECUTOR_WAIT_TIME_MS);
verify(fakeNextGlTextureProcessor, times(1)).signalEndOfCurrentInputStream(); verify(fakeConsumingGlTextureProcessor, times(1)).signalEndOfCurrentInputStream();
} }
private static class FakeGlTextureProcessor implements GlTextureProcessor { private static class FakeGlTextureProcessor implements GlTextureProcessor {
@ -142,7 +131,17 @@ public final class ChainingGlTextureProcessorListenerTest {
} }
@Override @Override
public void setListener(Listener listener) { public void setInputListener(InputListener inputListener) {
throw new UnsupportedOperationException();
}
@Override
public void setOutputListener(OutputListener outputListener) {
throw new UnsupportedOperationException();
}
@Override
public void setErrorListener(ErrorListener errorListener) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }