loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract FrameProcessor interface from GlEffectsFrameProcessor.

Browse Source 
PiperOrigin-RevId: 456814150
This commit is contained in:
hschlueter 2022-06-23 19:00:01 +01:00 committed by Ian Baker
parent 938d3c2e5b
commit f3893c146d
8 changed files with 107 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
View File

@ -355,7 +355,7 @@ public final class GlEffectsFrameProcessorPixelTest {
checkNotNull(
GlEffectsFrameProcessor.create(
context,
new GlEffectsFrameProcessor.Listener() {
new FrameProcessor.Listener() {
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
frameProcessingException.set(exception);

7
libraries/transformer/src/main/java/androidx/media3/transformer/ChainingGlTextureProcessorListener.java
View File

@ -31,7 +31,7 @@ import java.util.Queue;
@Nullable private final GlTextureProcessor previousGlTextureProcessor;
@Nullable private final GlTextureProcessor nextGlTextureProcessor;
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
private final GlEffectsFrameProcessor.Listener frameProcessorListener;
private final FrameProcessor.Listener frameProcessorListener;
private final Queue<Pair<TextureInfo, Long>> pendingFrames;
/**
@ -45,14 +45,13 @@ import java.util.Queue;
* OpenGL calls. All calls to the previous/next {@link GlTextureProcessor} will be executed by
* the {@link FrameProcessingTaskExecutor}. The caller is responsible for releasing the {@link
* FrameProcessingTaskExecutor}.
* @param frameProcessorListener The {@link GlEffectsFrameProcessor.Listener} to forward
* exceptions to.
* @param frameProcessorListener The {@link FrameProcessor.Listener} to forward exceptions to.
*/
public ChainingGlTextureProcessorListener(
@Nullable GlTextureProcessor previousGlTextureProcessor,
@Nullable GlTextureProcessor nextGlTextureProcessor,
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
GlEffectsFrameProcessor.Listener frameProcessorListener) {
FrameProcessor.Listener frameProcessorListener) {
this.previousGlTextureProcessor = previousGlTextureProcessor;
this.nextGlTextureProcessor = nextGlTextureProcessor;
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;

8
libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java
View File

@ -47,7 +47,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* dimensions specified by the provided {@link SurfaceInfo}.
*
* <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link
* GlTextureProcessor} instances used by {@link GlEffectsFrameProcessor}.
* GlTextureProcessor} instances used by {@link FrameProcessor}.
*/
/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor {
@ -60,7 +60,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final SurfaceInfo.Provider outputSurfaceProvider;
private final long streamOffsetUs;
private final Transformer.DebugViewProvider debugViewProvider;
private final GlEffectsFrameProcessor.Listener frameProcessorListener;
private final FrameProcessor.Listener frameProcessorListener;
private final boolean enableExperimentalHdrEditing;
private int inputWidth;
@ -78,7 +78,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ImmutableList<GlMatrixTransformation> matrixTransformations,
SurfaceInfo.Provider outputSurfaceProvider,
long streamOffsetUs,
GlEffectsFrameProcessor.Listener frameProcessorListener,
FrameProcessor.Listener frameProcessorListener,
Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing) {
this.context = context;
@ -97,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*
* <p>The {@code FinalMatrixTransformationProcessorWrapper} will only call {@link
* Listener#onInputFrameProcessed(TextureInfo)}. Other events are handled via the {@link
* GlEffectsFrameProcessor.Listener} passed to the constructor.
* FrameProcessor.Listener} passed to the constructor.
*/
@Override
public void setListener(Listener listener) {

8
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingTaskExecutor.java
View File

@ -30,19 +30,19 @@ import java.util.concurrent.atomic.AtomicBoolean;
* instances.
*
* <p>The wrapper handles calling {@link
* GlEffectsFrameProcessor.Listener#onFrameProcessingError(FrameProcessingException)} for errors
* that occur during these tasks.
* FrameProcessor.Listener#onFrameProcessingError(FrameProcessingException)} for errors that occur
* during these tasks.
*/
/* package */ final class FrameProcessingTaskExecutor {
private final ExecutorService singleThreadExecutorService;
private final GlEffectsFrameProcessor.Listener listener;
private final FrameProcessor.Listener listener;
private final ConcurrentLinkedQueue<Future<?>> futures;
private final AtomicBoolean shouldCancelTasks;
/** Creates a new instance. */
public FrameProcessingTaskExecutor(
ExecutorService singleThreadExecutorService, GlEffectsFrameProcessor.Listener listener) {
ExecutorService singleThreadExecutorService, FrameProcessor.Listener listener) {
this.singleThreadExecutorService = singleThreadExecutorService;
this.listener = listener;

77
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import android.view.Surface;
/** Interface for a frame processor that applies changes to individual video frames. */
/* package */ interface FrameProcessor {
/**
* Listener for asynchronous frame processing events.
*
* <p>All listener methods must be called from the same thread.
*/
interface Listener {
/**
* Called when an exception occurs during asynchronous frame processing.
*
* <p>If an error occurred, consuming and producing further frames will not work as expected and
* the {@link FrameProcessor} should be released.
*/
void onFrameProcessingError(FrameProcessingException exception);
/** Called after the {@link FrameProcessor} has produced its final output frame. */
void onFrameProcessingEnded();
}
/** Returns the input {@link Surface}. */
Surface getInputSurface();
/**
* Informs the {@code FrameProcessor} that a frame will be queued to its input surface.
*
* <p>Must be called before rendering a frame to the frame processor's input surface.
*
* @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
*/
void registerInputFrame();
/**
* Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
* but not processed off the {@linkplain #getInputSurface() input surface} yet.
*/
int getPendingInputFrameCount();
/**
* Informs the {@code FrameProcessor} that no further input frames should be accepted.
*
* @throws IllegalStateException If called more than once.
*/
void signalEndOfInputStream();
/**
* Releases all resources.
*
* <p>If the frame processor is released before it has {@linkplain
* Listener#onFrameProcessingEnded() ended}, it will attempt to cancel processing any input frames
* that have already become available. Input frames that become available after release are
* ignored.
*
* <p>This method blocks until all resources are released or releasing times out.
*/
void release();
}

75
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
View File

@ -37,37 +37,12 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
/**
* {@code GlEffectsFrameProcessor} applies changes to individual video frames.
*
* <p>Input becomes available on its {@linkplain #getInputSurface() input surface} asynchronously
* and is processed on a background thread as it becomes available. All input frames should be
* {@linkplain #registerInputFrame() registered} before they are rendered to the input surface.
* {@link #getPendingInputFrameCount()} can be used to check whether there are frames that have not
* been fully processed yet. Output is written to the provided {@linkplain #create(Context,
* Listener, float, int, int, long, List, SurfaceInfo.Provider, Transformer.DebugViewProvider,
* boolean) output surface}.
* A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a
* background thread.
*/
// TODO(b/227625423): Factor out FrameProcessor interface
/* package */ final class GlEffectsFrameProcessor {
/**
* Listener for asynchronous frame processing events.
*
* <p>This listener is only called from the {@link GlEffectsFrameProcessor} instance's background
* thread.
*/
public interface Listener {
/**
* Called when an exception occurs during asynchronous frame processing.
*
* <p>If an error occurred, consuming and producing further frames will not work as expected and
* the {@link GlEffectsFrameProcessor} should be released.
*/
void onFrameProcessingError(FrameProcessingException exception);
/** Called after the frame processor has produced its final output frame. */
void onFrameProcessingEnded();
}
/* package */ final class GlEffectsFrameProcessor implements FrameProcessor {
// TODO(b/227625423): Replace factory method with setters once output surface and effects can be
// replaced.
/**
* Creates a new instance.
@ -89,7 +64,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public static GlEffectsFrameProcessor create(
Context context,
GlEffectsFrameProcessor.Listener listener,
FrameProcessor.Listener listener,
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
@ -141,7 +116,7 @@ import java.util.concurrent.atomic.AtomicInteger;
@WorkerThread
private static GlEffectsFrameProcessor createOpenGlObjectsAndFrameProcessor(
Context context,
GlEffectsFrameProcessor.Listener listener,
FrameProcessor.Listener listener,
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
@ -245,7 +220,7 @@ import java.util.concurrent.atomic.AtomicInteger;
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder,
SurfaceInfo.Provider outputSurfaceProvider,
long streamOffsetUs,
GlEffectsFrameProcessor.Listener listener,
FrameProcessor.Listener listener,
Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing)
throws FrameProcessingException {
@ -290,7 +265,7 @@ import java.util.concurrent.atomic.AtomicInteger;
ExternalTextureProcessor externalTextureProcessor,
ImmutableList<GlTextureProcessor> textureProcessors,
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
GlEffectsFrameProcessor.Listener listener) {
FrameProcessor.Listener listener) {
externalTextureProcessor.setListener(
new ChainingGlTextureProcessorListener(
/* previousGlTextureProcessor= */ null,
@ -366,7 +341,7 @@ import java.util.concurrent.atomic.AtomicInteger;
inputSurfaceTextureTransformMatrix = new float[16];
}
/** Returns the input {@link Surface}. */
@Override
public Surface getInputSurface() {
// TODO(b/227625423): Allow input surface to be recreated for input size change.
inputSurfaceTexture.setOnFrameAvailableListener(
@ -374,47 +349,25 @@ import java.util.concurrent.atomic.AtomicInteger;
return inputSurface;
}
/**
* Informs the {@code GlEffectsFrameProcessor} that a frame will be queued to its input surface.
*
* <p>Must be called before rendering a frame to the frame processor's input surface.
*
* @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
*/
@Override
public void registerInputFrame() {
checkState(!inputStreamEnded);
pendingInputFrameCount.incrementAndGet();
}
/**
* Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
* but not processed off the {@linkplain #getInputSurface() input surface} yet.
*/
@Override
public int getPendingInputFrameCount() {
return pendingInputFrameCount.get();
}
/**
* Informs the {@code GlEffectsFrameProcessor} that no further input frames should be accepted.
*
* @throws IllegalStateException If called more than once.
*/
@Override
public void signalEndOfInputStream() {
checkState(!inputStreamEnded);
inputStreamEnded = true;
frameProcessingTaskExecutor.submit(this::processEndOfInputStream);
}
/**
* Releases all resources.
*
* <p>If the frame processor is released before it has {@linkplain
* Listener#onFrameProcessingEnded() ended}, it will attempt to cancel processing any input frames
* that have already become available. Input frames that become available after release are
* ignored.
*
* <p>This method blocks until all resources are released or releasing times out.
*/
@Override
public void release() {
try {
frameProcessingTaskExecutor.release(

4
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -44,7 +44,7 @@ import org.checkerframework.dataflow.qual.Pure;
private final Codec decoder;
private final ArrayList<Long> decodeOnlyPresentationTimestamps;
private final GlEffectsFrameProcessor frameProcessor;
private final FrameProcessor frameProcessor;
private final EncoderWrapper encoderWrapper;
private final DecoderInputBuffer encoderOutputBuffer;
@ -102,7 +102,7 @@ import org.checkerframework.dataflow.qual.Pure;
frameProcessor =
GlEffectsFrameProcessor.create(
context,
new GlEffectsFrameProcessor.Listener() {
new FrameProcessor.Listener() {
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
asyncErrorListener.onTransformationException(

4
libraries/transformer/src/test/java/androidx/media3/transformer/ChainingGlTextureProcessorListenerTest.java
View File

@ -31,8 +31,8 @@ import org.junit.runner.RunWith;
public final class ChainingGlTextureProcessorListenerTest {
private static final long EXECUTOR_WAIT_TIME_MS = 100;
private final GlEffectsFrameProcessor.Listener mockframeProcessorListener =
mock(GlEffectsFrameProcessor.Listener.class);
private final FrameProcessor.Listener mockframeProcessorListener =
mock(FrameProcessor.Listener.class);
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor(
Util.newSingleThreadExecutor("Test"), mockframeProcessorListener);