loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add listener for FrameProcessingExceptions.

Browse Source 
This listener replaces
FrameProcessorChain#getAndRethrowBackgroundExceptions.
The listener uses a new exception type FrameProcessingException
separate from TransformationException as the frame processing
components will be made reusable outside of transformer soon.

PiperOrigin-RevId: 447455746
This commit is contained in:
hschlueter 2022-05-09 14:46:53 +01:00 committed by Ian Baker
parent 1b15d5c370
commit 63dcdf5803
14 changed files with 373 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java
View File

@ -31,6 +31,7 @@ import android.util.Size;
import androidx.media3.common.C;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.GlFrameProcessor;
import java.io.IOException;
import java.util.Locale;
@ -116,28 +117,32 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void drawFrame(long presentationTimeUs) {
checkStateNotNull(glProgram);
glProgram.use();
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use();
// Draw to the canvas and store it in a texture.
String text =
String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND);
overlayBitmap.eraseColor(Color.TRANSPARENT);
overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint);
overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId);
GLUtils.texSubImage2D(
GLES20.GL_TEXTURE_2D,
/* level= */ 0,
/* xoffset= */ 0,
/* yoffset= */ 0,
flipBitmapVertically(overlayBitmap));
GlUtil.checkGlError();
// Draw to the canvas and store it in a texture.
String text =
String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND);
overlayBitmap.eraseColor(Color.TRANSPARENT);
overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint);
overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId);
GLUtils.texSubImage2D(
GLES20.GL_TEXTURE_2D,
/* level= */ 0,
/* xoffset= */ 0,
/* yoffset= */ 0,
flipBitmapVertically(overlayBitmap));
GlUtil.checkGlError();
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
@Override

22
demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java
View File

@ -23,6 +23,7 @@ import android.opengl.GLES20;
import android.util.Size;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.GlFrameProcessor;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -98,14 +99,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void drawFrame(long presentationTimeUs) {
checkStateNotNull(glProgram).use();
double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US;
float innerRadius = minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta));
glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius});
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use();
double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US;
float innerRadius =
minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta));
glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius});
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
@Override

7
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java
View File

@ -26,6 +26,7 @@ import androidx.media3.common.util.ConditionVariable;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.LibraryLoader;
import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.GlFrameProcessor;
import com.google.mediapipe.components.FrameProcessor;
import com.google.mediapipe.framework.AppTextureFrame;
@ -112,7 +113,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void drawFrame(long presentationTimeUs) {
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
frameProcessorConditionVariable.close();
// Pass the input frame to MediaPipe.
@ -133,7 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
if (frameProcessorPendingError != null) {
throw new IllegalStateException(frameProcessorPendingError);
throw new FrameProcessingException(frameProcessorPendingError);
}
// Copy from MediaPipe's output texture to the current output.
@ -148,6 +149,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
} finally {
checkStateNotNull(outputFrame).release();
}

2
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
View File

@ -50,6 +50,8 @@ public final class GlUtil {
}
}
// TODO(b/231937416): Consider removing this flag, enabling assertions by default, and making
// GlException checked.
/** Whether to throw a {@link GlException} in case of an OpenGL error. */
public static boolean glAssertionsEnabled = false;

42
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
@ -36,6 +37,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.MimeTypes;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
import org.junit.Test;
@ -78,6 +80,9 @@ public final class FrameProcessorChainPixelTest {
/** The ratio of width over height, for each pixel in a frame. */
private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1;
private final AtomicReference<FrameProcessingException> frameProcessingException =
new AtomicReference<>();
private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
private @MonotonicNonNull ImageReader outputImageReader;
private @MonotonicNonNull MediaFormat mediaFormat;
@ -229,6 +234,15 @@ public final class FrameProcessorChainPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void processData_withFrameProcessingException_callsListener() throws Exception {
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, ThrowingFrameProcessor::new);
Thread.sleep(FRAME_PROCESSING_WAIT_MS);
assertThat(frameProcessingException.get()).isNotNull();
}
/**
* Set up and prepare the first frame from an input video, as well as relevant test
* infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
@ -258,6 +272,7 @@ public final class FrameProcessorChainPixelTest {
frameProcessorChain =
FrameProcessorChain.create(
context,
/* listener= */ this.frameProcessingException::set,
pixelWidthHeightRatio,
inputWidth,
inputHeight,
@ -321,11 +336,11 @@ public final class FrameProcessorChainPixelTest {
}
}
private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException {
private Bitmap processFirstFrameAndEnd() throws InterruptedException {
checkNotNull(frameProcessorChain).signalEndOfInputStream();
Thread.sleep(FRAME_PROCESSING_WAIT_MS);
assertThat(frameProcessorChain.isEnded()).isTrue();
frameProcessorChain.getAndRethrowBackgroundExceptions();
assertThat(frameProcessingException.get()).isNull();
Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage();
Bitmap actualBitmap =
@ -333,4 +348,27 @@ public final class FrameProcessorChainPixelTest {
frameProcessorChainOutputImage.close();
return actualBitmap;
}
private static class ThrowingFrameProcessor implements GlFrameProcessor {
private @MonotonicNonNull Size outputSize;
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) {
outputSize = new Size(inputWidth, inputHeight);
}
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
}
@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
throw new FrameProcessingException("An exception occurred.");
}
@Override
public void release() {}
}
}

13
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
View File

@ -23,6 +23,7 @@ import android.util.Size;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -33,6 +34,8 @@ import org.junit.runner.RunWith;
*/
@RunWith(AndroidJUnit4.class)
public final class FrameProcessorChainTest {
private final AtomicReference<FrameProcessingException> frameProcessingException =
new AtomicReference<>();
@Test
public void getOutputSize_noOperation_returnsInputSize() throws Exception {
@ -46,6 +49,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(inputSize);
assertThat(frameProcessingException.get()).isNull();
}
@Test
@ -60,6 +64,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(new Size(400, 100));
assertThat(frameProcessingException.get()).isNull();
}
@Test
@ -74,6 +79,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(new Size(200, 200));
assertThat(frameProcessingException.get()).isNull();
}
@Test
@ -89,6 +95,7 @@ public final class FrameProcessorChainTest {
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize);
assertThat(frameProcessingException.get()).isNull();
}
@Test
@ -107,17 +114,19 @@ public final class FrameProcessorChainTest {
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3);
assertThat(frameProcessingException.get()).isNull();
}
private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
private FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes)
throws TransformationException {
throws FrameProcessingException {
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
for (Size element : frameProcessorOutputSizes) {
effects.add(() -> new FakeFrameProcessor(element));
}
return FrameProcessorChain.create(
getApplicationContext(),
/* listener= */ this.frameProcessingException::set,
pixelWidthHeightRatio,
inputSize.getWidth(),
inputSize.getHeight(),

15
libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java
View File

@ -104,12 +104,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void drawFrame(long presentationTimeUs) {
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram);
glProgram.use();
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
try {
glProgram.use();
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
@Override

92
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingException.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import androidx.media3.common.C;
import androidx.media3.common.util.UnstableApi;
/** Thrown when an exception occurs while applying effects to video frames. */
@UnstableApi
public final class FrameProcessingException extends Exception {
/**
* The microsecond timestamp of the frame being processed while the exception occurred or {@link
* C#TIME_UNSET} if unknown.
*/
public final long presentationTimeUs;
/**
* Creates an instance.
*
* @param message The detail message for this exception.
*/
public FrameProcessingException(String message) {
this(message, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(String message, long presentationTimeUs) {
super(message);
this.presentationTimeUs = presentationTimeUs;
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param cause The cause of this exception.
*/
public FrameProcessingException(String message, Throwable cause) {
this(message, cause, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param cause The cause of this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(String message, Throwable cause, long presentationTimeUs) {
super(message, cause);
this.presentationTimeUs = presentationTimeUs;
}
/**
* Creates an instance.
*
* @param cause The cause of this exception.
*/
public FrameProcessingException(Throwable cause) {
this(cause, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param cause The cause of this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(Throwable cause, long presentationTimeUs) {
super(cause);
this.presentationTimeUs = presentationTimeUs;
}
}

201
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
View File

@ -47,6 +47,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@ -67,10 +68,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GlUtil.glAssertionsEnabled = true;
}
/**
* Listener for asynchronous frame processing events.
*
* <p>This listener is only called from the {@link FrameProcessorChain}'s background thread.
*/
public interface Listener {
/** Called when an exception occurs during asynchronous frame processing. */
void onFrameProcessingError(FrameProcessingException exception);
}
/**
* Creates a new instance.
*
* @param context A {@link Context}.
* @param listener A {@link Listener}.
* @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded
* by this ratio so that the output frame's pixels have a ratio of 1.
* @param inputWidth The input frame width, in pixels.
@ -78,17 +90,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @return A new instance.
* @throws TransformationException If reading shader files fails, or an OpenGL error occurs while
* @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while
* creating and configuring the OpenGL components.
*/
public static FrameProcessorChain create(
Context context,
Listener listener,
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
List<GlEffect> effects,
boolean enableExperimentalHdrEditing)
throws TransformationException {
throws FrameProcessingException {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
@ -100,6 +113,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
() ->
createOpenGlObjectsAndFrameProcessorChain(
context,
listener,
pixelWidthHeightRatio,
inputWidth,
inputHeight,
@ -108,12 +122,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
singleThreadExecutorService))
.get();
} catch (ExecutionException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
throw new FrameProcessingException(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
throw new FrameProcessingException(e);
}
}
@ -127,6 +139,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@WorkerThread
private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
Context context,
Listener listener,
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
@ -177,6 +190,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
inputExternalTexId,
framebuffers,
frameProcessors,
listener,
enableExperimentalHdrEditing);
}
@ -241,6 +255,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*/
private final int[] framebuffers;
private final Listener listener;
/**
* Prevents further frame processing tasks from being scheduled or executed after {@link
* #release()} is called or an exception occurred.
*/
private final AtomicBoolean stopProcessing;
private int outputWidth;
private int outputHeight;
/**
@ -258,8 +279,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private @MonotonicNonNull EGLSurface debugPreviewEglSurface;
private boolean inputStreamEnded;
/** Prevents further frame processing tasks from being scheduled after {@link #release()}. */
private volatile boolean releaseRequested;
private FrameProcessorChain(
EGLDisplay eglDisplay,
@ -268,6 +287,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
int inputExternalTexId,
int[] framebuffers,
ImmutableList<GlFrameProcessor> frameProcessors,
Listener listener,
boolean enableExperimentalHdrEditing) {
checkState(!frameProcessors.isEmpty());
@ -276,6 +296,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
this.singleThreadExecutorService = singleThreadExecutorService;
this.framebuffers = framebuffers;
this.frameProcessors = frameProcessors;
this.listener = listener;
this.stopProcessing = new AtomicBoolean();
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
futures = new ConcurrentLinkedQueue<>();
@ -331,7 +353,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
inputSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> {
if (releaseRequested) {
if (stopProcessing.get()) {
// Frames can still become available after a transformation is cancelled but they can be
// ignored.
return;
@ -339,7 +361,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
try {
futures.add(singleThreadExecutorService.submit(this::processFrame));
} catch (RejectedExecutionException e) {
if (!releaseRequested) {
if (!stopProcessing.get()) {
throw e;
}
}
@ -371,28 +393,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return pendingFrameCount.get();
}
/**
* Checks whether any exceptions occurred during asynchronous frame processing and rethrows the
* first exception encountered.
*/
public void getAndRethrowBackgroundExceptions() throws TransformationException {
@Nullable Future<?> oldestGlProcessingFuture = futures.peek();
while (oldestGlProcessingFuture != null && oldestGlProcessingFuture.isDone()) {
futures.poll();
try {
oldestGlProcessingFuture.get();
} catch (ExecutionException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
}
oldestGlProcessingFuture = futures.peek();
}
}
/** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */
public void signalEndOfInputStream() {
inputStreamEnded = true;
@ -413,18 +413,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* <p>This method blocks until all OpenGL resources are released or releasing times out.
*/
public void release() {
releaseRequested = true;
stopProcessing.set(true);
while (!futures.isEmpty()) {
checkNotNull(futures.poll()).cancel(/* mayInterruptIfRunning= */ true);
}
futures.add(
singleThreadExecutorService.submit(
() -> {
for (int i = 0; i < frameProcessors.size(); i++) {
frameProcessors.get(i).release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}));
singleThreadExecutorService.submit(this::releaseFrameProcessorsAndDestroyGlContext));
if (inputSurfaceTexture != null) {
inputSurfaceTexture.release();
}
@ -448,22 +442,26 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*/
@WorkerThread
private void createOpenGlSurfaces(Surface outputSurface, @Nullable SurfaceView debugSurfaceView) {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
checkStateNotNull(eglDisplay);
try {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
checkStateNotNull(eglDisplay);
if (enableExperimentalHdrEditing) {
// TODO(b/209404935): Don't assume BT.2020 PQ input/output.
eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface);
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
}
} else {
eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
if (enableExperimentalHdrEditing) {
// TODO(b/209404935): Don't assume BT.2020 PQ input/output.
eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface);
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
}
} else {
eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
}
}
} catch (RuntimeException e) {
listener.onFrameProcessingError(new FrameProcessingException(e));
}
}
@ -475,44 +473,58 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@WorkerThread
@RequiresNonNull("inputSurfaceTexture")
private void processFrame() {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
checkStateNotNull(eglSurface, "No output surface set.");
inputSurfaceTexture.updateTexImage();
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
long presentationTimeUs = presentationTimeNs / 1000;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
((ExternalCopyFrameProcessor) frameProcessors.get(0))
.setTextureTransformMatrix(textureTransformMatrix);
for (int i = 0; i < frameProcessors.size() - 1; i++) {
Size intermediateSize = frameProcessors.get(i).getOutputSize();
GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
eglSurface,
framebuffers[i],
intermediateSize.getWidth(),
intermediateSize.getHeight());
clearOutputFrame();
frameProcessors.get(i).drawFrame(presentationTimeUs);
if (stopProcessing.get()) {
return;
}
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
clearOutputFrame();
getLast(frameProcessors).drawFrame(presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
long presentationTimeUs = C.TIME_UNSET;
try {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
checkStateNotNull(eglSurface, "No output surface set.");
if (debugPreviewEglSurface != null) {
GlUtil.focusEglSurface(
eglDisplay, eglContext, debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight);
inputSurfaceTexture.updateTexImage();
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
presentationTimeUs = presentationTimeNs / 1000;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
((ExternalCopyFrameProcessor) frameProcessors.get(0))
.setTextureTransformMatrix(textureTransformMatrix);
for (int i = 0; i < frameProcessors.size() - 1; i++) {
Size intermediateSize = frameProcessors.get(i).getOutputSize();
GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
eglSurface,
framebuffers[i],
intermediateSize.getWidth(),
intermediateSize.getHeight());
clearOutputFrame();
frameProcessors.get(i).drawFrame(presentationTimeUs);
}
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
clearOutputFrame();
getLast(frameProcessors).drawFrame(presentationTimeUs);
EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface);
}
checkState(pendingFrameCount.getAndDecrement() > 0);
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
if (debugPreviewEglSurface != null) {
GlUtil.focusEglSurface(
eglDisplay, eglContext, debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight);
clearOutputFrame();
getLast(frameProcessors).drawFrame(presentationTimeUs);
EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface);
}
checkState(pendingFrameCount.getAndDecrement() > 0);
} catch (FrameProcessingException | RuntimeException e) {
if (!stopProcessing.getAndSet(true)) {
listener.onFrameProcessingError(
e instanceof FrameProcessingException
? (FrameProcessingException) e
: new FrameProcessingException(e, presentationTimeUs));
}
}
}
private static void clearOutputFrame() {
@ -520,4 +532,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GlUtil.checkGlError();
}
/**
* Releases the {@link GlFrameProcessor GlFrameProcessors} and destroys the OpenGL context.
*
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
*/
@WorkerThread
private void releaseFrameProcessorsAndDestroyGlContext() {
try {
for (int i = 0; i < frameProcessors.size(); i++) {
frameProcessors.get(i).release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
} catch (RuntimeException e) {
listener.onFrameProcessingError(new FrameProcessingException(e));
}
}
}

4
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
View File

@ -46,6 +46,7 @@ public interface GlFrameProcessor {
* @param inputTexId Identifier of a 2D OpenGL texture.
* @param inputWidth The input width, in pixels.
* @param inputHeight The input height, in pixels.
* @throws IOException If an error occurs while reading resources.
*/
void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException;
@ -69,8 +70,9 @@ public interface GlFrameProcessor {
* program's vertex attributes and uniforms, and issue a drawing command.
*
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
* @throws FrameProcessingException If an error occurs while processing or drawing the frame.
*/
void drawFrame(long presentationTimeUs);
void drawFrame(long presentationTimeUs) throws FrameProcessingException;
/** Releases all resources. */
void release();

24
libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java
View File

@ -97,16 +97,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void drawFrame(long presentationTimeUs) {
checkStateNotNull(glProgram).use();
float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs);
checkState(
transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements");
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use();
float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs);
checkState(
transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements");
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
@Override

26
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -721,6 +721,8 @@ public final class Transformer {
DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10,
DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10)
.build();
TransformerPlayerListener playerListener =
new TransformerPlayerListener(mediaItem, muxerWrapper, looper);
ExoPlayer.Builder playerBuilder =
new ExoPlayer.Builder(
context,
@ -734,6 +736,7 @@ public final class Transformer {
encoderFactory,
decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest),
playerListener,
debugViewProvider))
.setMediaSourceFactory(mediaSourceFactory)
.setTrackSelector(trackSelector)
@ -748,7 +751,7 @@ public final class Transformer {
player = playerBuilder.build();
player.setMediaItem(mediaItem);
player.addListener(new TransformerPlayerListener(mediaItem, muxerWrapper));
player.addListener(playerListener);
player.prepare();
progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
@ -846,6 +849,7 @@ public final class Transformer {
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener;
private final FrameProcessorChain.Listener frameProcessorChainListener;
private final Transformer.DebugViewProvider debugViewProvider;
public TransformerRenderersFactory(
@ -858,6 +862,7 @@ public final class Transformer {
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider) {
this.context = context;
this.muxerWrapper = muxerWrapper;
@ -868,6 +873,7 @@ public final class Transformer {
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener;
this.frameProcessorChainListener = frameProcessorChainListener;
this.debugViewProvider = debugViewProvider;
mediaClock = new TransformerMediaClock();
}
@ -904,6 +910,7 @@ public final class Transformer {
encoderFactory,
decoderFactory,
fallbackListener,
frameProcessorChainListener,
debugViewProvider);
index++;
}
@ -911,14 +918,18 @@ public final class Transformer {
}
}
private final class TransformerPlayerListener implements Player.Listener {
private final class TransformerPlayerListener
implements Player.Listener, FrameProcessorChain.Listener {
private final MediaItem mediaItem;
private final MuxerWrapper muxerWrapper;
private final Handler handler;
public TransformerPlayerListener(MediaItem mediaItem, MuxerWrapper muxerWrapper) {
public TransformerPlayerListener(
MediaItem mediaItem, MuxerWrapper muxerWrapper, Looper looper) {
this.mediaItem = mediaItem;
this.muxerWrapper = muxerWrapper;
handler = new Handler(looper);
}
@Override
@ -1013,5 +1024,14 @@ public final class Transformer {
}
listeners.flushEvents();
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
handler.post(
() ->
handleTransformationEnded(
TransformationException.createForFrameProcessorChain(
exception, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED)));
}
}
}

4
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
View File

@ -38,6 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final ImmutableList<GlEffect> effects;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final FrameProcessorChain.Listener frameProcessorChainListener;
private final Transformer.DebugViewProvider debugViewProvider;
private final DecoderInputBuffer decoderInputBuffer;
@ -52,12 +53,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
this.context = context;
this.effects = effects;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.frameProcessorChainListener = frameProcessorChainListener;
this.debugViewProvider = debugViewProvider;
decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
@ -95,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
fallbackListener,
frameProcessorChainListener,
debugViewProvider);
}
if (transformationRequest.flattenForSlowMotion) {

24
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -55,6 +55,7 @@ import org.checkerframework.dataflow.qual.Pure;
Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes,
FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider)
throws TransformationException {
decoderInputBuffer =
@ -86,14 +87,20 @@ import org.checkerframework.dataflow.qual.Pure;
EncoderCompatibilityTransformation encoderCompatibilityTransformation =
new EncoderCompatibilityTransformation();
effectsListBuilder.add(encoderCompatibilityTransformation);
frameProcessorChain =
FrameProcessorChain.create(
context,
inputFormat.pixelWidthHeightRatio,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
effectsListBuilder.build(),
transformationRequest.enableHdrEditing);
try {
frameProcessorChain =
FrameProcessorChain.create(
context,
frameProcessorChainListener,
inputFormat.pixelWidthHeightRatio,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
effectsListBuilder.build(),
transformationRequest.enableHdrEditing);
} catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = encoderCompatibilityTransformation.getOutputRotationDegrees();
@ -146,7 +153,6 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public boolean processData() throws TransformationException {
frameProcessorChain.getAndRethrowBackgroundExceptions();
if (frameProcessorChain.isEnded()) {
if (!signaledEndOfStreamToEncoder) {
encoder.signalEndOfInputStream();