From 63dcdf58033963f06056358a0884ebfb891a1109 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Mon, 9 May 2022 14:46:53 +0100 Subject: [PATCH] Add listener for FrameProcessingExceptions. This listener replaces FrameProcessorChain#getAndRethrowBackgroundExceptions. The listener uses a new exception type FrameProcessingException separate from TransformationException as the frame processing components will be made reusable outside of transformer soon. PiperOrigin-RevId: 447455746 --- .../BitmapOverlayFrameProcessor.java | 45 ++-- .../PeriodicVignetteFrameProcessor.java | 22 +- .../transformer/MediaPipeFrameProcessor.java | 7 +- .../androidx/media3/common/util/GlUtil.java | 2 + .../FrameProcessorChainPixelTest.java | 42 +++- .../transformer/FrameProcessorChainTest.java | 13 +- .../ExternalCopyFrameProcessor.java | 15 +- .../transformer/FrameProcessingException.java | 92 ++++++++ .../transformer/FrameProcessorChain.java | 201 ++++++++++-------- .../media3/transformer/GlFrameProcessor.java | 4 +- .../MatrixTransformationFrameProcessor.java | 24 ++- .../media3/transformer/Transformer.java | 26 ++- .../transformer/TransformerVideoRenderer.java | 4 + .../VideoTranscodingSamplePipeline.java | 24 ++- 14 files changed, 373 insertions(+), 148 deletions(-) create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingException.java diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java index e4eba6e3bf..5de56cd0e0 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java @@ -31,6 +31,7 @@ import android.util.Size; import androidx.media3.common.C; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; +import androidx.media3.transformer.FrameProcessingException; import androidx.media3.transformer.GlFrameProcessor; import java.io.IOException; import java.util.Locale; @@ -116,28 +117,32 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void drawFrame(long presentationTimeUs) { - checkStateNotNull(glProgram); - glProgram.use(); + public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + try { + checkStateNotNull(glProgram).use(); - // Draw to the canvas and store it in a texture. - String text = - String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND); - overlayBitmap.eraseColor(Color.TRANSPARENT); - overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint); - overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId); - GLUtils.texSubImage2D( - GLES20.GL_TEXTURE_2D, - /* level= */ 0, - /* xoffset= */ 0, - /* yoffset= */ 0, - flipBitmapVertically(overlayBitmap)); - GlUtil.checkGlError(); + // Draw to the canvas and store it in a texture. + String text = + String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND); + overlayBitmap.eraseColor(Color.TRANSPARENT); + overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint); + overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId); + GLUtils.texSubImage2D( + GLES20.GL_TEXTURE_2D, + /* level= */ 0, + /* xoffset= */ 0, + /* yoffset= */ 0, + flipBitmapVertically(overlayBitmap)); + GlUtil.checkGlError(); - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } } @Override diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java index 760c681d97..1b91e10687 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java @@ -23,6 +23,7 @@ import android.opengl.GLES20; import android.util.Size; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; +import androidx.media3.transformer.FrameProcessingException; import androidx.media3.transformer.GlFrameProcessor; import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -98,14 +99,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void drawFrame(long presentationTimeUs) { - checkStateNotNull(glProgram).use(); - double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; - float innerRadius = minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); - glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius}); - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + try { + checkStateNotNull(glProgram).use(); + double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; + float innerRadius = + minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); + glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius}); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } } @Override diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java index 163fa91cad..49651c9c0d 100644 --- a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java @@ -26,6 +26,7 @@ import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.LibraryLoader; +import androidx.media3.transformer.FrameProcessingException; import androidx.media3.transformer.GlFrameProcessor; import com.google.mediapipe.components.FrameProcessor; import com.google.mediapipe.framework.AppTextureFrame; @@ -112,7 +113,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void drawFrame(long presentationTimeUs) { + public void drawFrame(long presentationTimeUs) throws FrameProcessingException { frameProcessorConditionVariable.close(); // Pass the input frame to MediaPipe. @@ -133,7 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } if (frameProcessorPendingError != null) { - throw new IllegalStateException(frameProcessorPendingError); + throw new FrameProcessingException(frameProcessorPendingError); } // Copy from MediaPipe's output texture to the current output. @@ -148,6 +149,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; glProgram.bindAttributesAndUniforms(); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); } finally { checkStateNotNull(outputFrame).release(); } diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index a28bca30f0..6aaaedbe39 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -50,6 +50,8 @@ public final class GlUtil { } } + // TODO(b/231937416): Consider removing this flag, enabling assertions by default, and making + // GlException checked. /** Whether to throw a {@link GlException} in case of an OpenGL error. */ public static boolean glAssertionsEnabled = false; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index 7e3ec259c6..cef6f29d1e 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; @@ -36,6 +37,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; import org.junit.Test; @@ -78,6 +80,9 @@ public final class FrameProcessorChainPixelTest { /** The ratio of width over height, for each pixel in a frame. */ private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1; + private final AtomicReference frameProcessingException = + new AtomicReference<>(); + private @MonotonicNonNull FrameProcessorChain frameProcessorChain; private @MonotonicNonNull ImageReader outputImageReader; private @MonotonicNonNull MediaFormat mediaFormat; @@ -229,6 +234,15 @@ public final class FrameProcessorChainPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void processData_withFrameProcessingException_callsListener() throws Exception { + setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, ThrowingFrameProcessor::new); + + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNotNull(); + } + /** * Set up and prepare the first frame from an input video, as well as relevant test * infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be @@ -258,6 +272,7 @@ public final class FrameProcessorChainPixelTest { frameProcessorChain = FrameProcessorChain.create( context, + /* listener= */ this.frameProcessingException::set, pixelWidthHeightRatio, inputWidth, inputHeight, @@ -321,11 +336,11 @@ public final class FrameProcessorChainPixelTest { } } - private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException { + private Bitmap processFirstFrameAndEnd() throws InterruptedException { checkNotNull(frameProcessorChain).signalEndOfInputStream(); Thread.sleep(FRAME_PROCESSING_WAIT_MS); assertThat(frameProcessorChain.isEnded()).isTrue(); - frameProcessorChain.getAndRethrowBackgroundExceptions(); + assertThat(frameProcessingException.get()).isNull(); Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage(); Bitmap actualBitmap = @@ -333,4 +348,27 @@ public final class FrameProcessorChainPixelTest { frameProcessorChainOutputImage.close(); return actualBitmap; } + + private static class ThrowingFrameProcessor implements GlFrameProcessor { + + private @MonotonicNonNull Size outputSize; + + @Override + public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) { + outputSize = new Size(inputWidth, inputHeight); + } + + @Override + public Size getOutputSize() { + return checkStateNotNull(outputSize); + } + + @Override + public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + throw new FrameProcessingException("An exception occurred."); + } + + @Override + public void release() {} + } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java index ae247be4a2..99c29fbfb5 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java @@ -23,6 +23,7 @@ import android.util.Size; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,6 +34,8 @@ import org.junit.runner.RunWith; */ @RunWith(AndroidJUnit4.class) public final class FrameProcessorChainTest { + private final AtomicReference frameProcessingException = + new AtomicReference<>(); @Test public void getOutputSize_noOperation_returnsInputSize() throws Exception { @@ -46,6 +49,7 @@ public final class FrameProcessorChainTest { Size outputSize = frameProcessorChain.getOutputSize(); assertThat(outputSize).isEqualTo(inputSize); + assertThat(frameProcessingException.get()).isNull(); } @Test @@ -60,6 +64,7 @@ public final class FrameProcessorChainTest { Size outputSize = frameProcessorChain.getOutputSize(); assertThat(outputSize).isEqualTo(new Size(400, 100)); + assertThat(frameProcessingException.get()).isNull(); } @Test @@ -74,6 +79,7 @@ public final class FrameProcessorChainTest { Size outputSize = frameProcessorChain.getOutputSize(); assertThat(outputSize).isEqualTo(new Size(200, 200)); + assertThat(frameProcessingException.get()).isNull(); } @Test @@ -89,6 +95,7 @@ public final class FrameProcessorChainTest { Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize); + assertThat(frameProcessingException.get()).isNull(); } @Test @@ -107,17 +114,19 @@ public final class FrameProcessorChainTest { Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3); + assertThat(frameProcessingException.get()).isNull(); } - private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors( + private FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors( float pixelWidthHeightRatio, Size inputSize, List frameProcessorOutputSizes) - throws TransformationException { + throws FrameProcessingException { ImmutableList.Builder effects = new ImmutableList.Builder<>(); for (Size element : frameProcessorOutputSizes) { effects.add(() -> new FakeFrameProcessor(element)); } return FrameProcessorChain.create( getApplicationContext(), + /* listener= */ this.frameProcessingException::set, pixelWidthHeightRatio, inputSize.getWidth(), inputSize.getHeight(), diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java index 05b291512e..83c3abffde 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java @@ -104,12 +104,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void drawFrame(long presentationTimeUs) { + public void drawFrame(long presentationTimeUs) throws FrameProcessingException { checkStateNotNull(glProgram); - glProgram.use(); - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + try { + glProgram.use(); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingException.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingException.java new file mode 100644 index 0000000000..6d413fd38a --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingException.java @@ -0,0 +1,92 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import androidx.media3.common.C; +import androidx.media3.common.util.UnstableApi; + +/** Thrown when an exception occurs while applying effects to video frames. */ +@UnstableApi +public final class FrameProcessingException extends Exception { + + /** + * The microsecond timestamp of the frame being processed while the exception occurred or {@link + * C#TIME_UNSET} if unknown. + */ + public final long presentationTimeUs; + + /** + * Creates an instance. + * + * @param message The detail message for this exception. + */ + public FrameProcessingException(String message) { + this(message, /* presentationTimeUs= */ C.TIME_UNSET); + } + + /** + * Creates an instance. + * + * @param message The detail message for this exception. + * @param presentationTimeUs The timestamp of the frame for which the exception occurred. + */ + public FrameProcessingException(String message, long presentationTimeUs) { + super(message); + this.presentationTimeUs = presentationTimeUs; + } + + /** + * Creates an instance. + * + * @param message The detail message for this exception. + * @param cause The cause of this exception. + */ + public FrameProcessingException(String message, Throwable cause) { + this(message, cause, /* presentationTimeUs= */ C.TIME_UNSET); + } + + /** + * Creates an instance. + * + * @param message The detail message for this exception. + * @param cause The cause of this exception. + * @param presentationTimeUs The timestamp of the frame for which the exception occurred. + */ + public FrameProcessingException(String message, Throwable cause, long presentationTimeUs) { + super(message, cause); + this.presentationTimeUs = presentationTimeUs; + } + + /** + * Creates an instance. + * + * @param cause The cause of this exception. + */ + public FrameProcessingException(Throwable cause) { + this(cause, /* presentationTimeUs= */ C.TIME_UNSET); + } + + /** + * Creates an instance. + * + * @param cause The cause of this exception. + * @param presentationTimeUs The timestamp of the frame for which the exception occurred. + */ + public FrameProcessingException(Throwable cause, long presentationTimeUs) { + super(cause); + this.presentationTimeUs = presentationTimeUs; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index a61d74699f..6512b5bdb9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -47,6 +47,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -67,10 +68,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; GlUtil.glAssertionsEnabled = true; } + /** + * Listener for asynchronous frame processing events. + * + *

This listener is only called from the {@link FrameProcessorChain}'s background thread. + */ + public interface Listener { + /** Called when an exception occurs during asynchronous frame processing. */ + void onFrameProcessingError(FrameProcessingException exception); + } + /** * Creates a new instance. * * @param context A {@link Context}. + * @param listener A {@link Listener}. * @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded * by this ratio so that the output frame's pixels have a ratio of 1. * @param inputWidth The input frame width, in pixels. @@ -78,17 +90,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * @param effects The {@link GlEffect GlEffects} to apply to each frame. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @return A new instance. - * @throws TransformationException If reading shader files fails, or an OpenGL error occurs while + * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while * creating and configuring the OpenGL components. */ public static FrameProcessorChain create( Context context, + Listener listener, float pixelWidthHeightRatio, int inputWidth, int inputHeight, List effects, boolean enableExperimentalHdrEditing) - throws TransformationException { + throws FrameProcessingException { checkArgument(inputWidth > 0, "inputWidth must be positive"); checkArgument(inputHeight > 0, "inputHeight must be positive"); @@ -100,6 +113,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; () -> createOpenGlObjectsAndFrameProcessorChain( context, + listener, pixelWidthHeightRatio, inputWidth, inputHeight, @@ -108,12 +122,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; singleThreadExecutorService)) .get(); } catch (ExecutionException e) { - throw TransformationException.createForFrameProcessorChain( - e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + throw new FrameProcessingException(e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw TransformationException.createForFrameProcessorChain( - e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + throw new FrameProcessingException(e); } } @@ -127,6 +139,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @WorkerThread private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain( Context context, + Listener listener, float pixelWidthHeightRatio, int inputWidth, int inputHeight, @@ -177,6 +190,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; inputExternalTexId, framebuffers, frameProcessors, + listener, enableExperimentalHdrEditing); } @@ -241,6 +255,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; */ private final int[] framebuffers; + private final Listener listener; + /** + * Prevents further frame processing tasks from being scheduled or executed after {@link + * #release()} is called or an exception occurred. + */ + private final AtomicBoolean stopProcessing; + private int outputWidth; private int outputHeight; /** @@ -258,8 +279,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private @MonotonicNonNull EGLSurface debugPreviewEglSurface; private boolean inputStreamEnded; - /** Prevents further frame processing tasks from being scheduled after {@link #release()}. */ - private volatile boolean releaseRequested; private FrameProcessorChain( EGLDisplay eglDisplay, @@ -268,6 +287,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; int inputExternalTexId, int[] framebuffers, ImmutableList frameProcessors, + Listener listener, boolean enableExperimentalHdrEditing) { checkState(!frameProcessors.isEmpty()); @@ -276,6 +296,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; this.singleThreadExecutorService = singleThreadExecutorService; this.framebuffers = framebuffers; this.frameProcessors = frameProcessors; + this.listener = listener; + this.stopProcessing = new AtomicBoolean(); this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; futures = new ConcurrentLinkedQueue<>(); @@ -331,7 +353,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; inputSurfaceTexture.setOnFrameAvailableListener( surfaceTexture -> { - if (releaseRequested) { + if (stopProcessing.get()) { // Frames can still become available after a transformation is cancelled but they can be // ignored. return; @@ -339,7 +361,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; try { futures.add(singleThreadExecutorService.submit(this::processFrame)); } catch (RejectedExecutionException e) { - if (!releaseRequested) { + if (!stopProcessing.get()) { throw e; } } @@ -371,28 +393,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return pendingFrameCount.get(); } - /** - * Checks whether any exceptions occurred during asynchronous frame processing and rethrows the - * first exception encountered. - */ - public void getAndRethrowBackgroundExceptions() throws TransformationException { - @Nullable Future oldestGlProcessingFuture = futures.peek(); - while (oldestGlProcessingFuture != null && oldestGlProcessingFuture.isDone()) { - futures.poll(); - try { - oldestGlProcessingFuture.get(); - } catch (ExecutionException e) { - throw TransformationException.createForFrameProcessorChain( - e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw TransformationException.createForFrameProcessorChain( - e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); - } - oldestGlProcessingFuture = futures.peek(); - } - } - /** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */ public void signalEndOfInputStream() { inputStreamEnded = true; @@ -413,18 +413,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; *

This method blocks until all OpenGL resources are released or releasing times out. */ public void release() { - releaseRequested = true; + stopProcessing.set(true); while (!futures.isEmpty()) { checkNotNull(futures.poll()).cancel(/* mayInterruptIfRunning= */ true); } futures.add( - singleThreadExecutorService.submit( - () -> { - for (int i = 0; i < frameProcessors.size(); i++) { - frameProcessors.get(i).release(); - } - GlUtil.destroyEglContext(eglDisplay, eglContext); - })); + singleThreadExecutorService.submit(this::releaseFrameProcessorsAndDestroyGlContext)); if (inputSurfaceTexture != null) { inputSurfaceTexture.release(); } @@ -448,22 +442,26 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; */ @WorkerThread private void createOpenGlSurfaces(Surface outputSurface, @Nullable SurfaceView debugSurfaceView) { - checkState(Thread.currentThread().getName().equals(THREAD_NAME)); - checkStateNotNull(eglDisplay); + try { + checkState(Thread.currentThread().getName().equals(THREAD_NAME)); + checkStateNotNull(eglDisplay); - if (enableExperimentalHdrEditing) { - // TODO(b/209404935): Don't assume BT.2020 PQ input/output. - eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface); - if (debugSurfaceView != null) { - debugPreviewEglSurface = - GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); - } - } else { - eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); - if (debugSurfaceView != null) { - debugPreviewEglSurface = - GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); + if (enableExperimentalHdrEditing) { + // TODO(b/209404935): Don't assume BT.2020 PQ input/output. + eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface); + if (debugSurfaceView != null) { + debugPreviewEglSurface = + GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); + } + } else { + eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); + if (debugSurfaceView != null) { + debugPreviewEglSurface = + GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); + } } + } catch (RuntimeException e) { + listener.onFrameProcessingError(new FrameProcessingException(e)); } } @@ -475,44 +473,58 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @WorkerThread @RequiresNonNull("inputSurfaceTexture") private void processFrame() { - checkState(Thread.currentThread().getName().equals(THREAD_NAME)); - checkStateNotNull(eglSurface, "No output surface set."); - - inputSurfaceTexture.updateTexImage(); - long presentationTimeNs = inputSurfaceTexture.getTimestamp(); - long presentationTimeUs = presentationTimeNs / 1000; - inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); - ((ExternalCopyFrameProcessor) frameProcessors.get(0)) - .setTextureTransformMatrix(textureTransformMatrix); - - for (int i = 0; i < frameProcessors.size() - 1; i++) { - Size intermediateSize = frameProcessors.get(i).getOutputSize(); - GlUtil.focusFramebuffer( - eglDisplay, - eglContext, - eglSurface, - framebuffers[i], - intermediateSize.getWidth(), - intermediateSize.getHeight()); - clearOutputFrame(); - frameProcessors.get(i).drawFrame(presentationTimeUs); + if (stopProcessing.get()) { + return; } - GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); - clearOutputFrame(); - getLast(frameProcessors).drawFrame(presentationTimeUs); - EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs); - EGL14.eglSwapBuffers(eglDisplay, eglSurface); + long presentationTimeUs = C.TIME_UNSET; + try { + checkState(Thread.currentThread().getName().equals(THREAD_NAME)); + checkStateNotNull(eglSurface, "No output surface set."); - if (debugPreviewEglSurface != null) { - GlUtil.focusEglSurface( - eglDisplay, eglContext, debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); + inputSurfaceTexture.updateTexImage(); + long presentationTimeNs = inputSurfaceTexture.getTimestamp(); + presentationTimeUs = presentationTimeNs / 1000; + inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); + ((ExternalCopyFrameProcessor) frameProcessors.get(0)) + .setTextureTransformMatrix(textureTransformMatrix); + + for (int i = 0; i < frameProcessors.size() - 1; i++) { + Size intermediateSize = frameProcessors.get(i).getOutputSize(); + GlUtil.focusFramebuffer( + eglDisplay, + eglContext, + eglSurface, + framebuffers[i], + intermediateSize.getWidth(), + intermediateSize.getHeight()); + clearOutputFrame(); + frameProcessors.get(i).drawFrame(presentationTimeUs); + } + GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); clearOutputFrame(); getLast(frameProcessors).drawFrame(presentationTimeUs); - EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface); - } - checkState(pendingFrameCount.getAndDecrement() > 0); + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs); + EGL14.eglSwapBuffers(eglDisplay, eglSurface); + + if (debugPreviewEglSurface != null) { + GlUtil.focusEglSurface( + eglDisplay, eglContext, debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); + clearOutputFrame(); + getLast(frameProcessors).drawFrame(presentationTimeUs); + EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface); + } + + checkState(pendingFrameCount.getAndDecrement() > 0); + } catch (FrameProcessingException | RuntimeException e) { + if (!stopProcessing.getAndSet(true)) { + listener.onFrameProcessingError( + e instanceof FrameProcessingException + ? (FrameProcessingException) e + : new FrameProcessingException(e, presentationTimeUs)); + } + } } private static void clearOutputFrame() { @@ -520,4 +532,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); GlUtil.checkGlError(); } + + /** + * Releases the {@link GlFrameProcessor GlFrameProcessors} and destroys the OpenGL context. + * + *

This method must be called on the {@linkplain #THREAD_NAME background thread}. + */ + @WorkerThread + private void releaseFrameProcessorsAndDestroyGlContext() { + try { + for (int i = 0; i < frameProcessors.size(); i++) { + frameProcessors.get(i).release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + } catch (RuntimeException e) { + listener.onFrameProcessingError(new FrameProcessingException(e)); + } + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java index d2e80327e3..edecda427c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java @@ -46,6 +46,7 @@ public interface GlFrameProcessor { * @param inputTexId Identifier of a 2D OpenGL texture. * @param inputWidth The input width, in pixels. * @param inputHeight The input height, in pixels. + * @throws IOException If an error occurs while reading resources. */ void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) throws IOException; @@ -69,8 +70,9 @@ public interface GlFrameProcessor { * program's vertex attributes and uniforms, and issue a drawing command. * * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + * @throws FrameProcessingException If an error occurs while processing or drawing the frame. */ - void drawFrame(long presentationTimeUs); + void drawFrame(long presentationTimeUs) throws FrameProcessingException; /** Releases all resources. */ void release(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java index 48f53dbcc6..cace85cbdc 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java @@ -97,16 +97,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void drawFrame(long presentationTimeUs) { - checkStateNotNull(glProgram).use(); - float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs); - checkState( - transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements"); - glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix); - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - GlUtil.checkGlError(); + public void drawFrame(long presentationTimeUs) throws FrameProcessingException { + try { + checkStateNotNull(glProgram).use(); + float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs); + checkState( + transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements"); + glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index ceb28250a5..3981edfe62 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -721,6 +721,8 @@ public final class Transformer { DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10, DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10) .build(); + TransformerPlayerListener playerListener = + new TransformerPlayerListener(mediaItem, muxerWrapper, looper); ExoPlayer.Builder playerBuilder = new ExoPlayer.Builder( context, @@ -734,6 +736,7 @@ public final class Transformer { encoderFactory, decoderFactory, new FallbackListener(mediaItem, listeners, transformationRequest), + playerListener, debugViewProvider)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) @@ -748,7 +751,7 @@ public final class Transformer { player = playerBuilder.build(); player.setMediaItem(mediaItem); - player.addListener(new TransformerPlayerListener(mediaItem, muxerWrapper)); + player.addListener(playerListener); player.prepare(); progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY; @@ -846,6 +849,7 @@ public final class Transformer { private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; private final FallbackListener fallbackListener; + private final FrameProcessorChain.Listener frameProcessorChainListener; private final Transformer.DebugViewProvider debugViewProvider; public TransformerRenderersFactory( @@ -858,6 +862,7 @@ public final class Transformer { Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, FallbackListener fallbackListener, + FrameProcessorChain.Listener frameProcessorChainListener, Transformer.DebugViewProvider debugViewProvider) { this.context = context; this.muxerWrapper = muxerWrapper; @@ -868,6 +873,7 @@ public final class Transformer { this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; this.fallbackListener = fallbackListener; + this.frameProcessorChainListener = frameProcessorChainListener; this.debugViewProvider = debugViewProvider; mediaClock = new TransformerMediaClock(); } @@ -904,6 +910,7 @@ public final class Transformer { encoderFactory, decoderFactory, fallbackListener, + frameProcessorChainListener, debugViewProvider); index++; } @@ -911,14 +918,18 @@ public final class Transformer { } } - private final class TransformerPlayerListener implements Player.Listener { + private final class TransformerPlayerListener + implements Player.Listener, FrameProcessorChain.Listener { private final MediaItem mediaItem; private final MuxerWrapper muxerWrapper; + private final Handler handler; - public TransformerPlayerListener(MediaItem mediaItem, MuxerWrapper muxerWrapper) { + public TransformerPlayerListener( + MediaItem mediaItem, MuxerWrapper muxerWrapper, Looper looper) { this.mediaItem = mediaItem; this.muxerWrapper = muxerWrapper; + handler = new Handler(looper); } @Override @@ -1013,5 +1024,14 @@ public final class Transformer { } listeners.flushEvents(); } + + @Override + public void onFrameProcessingError(FrameProcessingException exception) { + handler.post( + () -> + handleTransformationEnded( + TransformationException.createForFrameProcessorChain( + exception, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED))); + } } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index f4d8dabc7e..e0bf43732c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -38,6 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final ImmutableList effects; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; + private final FrameProcessorChain.Listener frameProcessorChainListener; private final Transformer.DebugViewProvider debugViewProvider; private final DecoderInputBuffer decoderInputBuffer; @@ -52,12 +53,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, FallbackListener fallbackListener, + FrameProcessorChain.Listener frameProcessorChainListener, Transformer.DebugViewProvider debugViewProvider) { super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); this.context = context; this.effects = effects; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; + this.frameProcessorChainListener = frameProcessorChainListener; this.debugViewProvider = debugViewProvider; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); @@ -95,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; encoderFactory, muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), fallbackListener, + frameProcessorChainListener, debugViewProvider); } if (transformationRequest.flattenForSlowMotion) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 6d28f8925f..534ae5d667 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -55,6 +55,7 @@ import org.checkerframework.dataflow.qual.Pure; Codec.EncoderFactory encoderFactory, List allowedOutputMimeTypes, FallbackListener fallbackListener, + FrameProcessorChain.Listener frameProcessorChainListener, Transformer.DebugViewProvider debugViewProvider) throws TransformationException { decoderInputBuffer = @@ -86,14 +87,20 @@ import org.checkerframework.dataflow.qual.Pure; EncoderCompatibilityTransformation encoderCompatibilityTransformation = new EncoderCompatibilityTransformation(); effectsListBuilder.add(encoderCompatibilityTransformation); - frameProcessorChain = - FrameProcessorChain.create( - context, - inputFormat.pixelWidthHeightRatio, - /* inputWidth= */ decodedWidth, - /* inputHeight= */ decodedHeight, - effectsListBuilder.build(), - transformationRequest.enableHdrEditing); + try { + frameProcessorChain = + FrameProcessorChain.create( + context, + frameProcessorChainListener, + inputFormat.pixelWidthHeightRatio, + /* inputWidth= */ decodedWidth, + /* inputHeight= */ decodedHeight, + effectsListBuilder.build(), + transformationRequest.enableHdrEditing); + } catch (FrameProcessingException e) { + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } Size requestedEncoderSize = frameProcessorChain.getOutputSize(); outputRotationDegrees = encoderCompatibilityTransformation.getOutputRotationDegrees(); @@ -146,7 +153,6 @@ import org.checkerframework.dataflow.qual.Pure; @Override public boolean processData() throws TransformationException { - frameProcessorChain.getAndRethrowBackgroundExceptions(); if (frameProcessorChain.isEnded()) { if (!signaledEndOfStreamToEncoder) { encoder.signalEndOfInputStream();