diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index b2aed28c87..78df45a3ad 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -260,7 +260,7 @@ public final class FrameProcessorChainPixelTest { outputSize.getHeight(), PixelFormat.RGBA_8888, /* maxImages= */ 1); - frameProcessorChain.configure( + frameProcessorChain.setOutputSurface( outputImageReader.getSurface(), outputSize.getWidth(), outputSize.getHeight(), diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java similarity index 97% rename from libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java index 933e7c2ba4..051eb3aefe 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java @@ -28,10 +28,9 @@ import org.junit.Test; import org.junit.runner.RunWith; /** - * Robolectric tests for {@link FrameProcessorChain}. + * Tests for creating and configuring a {@link FrameProcessorChain}. * - *

See {@code FrameProcessorChainPixelTest} in the androidTest directory for instrumentation - * tests. + *

See {@link FrameProcessorChainPixelTest} for data processing tests. */ @RunWith(AndroidJUnit4.class) public final class FrameProcessorChainTest { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index a0cb50ea7b..3199ed5b96 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -54,7 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * and is processed on a background thread as it becomes available. All input frames should be * {@linkplain #registerInputFrame() registered} before they are rendered to the input surface. * {@link #getPendingFrameCount()} can be used to check whether there are frames that have not been - * fully processed yet. Output is written to its {@linkplain #configure(Surface, int, int, + * fully processed yet. Output is written to its {@linkplain #setOutputSurface(Surface, int, int, * SurfaceView) output surface}. */ /* package */ final class FrameProcessorChain { @@ -73,7 +73,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @return A new instance. - * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1. + * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader + * files fails, or an OpenGL error occurs while creating and configuring the OpenGL + * components. */ public static FrameProcessorChain create( Context context, @@ -93,42 +95,104 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationException.ERROR_CODE_GL_INIT_FAILED); } + ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); ExternalCopyFrameProcessor externalCopyFrameProcessor = new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); - externalCopyFrameProcessor.setInputSize(inputWidth, inputHeight); - Size inputSize = externalCopyFrameProcessor.getOutputSize(); - for (int i = 0; i < frameProcessors.size(); i++) { - frameProcessors.get(i).setInputSize(inputSize.getWidth(), inputSize.getHeight()); - inputSize = frameProcessors.get(i).getOutputSize(); + + try { + return singleThreadExecutorService + .submit( + () -> + createOpenGlObjectsAndFrameProcessorChain( + inputWidth, + inputHeight, + frameProcessors, + enableExperimentalHdrEditing, + singleThreadExecutorService, + externalCopyFrameProcessor)) + .get(); + } catch (ExecutionException e) { + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } + } + + /** + * Creates the OpenGL textures, framebuffers, initializes the {@link GlFrameProcessor + * GlFrameProcessors} and returns a new {@code FrameProcessorChain}. + * + *

This method must by executed using the {@code singleThreadExecutorService}. + */ + private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain( + int inputWidth, + int inputHeight, + List frameProcessors, + boolean enableExperimentalHdrEditing, + ExecutorService singleThreadExecutorService, + ExternalCopyFrameProcessor externalCopyFrameProcessor) + throws IOException { + checkState(Thread.currentThread().getName().equals(THREAD_NAME)); + + EGLDisplay eglDisplay = GlUtil.createEglDisplay(); + EGLContext eglContext = + enableExperimentalHdrEditing + ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay) + : GlUtil.createEglContext(eglDisplay); + + if (GlUtil.isSurfacelessContextExtensionSupported()) { + GlUtil.focusEglSurface( + eglDisplay, eglContext, EGL14.EGL_NO_SURFACE, /* width= */ 1, /* height= */ 1); + } else if (enableExperimentalHdrEditing) { + // TODO(b/209404935): Don't assume BT.2020 PQ input/output. + GlUtil.focusPlaceholderEglSurfaceBt2020Pq(eglContext, eglDisplay); + } else { + GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); } + int inputExternalTexId = GlUtil.createExternalTexture(); + externalCopyFrameProcessor.setInputSize(inputWidth, inputHeight); + externalCopyFrameProcessor.initialize(inputExternalTexId); + + int[] framebuffers = new int[frameProcessors.size()]; + Size inputSize = externalCopyFrameProcessor.getOutputSize(); + for (int i = 0; i < frameProcessors.size(); i++) { + int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight()); + framebuffers[i] = GlUtil.createFboForTexture(inputTexId); + frameProcessors.get(i).setInputSize(inputSize.getWidth(), inputSize.getHeight()); + frameProcessors.get(i).initialize(inputTexId); + inputSize = frameProcessors.get(i).getOutputSize(); + } return new FrameProcessorChain( - externalCopyFrameProcessor, frameProcessors, enableExperimentalHdrEditing); + eglDisplay, + eglContext, + singleThreadExecutorService, + inputExternalTexId, + externalCopyFrameProcessor, + framebuffers, + ImmutableList.copyOf(frameProcessors), + enableExperimentalHdrEditing); } private static final String THREAD_NAME = "Transformer:FrameProcessorChain"; private final boolean enableExperimentalHdrEditing; - private @MonotonicNonNull EGLDisplay eglDisplay; - private @MonotonicNonNull EGLContext eglContext; + private final EGLDisplay eglDisplay; + private final EGLContext eglContext; /** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */ private final ExecutorService singleThreadExecutorService; /** Futures corresponding to the executor service's pending tasks. */ private final ConcurrentLinkedQueue> futures; /** Number of frames {@linkplain #registerInputFrame() registered} but not fully processed. */ private final AtomicInteger pendingFrameCount; - /** Prevents further frame processing tasks from being scheduled after {@link #release()}. */ - private volatile boolean releaseRequested; - private boolean inputStreamEnded; /** Wraps the {@link #inputSurfaceTexture}. */ - private @MonotonicNonNull Surface inputSurface; + private final Surface inputSurface; /** Associated with an OpenGL external texture. */ - private @MonotonicNonNull SurfaceTexture inputSurfaceTexture; - /** - * Identifier of the external texture the {@link ExternalCopyFrameProcessor} reads its input from. - */ - private int inputExternalTexId; + private final SurfaceTexture inputSurfaceTexture; /** Transformation matrix associated with the {@link #inputSurfaceTexture}. */ private final float[] textureTransformMatrix; @@ -158,19 +222,33 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; */ private @MonotonicNonNull EGLSurface debugPreviewEglSurface; + private boolean inputStreamEnded; + /** Prevents further frame processing tasks from being scheduled after {@link #release()}. */ + private volatile boolean releaseRequested; + private FrameProcessorChain( + EGLDisplay eglDisplay, + EGLContext eglContext, + ExecutorService singleThreadExecutorService, + int inputExternalTexId, ExternalCopyFrameProcessor externalCopyFrameProcessor, - List frameProcessors, + int[] framebuffers, + ImmutableList frameProcessors, boolean enableExperimentalHdrEditing) { + + this.eglDisplay = eglDisplay; + this.eglContext = eglContext; + this.singleThreadExecutorService = singleThreadExecutorService; this.externalCopyFrameProcessor = externalCopyFrameProcessor; - this.frameProcessors = ImmutableList.copyOf(frameProcessors); + this.framebuffers = framebuffers; + this.frameProcessors = frameProcessors; this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; - singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); futures = new ConcurrentLinkedQueue<>(); pendingFrameCount = new AtomicInteger(); + inputSurfaceTexture = new SurfaceTexture(inputExternalTexId); + inputSurface = new Surface(inputSurfaceTexture); textureTransformMatrix = new float[16]; - framebuffers = new int[frameProcessors.size()]; outputSize = frameProcessors.isEmpty() ? externalCopyFrameProcessor.getOutputSize() @@ -185,26 +263,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } /** - * Configures the {@code FrameProcessorChain} to process frames to the specified output targets. + * Sets the output {@link Surface}. * - *

This method may only be called once and may override the {@linkplain - * GlFrameProcessor#setInputSize(int, int) output size} of the final {@link GlFrameProcessor}. + *

This method may override the output size of the final {@link GlFrameProcessor}. * * @param outputSurface The output {@link Surface}. * @param outputWidth The output width, in pixels. * @param outputHeight The output height, in pixels. * @param debugSurfaceView Optional debug {@link SurfaceView} to show output. - * @throws IllegalStateException If the {@code FrameProcessorChain} has already been configured. - * @throws TransformationException If reading shader files fails, or an OpenGL error occurs while - * creating and configuring the OpenGL components. */ - public void configure( + public void setOutputSurface( Surface outputSurface, int outputWidth, int outputHeight, - @Nullable SurfaceView debugSurfaceView) - throws TransformationException { - checkState(inputSurface == null, "The FrameProcessorChain has already been configured."); + @Nullable SurfaceView debugSurfaceView) { // TODO(b/218488308): Don't override output size for encoder fallback. Instead allow the final // GlFrameProcessor to be re-configured or append another GlFrameProcessor. outputSize = new Size(outputWidth, outputHeight); @@ -214,21 +286,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; debugPreviewHeight = debugSurfaceView.getHeight(); } - try { - // Wait for task to finish to be able to use inputExternalTexId to create the SurfaceTexture. - singleThreadExecutorService - .submit(this::createOpenGlObjectsAndInitializeFrameProcessors) - .get(); - } catch (ExecutionException e) { - throw TransformationException.createForFrameProcessorChain( - e, TransformationException.ERROR_CODE_GL_INIT_FAILED); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw TransformationException.createForFrameProcessorChain( - e, TransformationException.ERROR_CODE_GL_INIT_FAILED); - } + futures.add( + singleThreadExecutorService.submit( + () -> createOpenGlSurfaces(outputSurface, debugSurfaceView))); - inputSurfaceTexture = new SurfaceTexture(inputExternalTexId); inputSurfaceTexture.setOnFrameAvailableListener( surfaceTexture -> { if (releaseRequested) { @@ -244,21 +305,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } } }); - inputSurface = new Surface(inputSurfaceTexture); - - futures.add( - singleThreadExecutorService.submit( - () -> createOpenGlSurfaces(outputSurface, debugSurfaceView))); } - /** - * Returns the input {@link Surface}. - * - *

The {@code FrameProcessorChain} must be {@linkplain #configure(Surface, int, int, - * SurfaceView) configured}. - */ + /** Returns the input {@link Surface}. */ public Surface getInputSurface() { - checkStateNotNull(inputSurface, "The FrameProcessorChain must be configured."); return inputSurface; } @@ -347,9 +397,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * Creates the OpenGL surfaces. * - *

This method should only be called after {@link - * #createOpenGlObjectsAndInitializeFrameProcessors()} and must be called on the background - * thread. + *

This method should only be called on the {@linkplain #THREAD_NAME background thread}. */ private void createOpenGlSurfaces(Surface outputSurface, @Nullable SurfaceView debugSurfaceView) { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); @@ -371,57 +419,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } } - /** - * Creates the OpenGL textures and framebuffers, and initializes the {@link GlFrameProcessor - * GlFrameProcessors}. - * - *

This method should only be called on the background thread. - */ - private Void createOpenGlObjectsAndInitializeFrameProcessors() throws IOException { - checkState(Thread.currentThread().getName().equals(THREAD_NAME)); - - eglDisplay = GlUtil.createEglDisplay(); - eglContext = - enableExperimentalHdrEditing - ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay) - : GlUtil.createEglContext(eglDisplay); - - if (GlUtil.isSurfacelessContextExtensionSupported()) { - GlUtil.focusEglSurface( - eglDisplay, eglContext, EGL14.EGL_NO_SURFACE, /* width= */ 1, /* height= */ 1); - } else if (enableExperimentalHdrEditing) { - // TODO(b/209404935): Don't assume BT.2020 PQ input/output. - GlUtil.focusPlaceholderEglSurfaceBt2020Pq(eglContext, eglDisplay); - } else { - GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); - } - - inputExternalTexId = GlUtil.createExternalTexture(); - externalCopyFrameProcessor.initialize(inputExternalTexId); - - Size intermediateSize = externalCopyFrameProcessor.getOutputSize(); - for (int i = 0; i < frameProcessors.size(); i++) { - int inputTexId = - GlUtil.createTexture(intermediateSize.getWidth(), intermediateSize.getHeight()); - framebuffers[i] = GlUtil.createFboForTexture(inputTexId); - frameProcessors.get(i).initialize(inputTexId); - intermediateSize = frameProcessors.get(i).getOutputSize(); - } - // Return something because only Callables not Runnables can throw checked exceptions. - return null; - } - /** * Processes an input frame. * - *

This method should only be called on the background thread. + *

This method should only be called on the {@linkplain #THREAD_NAME background thread}. */ @RequiresNonNull("inputSurfaceTexture") private void processFrame() { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); - checkStateNotNull(eglSurface); - checkStateNotNull(eglContext); - checkStateNotNull(eglDisplay); + checkStateNotNull(eglSurface, "No output surface set."); if (frameProcessors.isEmpty()) { GlUtil.focusEglSurface( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 1f23f8bdd7..c0dc977c8e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -116,7 +116,7 @@ import org.checkerframework.dataflow.qual.Pure; requestedEncoderFormat, encoderSupportedFormat)); - frameProcessorChain.configure( + frameProcessorChain.setOutputSurface( /* outputSurface= */ encoder.getInputSurface(), /* outputWidth= */ encoderSupportedFormat.width, /* outputHeight= */ encoderSupportedFormat.height, diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java index 0051978bdc..6fd88d043b 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java @@ -114,13 +114,19 @@ public class DefaultEncoderFactoryTest { @Test public void createForVideoEncoding_withNoSupportedEncoder_throws() { Format requestedVideoFormat = createVideoFormat(MimeTypes.VIDEO_H264, 1920, 1080, 30); - assertThrows( - TransformationException.class, - () -> - new DefaultEncoderFactory() - .createForVideoEncoding( - requestedVideoFormat, - /* allowedMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H265))); + + TransformationException exception = + assertThrows( + TransformationException.class, + () -> + new DefaultEncoderFactory() + .createForVideoEncoding( + requestedVideoFormat, + /* allowedMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H265))); + + assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class); + assertThat(exception.errorCode) + .isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); } @Test diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java index ef47ce3433..510b74af10 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -30,7 +30,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import android.content.Context; -import android.media.MediaCodecInfo; import android.media.MediaCrypto; import android.media.MediaFormat; import android.os.Handler; @@ -405,26 +404,6 @@ public final class TransformerEndToEndTest { .isEqualTo(TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED); } - @Test - public void startTransformation_withVideoEncoderFormatUnsupported_completesWithError() - throws Exception { - Transformer transformer = - createTransformerBuilder(/* enableFallback= */ false) - .setTransformationRequest( - new TransformationRequest.Builder() - .setVideoMimeType(MimeTypes.VIDEO_H263) // unsupported encoder MIME type - .build()) - .build(); - MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_VIDEO_ONLY); - - transformer.startTransformation(mediaItem, outputPath); - TransformationException exception = TransformerTestRunner.runUntilError(transformer); - - assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class); - assertThat(exception.errorCode) - .isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); - } - @Test public void startTransformation_withIoError_completesWithError() throws Exception { Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build(); @@ -801,11 +780,6 @@ public final class TransformerEndToEndTest { throwingCodecConfig, /* colorFormats= */ ImmutableList.of(), /* isDecoder= */ true); - addCodec( - MimeTypes.VIDEO_H263, - throwingCodecConfig, - ImmutableList.of(MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible), - /* isDecoder= */ false); addCodec( MimeTypes.AUDIO_AMR_NB, throwingCodecConfig,