From 20daaa20efdb69ec0979f9d11847c13ea3a68ddc Mon Sep 17 00:00:00 2001 From: hschlueter Date: Thu, 24 Mar 2022 17:33:31 +0000 Subject: [PATCH] Move OpenGL setup to FrameProcessorChain#configure(). The factory method is replaced by a public constructor and configure() method which configures the input/output surfaces and handles the OpenGL setup. This is a prerequisite for removing the responsibility of the caller to configureSizes() before creating the chain in a follow-up. PiperOrigin-RevId: 437028882 --- .../FrameProcessorChainPixelTest.java | 20 +- .../transformer/FrameProcessorChainTest.java | 76 ---- .../transformer/FrameProcessorChain.java | 391 ++++++++---------- .../VideoTranscodingSamplePipeline.java | 20 +- .../transformer/FrameProcessorChainTest.java | 38 +- 5 files changed, 235 insertions(+), 310 deletions(-) delete mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index 99ce783e06..6cee080d27 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -251,31 +251,27 @@ public final class FrameProcessorChainPixelTest { List sizes = FrameProcessorChain.configureSizes(inputWidth, inputHeight, frameProcessorsList); assertThat(sizes).isNotEmpty(); + int outputWidth = Iterables.getLast(sizes).getWidth(); + int outputHeight = Iterables.getLast(sizes).getHeight(); outputImageReader = ImageReader.newInstance( - Iterables.getLast(sizes).getWidth(), - Iterables.getLast(sizes).getHeight(), - PixelFormat.RGBA_8888, - /* maxImages= */ 1); + outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1); frameProcessorChain = - FrameProcessorChain.create( + new FrameProcessorChain( context, PIXEL_WIDTH_HEIGHT_RATIO, frameProcessorsList, sizes, - outputImageReader.getSurface(), - /* enableExperimentalHdrEditing= */ false, - Transformer.DebugViewProvider.NONE); + /* enableExperimentalHdrEditing= */ false); + frameProcessorChain.configure( + outputImageReader.getSurface(), outputWidth, outputHeight, /* debugSurfaceView= */ null); frameProcessorChain.registerInputFrame(); // Queue the first video frame from the extractor. String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); mediaCodec = MediaCodec.createDecoderByType(mimeType); mediaCodec.configure( - mediaFormat, - frameProcessorChain.createInputSurface(), - /* crypto= */ null, - /* flags= */ 0); + mediaFormat, frameProcessorChain.getInputSurface(), /* crypto= */ null, /* flags= */ 0); mediaCodec.start(); int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java deleted file mode 100644 index 6f55bee945..0000000000 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static androidx.test.core.app.ApplicationProvider.getApplicationContext; -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; - -import android.content.Context; -import android.graphics.SurfaceTexture; -import android.util.Size; -import android.view.Surface; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.google.common.collect.ImmutableList; -import java.util.List; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - * Test for {@link FrameProcessorChain#create(Context, float, List, List, Surface, boolean, - * Transformer.DebugViewProvider) creating} a {@link FrameProcessorChain}. - */ -@RunWith(AndroidJUnit4.class) -public final class FrameProcessorChainTest { - // TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a - // wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20. - - @Test - public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully() - throws TransformationException { - Context context = getApplicationContext(); - - FrameProcessorChain.create( - context, - /* pixelWidthHeightRatio= */ 1, - /* frameProcessors= */ ImmutableList.of(), - /* sizes= */ ImmutableList.of(new Size(200, 100)), - /* outputSurface= */ new Surface(new SurfaceTexture(false)), - /* enableExperimentalHdrEditing= */ false, - Transformer.DebugViewProvider.NONE); - } - - @Test - public void create_withUnsupportedPixelWidthHeightRatio_throwsException() { - Context context = getApplicationContext(); - - TransformationException exception = - assertThrows( - TransformationException.class, - () -> - FrameProcessorChain.create( - context, - /* pixelWidthHeightRatio= */ 2, - /* frameProcessors= */ ImmutableList.of(), - /* sizes= */ ImmutableList.of(new Size(200, 100)), - /* outputSurface= */ new Surface(new SurfaceTexture(false)), - /* enableExperimentalHdrEditing= */ false, - Transformer.DebugViewProvider.NONE)); - - assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class); - assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio"); - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index 559889cd25..8b445864b9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -18,6 +18,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; import static com.google.common.collect.Iterables.getLast; import android.content.Context; @@ -44,18 +45,19 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicInteger; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * {@code FrameProcessorChain} applies changes to individual video frames. * - *

Input becomes available on its {@link #createInputSurface() input surface} asynchronously and - * is processed on a background thread as it becomes available. All input frames should be {@link + *

Input becomes available on its {@link #getInputSurface() input surface} asynchronously and is + * processed on a background thread as it becomes available. All input frames should be {@link * #registerInputFrame() registered} before they are rendered to the input surface. {@link * #hasPendingFrames()} can be used to check whether there are frames that have not been fully * processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link - * #create(Context, float, List, List, Surface, boolean, Transformer.DebugViewProvider)}. + * #configure(Surface, int, int, SurfaceView)}. */ /* package */ final class FrameProcessorChain { @@ -89,170 +91,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return sizes; } - /** - * Returns a new {@code FrameProcessorChain} for applying changes to individual frames. - * - * @param context A {@link Context}. - * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. - * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame. - * Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)} - * configured}. - * @param sizes The input {@link Size} as well as the output {@link Size} of each {@link - * GlFrameProcessor}. - * @param outputSurface The {@link Surface}. - * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. - * @param debugViewProvider Provider for optional debug views to show intermediate output. - * @return A configured {@code FrameProcessorChain}. - * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader - * files fails, or an OpenGL error occurs while creating and configuring the OpenGL - * components. - */ - public static FrameProcessorChain create( - Context context, - float pixelWidthHeightRatio, - List frameProcessors, - List sizes, - Surface outputSurface, - boolean enableExperimentalHdrEditing, - Transformer.DebugViewProvider debugViewProvider) - throws TransformationException { - checkArgument(frameProcessors.size() + 1 == sizes.size()); - - if (pixelWidthHeightRatio != 1.0f) { - // TODO(b/211782176): Consider implementing support for non-square pixels. - throw TransformationException.createForFrameProcessorChain( - new UnsupportedOperationException( - "Transformer's FrameProcessorChain currently does not support frame edits on" - + " non-square pixels. The pixelWidthHeightRatio is: " - + pixelWidthHeightRatio), - TransformationException.ERROR_CODE_GL_INIT_FAILED); - } - - @Nullable - SurfaceView debugSurfaceView = - debugViewProvider.getDebugPreviewSurfaceView( - getLast(sizes).getWidth(), getLast(sizes).getHeight()); - int debugPreviewWidth; - int debugPreviewHeight; - if (debugSurfaceView != null) { - debugPreviewWidth = debugSurfaceView.getWidth(); - debugPreviewHeight = debugSurfaceView.getHeight(); - } else { - debugPreviewWidth = C.LENGTH_UNSET; - debugPreviewHeight = C.LENGTH_UNSET; - } - - ExternalCopyFrameProcessor externalCopyFrameProcessor = - new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); - - ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); - Future frameProcessorChainFuture = - singleThreadExecutorService.submit( - () -> - createOpenGlObjectsAndFrameProcessorChain( - singleThreadExecutorService, - externalCopyFrameProcessor, - frameProcessors, - sizes, - outputSurface, - enableExperimentalHdrEditing, - debugSurfaceView, - debugPreviewWidth, - debugPreviewHeight)); - try { - return frameProcessorChainFuture.get(); - } catch (ExecutionException e) { - throw TransformationException.createForFrameProcessorChain( - e, TransformationException.ERROR_CODE_GL_INIT_FAILED); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw TransformationException.createForFrameProcessorChain( - e, TransformationException.ERROR_CODE_GL_INIT_FAILED); - } - } - - /** - * Creates a {@code FrameProcessorChain} and its OpenGL objects. - * - *

As the {@code FrameProcessorChain} will call OpenGL commands on the {@code - * singleThreadExecutorService}'s thread, the OpenGL context and objects also need to be created - * on that thread. So this method should only be called on the {@code - * singleThreadExecutorService}'s thread. - */ - private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain( - ExecutorService singleThreadExecutorService, - ExternalCopyFrameProcessor externalCopyFrameProcessor, - List frameProcessors, - List sizes, - Surface outputSurface, - boolean enableExperimentalHdrEditing, - @Nullable SurfaceView debugSurfaceView, - int debugPreviewWidth, - int debugPreviewHeight) - throws IOException { - EGLDisplay eglDisplay = GlUtil.createEglDisplay(); - - final EGLContext eglContext; - final EGLSurface eglSurface; - @Nullable EGLSurface debugPreviewEglSurface = null; - if (enableExperimentalHdrEditing) { - eglContext = GlUtil.createEglContextEs3Rgba1010102(eglDisplay); - // TODO(b/209404935): Don't assume BT.2020 PQ input/output. - eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface); - if (debugSurfaceView != null) { - debugPreviewEglSurface = - GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); - } - } else { - eglContext = GlUtil.createEglContext(eglDisplay); - eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); - if (debugSurfaceView != null) { - debugPreviewEglSurface = - GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); - } - } - - GlUtil.focusEglSurface( - eglDisplay, eglContext, eglSurface, getLast(sizes).getWidth(), getLast(sizes).getHeight()); - - int inputExternalTexId = GlUtil.createExternalTexture(); - externalCopyFrameProcessor.configureOutputSize( - /* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight()); - externalCopyFrameProcessor.initialize(inputExternalTexId); - - int[] framebuffers = new int[frameProcessors.size()]; - for (int i = 0; i < frameProcessors.size(); i++) { - int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight()); - framebuffers[i] = GlUtil.createFboForTexture(inputTexId); - frameProcessors.get(i).initialize(inputTexId); - } - - return new FrameProcessorChain( - singleThreadExecutorService, - eglDisplay, - eglContext, - eglSurface, - externalCopyFrameProcessor, - frameProcessors, - inputExternalTexId, - framebuffers, - sizes, - debugPreviewEglSurface, - debugPreviewWidth, - debugPreviewHeight); - } - private static final String THREAD_NAME = "Transformer:FrameProcessorChain"; - private final EGLContext eglContext; + private final boolean enableExperimentalHdrEditing; private final EGLDisplay eglDisplay; - /** - * Wraps the output {@link Surface} that is populated with the output of the final {@link - * GlFrameProcessor} for each frame. - */ - private final EGLSurface eglSurface; + private final EGLContext eglContext; /** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */ private final ExecutorService singleThreadExecutorService; + /** The {@link #singleThreadExecutorService} thread. */ + private @MonotonicNonNull Thread glThread; /** Futures corresponding to the executor service's pending tasks. */ private final ConcurrentLinkedQueue> futures; /** Number of frames {@link #registerInputFrame() registered} but not fully processed. */ @@ -268,8 +115,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * Identifier of the external texture the {@link ExternalCopyFrameProcessor} reads its input from. */ - private final int inputExternalTexId; - /** Transformation matrix associated with the surface texture. */ + private int inputExternalTexId; + /** Transformation matrix associated with the {@link #inputSurfaceTexture}. */ private final float[] textureTransformMatrix; private final ExternalCopyFrameProcessor externalCopyFrameProcessor; @@ -289,55 +136,129 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; */ private final List sizes; - private final int debugPreviewWidth; - private final int debugPreviewHeight; + private int outputWidth; + private int outputHeight; + /** + * Wraps the output {@link Surface} that is populated with the output of the final {@link + * GlFrameProcessor} for each frame. + */ + private @MonotonicNonNull EGLSurface eglSurface; + + private int debugPreviewWidth; + private int debugPreviewHeight; /** * Wraps a debug {@link SurfaceView} that is populated with the output of the final {@link * GlFrameProcessor} for each frame. */ - @Nullable private final EGLSurface debugPreviewEglSurface; + private @MonotonicNonNull EGLSurface debugPreviewEglSurface; - private FrameProcessorChain( - ExecutorService singleThreadExecutorService, - EGLDisplay eglDisplay, - EGLContext eglContext, - EGLSurface eglSurface, - ExternalCopyFrameProcessor externalCopyFrameProcessor, + /** + * Creates a new instance. + * + * @param context A {@link Context}. + * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. + * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame. + * Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)} + * configured}. + * @param sizes The input {@link Size} as well as the output {@link Size} of each {@link + * GlFrameProcessor}. + * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. + * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1. + */ + public FrameProcessorChain( + Context context, + float pixelWidthHeightRatio, List frameProcessors, - int inputExternalTexId, - int[] framebuffers, List sizes, - @Nullable EGLSurface debugPreviewEglSurface, - int debugPreviewWidth, - int debugPreviewHeight) { - this.singleThreadExecutorService = singleThreadExecutorService; - this.eglDisplay = eglDisplay; - this.eglContext = eglContext; - this.eglSurface = eglSurface; - this.externalCopyFrameProcessor = externalCopyFrameProcessor; - this.frameProcessors = frameProcessors; - this.inputExternalTexId = inputExternalTexId; - this.framebuffers = framebuffers; - this.sizes = sizes; - this.debugPreviewEglSurface = debugPreviewEglSurface; - this.debugPreviewWidth = debugPreviewWidth; - this.debugPreviewHeight = debugPreviewHeight; + boolean enableExperimentalHdrEditing) + throws TransformationException { + checkArgument(frameProcessors.size() + 1 == sizes.size()); + if (pixelWidthHeightRatio != 1.0f) { + // TODO(b/211782176): Consider implementing support for non-square pixels. + throw TransformationException.createForFrameProcessorChain( + new UnsupportedOperationException( + "Transformer's FrameProcessorChain currently does not support frame edits on" + + " non-square pixels. The pixelWidthHeightRatio is: " + + pixelWidthHeightRatio), + TransformationException.ERROR_CODE_GL_INIT_FAILED); + } + + this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; + this.frameProcessors = frameProcessors; + this.sizes = sizes; + + try { + eglDisplay = GlUtil.createEglDisplay(); + eglContext = + enableExperimentalHdrEditing + ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay) + : GlUtil.createEglContext(eglDisplay); + } catch (GlUtil.GlException e) { + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } + singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); futures = new ConcurrentLinkedQueue<>(); pendingFrameCount = new AtomicInteger(); textureTransformMatrix = new float[16]; + externalCopyFrameProcessor = + new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); + framebuffers = new int[frameProcessors.size()]; + outputWidth = getLast(sizes).getWidth(); + outputHeight = getLast(sizes).getHeight(); + debugPreviewWidth = C.LENGTH_UNSET; + debugPreviewHeight = C.LENGTH_UNSET; } /** - * Creates the input {@link Surface} and configures it to process frames. + * Configures the {@code FrameProcessorChain} to process frames to the specified output targets. * - *

This method must not be called again after creating an input surface. + *

This method may only be called once and may override the {@link + * GlFrameProcessor#configureOutputSize(int, int) output size} of the final {@link + * GlFrameProcessor}. * - * @return The configured input {@link Surface}. - * @throws IllegalStateException If an input {@link Surface} has already been created. + * @param outputSurface The output {@link Surface}. + * @param outputWidth The output width, in pixels. + * @param outputHeight The output height, in pixels. + * @param debugSurfaceView Optional debug {@link SurfaceView} to show output. + * @throws IllegalStateException If the {@code FrameProcessorChain} has already been configured. + * @throws TransformationException If reading shader files fails, or an OpenGL error occurs while + * creating and configuring the OpenGL components. */ - public Surface createInputSurface() { - checkState(inputSurface == null, "The input surface has already been created."); + public void configure( + Surface outputSurface, + int outputWidth, + int outputHeight, + @Nullable SurfaceView debugSurfaceView) + throws TransformationException { + checkState(inputSurface == null, "The FrameProcessorChain has already been configured."); + // TODO(b/218488308): Don't override output size for encoder fallback. Instead allow the final + // GlFrameProcessor to be re-configured or append another GlFrameProcessor. + this.outputWidth = outputWidth; + this.outputHeight = outputHeight; + + if (debugSurfaceView != null) { + debugPreviewWidth = debugSurfaceView.getWidth(); + debugPreviewHeight = debugSurfaceView.getHeight(); + } + + try { + // Wait for task to finish to be able to use inputExternalTexId to create the SurfaceTexture. + singleThreadExecutorService + .submit( + () -> + createOpenGlObjectsAndInitializeFrameProcessors(outputSurface, debugSurfaceView)) + .get(); + } catch (ExecutionException e) { + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } + inputSurfaceTexture = new SurfaceTexture(inputExternalTexId); inputSurfaceTexture.setOnFrameAvailableListener( surfaceTexture -> { @@ -355,6 +276,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } }); inputSurface = new Surface(inputSurfaceTexture); + } + + /** + * Returns the input {@link Surface}. + * + *

The {@code FrameProcessorChain} must be {@link #configure(Surface, int, int, SurfaceView) + * configured}. + */ + public Surface getInputSurface() { + checkStateNotNull(inputSurface, "The FrameProcessorChain must be configured."); return inputSurface; } @@ -440,16 +371,60 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; singleThreadExecutorService.shutdown(); } - /** Processes an input frame. */ - @RequiresNonNull("inputSurfaceTexture") + /** + * Creates the OpenGL textures, framebuffers, surfaces, and initializes the {@link + * GlFrameProcessor GlFrameProcessors}. + * + *

This method must by executed on the same thread as {@link #processFrame()}, i.e., executed + * by the {@link #singleThreadExecutorService}. + */ + @EnsuresNonNull("eglSurface") + private Void createOpenGlObjectsAndInitializeFrameProcessors( + Surface outputSurface, @Nullable SurfaceView debugSurfaceView) throws IOException { + glThread = Thread.currentThread(); + if (enableExperimentalHdrEditing) { + // TODO(b/209404935): Don't assume BT.2020 PQ input/output. + eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface); + if (debugSurfaceView != null) { + debugPreviewEglSurface = + GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); + } + } else { + eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); + if (debugSurfaceView != null) { + debugPreviewEglSurface = + GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); + } + } + GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); + + inputExternalTexId = GlUtil.createExternalTexture(); + externalCopyFrameProcessor.configureOutputSize( + /* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight()); + externalCopyFrameProcessor.initialize(inputExternalTexId); + + for (int i = 0; i < frameProcessors.size(); i++) { + int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight()); + framebuffers[i] = GlUtil.createFboForTexture(inputTexId); + frameProcessors.get(i).initialize(inputTexId); + } + // Return something because only Callables not Runnables can throw checked exceptions. + return null; + } + + /** + * Processes an input frame. + * + *

This method must by executed on the same thread as {@link + * #createOpenGlObjectsAndInitializeFrameProcessors(Surface,SurfaceView)}, i.e., executed by the + * {@link #singleThreadExecutorService}. + */ + @RequiresNonNull({"inputSurfaceTexture", "eglSurface"}) private void processFrame() { - inputSurfaceTexture.updateTexImage(); - inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); - long presentationTimeNs = inputSurfaceTexture.getTimestamp(); + checkState(Thread.currentThread().equals(glThread)); if (frameProcessors.isEmpty()) { - GlUtil.focusEglSurface( - eglDisplay, eglContext, eglSurface, sizes.get(0).getWidth(), sizes.get(0).getHeight()); + GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); } else { GlUtil.focusFramebuffer( eglDisplay, @@ -459,7 +434,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; sizes.get(0).getWidth(), sizes.get(0).getHeight()); } + inputSurfaceTexture.updateTexImage(); + inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix); + long presentationTimeNs = inputSurfaceTexture.getTimestamp(); externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs); for (int i = 0; i < frameProcessors.size() - 1; i++) { @@ -473,12 +451,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs); } if (!frameProcessors.isEmpty()) { - GlUtil.focusEglSurface( - eglDisplay, - eglContext, - eglSurface, - getLast(sizes).getWidth(), - getLast(sizes).getHeight()); + GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); getLast(frameProcessors).updateProgramAndDraw(presentationTimeNs); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 4c27eca73c..1c10c4561b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -110,24 +110,22 @@ import org.checkerframework.dataflow.qual.Pure; requestedEncoderFormat, encoderSupportedFormat)); - // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size - // has to change due to encoder fallback or append another GlFrameProcessor. - frameProcessorSizes.set( - frameProcessorSizes.size() - 1, - new Size(encoderSupportedFormat.width, encoderSupportedFormat.height)); frameProcessorChain = - FrameProcessorChain.create( + new FrameProcessorChain( context, inputFormat.pixelWidthHeightRatio, frameProcessors, frameProcessorSizes, - /* outputSurface= */ encoder.getInputSurface(), - transformationRequest.enableHdrEditing, - debugViewProvider); + transformationRequest.enableHdrEditing); + frameProcessorChain.configure( + /* outputSurface= */ encoder.getInputSurface(), + /* outputWidth= */ encoderSupportedFormat.width, + /* outputHeight= */ encoderSupportedFormat.height, + debugViewProvider.getDebugPreviewSurfaceView( + encoderSupportedFormat.width, encoderSupportedFormat.height)); decoder = - decoderFactory.createForVideoDecoding( - inputFormat, frameProcessorChain.createInputSurface()); + decoderFactory.createForVideoDecoding(inputFormat, frameProcessorChain.getInputSurface()); } @Override diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java index b2b9a1a3d0..d1ce4a5141 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java @@ -15,8 +15,11 @@ */ package androidx.media3.transformer; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import android.content.Context; import android.util.Size; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -27,11 +30,42 @@ import org.junit.runner.RunWith; /** * Robolectric tests for {@link FrameProcessorChain}. * - *

See {@code FrameProcessorChainTest} and {@code FrameProcessorChainPixelTest} in the - * androidTest directory for instrumentation tests. + *

See {@code FrameProcessorChainPixelTest} in the androidTest directory for instrumentation + * tests. */ @RunWith(AndroidJUnit4.class) public final class FrameProcessorChainTest { + @Test + public void construct_withSupportedPixelWidthHeightRatio_completesSuccessfully() + throws TransformationException { + Context context = getApplicationContext(); + + new FrameProcessorChain( + context, + /* pixelWidthHeightRatio= */ 1, + /* frameProcessors= */ ImmutableList.of(), + /* sizes= */ ImmutableList.of(new Size(200, 100)), + /* enableExperimentalHdrEditing= */ false); + } + + @Test + public void construct_withUnsupportedPixelWidthHeightRatio_throwsException() { + Context context = getApplicationContext(); + + TransformationException exception = + assertThrows( + TransformationException.class, + () -> + new FrameProcessorChain( + context, + /* pixelWidthHeightRatio= */ 2, + /* frameProcessors= */ ImmutableList.of(), + /* sizes= */ ImmutableList.of(new Size(200, 100)), + /* enableExperimentalHdrEditing= */ false)); + + assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class); + assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio"); + } @Test public void configureOutputDimensions_withEmptyList_returnsInputSize() {