diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl deleted file mode 100644 index 20f3058ce2..0000000000 --- a/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl +++ /dev/null @@ -1,27 +0,0 @@ -#version 100 -// Copyright 2022 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// ES 2 vertex shader that applies an external surface texture's 4 * 4 texture -// transformation matrix to convert the texture coordinates to the sampling -// locations. - -attribute vec4 aFramePosition; -uniform mat4 uTexTransform; -varying vec2 vTexSamplingCoord; -void main() { - gl_Position = aFramePosition; - vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0); - vTexSamplingCoord = (uTexTransform * texturePosition).xy; -} diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl index 2491e3d2a2..06164bad5e 100644 --- a/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl +++ b/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl @@ -13,13 +13,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -// ES 2 vertex shader that applies the 4 * 4 transformation matrix -// uTransformationMatrix. +// ES 2 vertex shader that applies the 4 * 4 transformation matrices +// uTransformationMatrix and the uTexTransformationMatrix. attribute vec4 aFramePosition; uniform mat4 uTransformationMatrix; +uniform mat4 uTexTransformationMatrix; varying vec2 vTexSamplingCoord; void main() { gl_Position = uTransformationMatrix * aFramePosition; - vTexSamplingCoord = vec2(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5); + vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0); + vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy; } diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl similarity index 66% rename from libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl rename to libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl index f732294c90..c99b31112e 100644 --- a/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl +++ b/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl @@ -1,5 +1,5 @@ #version 300 es -// Copyright 2022 The Android Open Source Project +// Copyright 2021 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,15 +13,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture -// transformation matrix to convert the texture coordinates to the sampling -// locations. +// ES 3 vertex shader that applies the 4 * 4 transformation matrices +// uTransformationMatrix and the uTexTransformationMatrix. in vec4 aFramePosition; -uniform mat4 uTexTransform; +uniform mat4 uTransformationMatrix; +uniform mat4 uTexTransformationMatrix; out vec2 vTexSamplingCoord; void main() { - gl_Position = aFramePosition; + gl_Position = uTransformationMatrix * aFramePosition; vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0); - vTexSamplingCoord = (uTexTransform * texturePosition).xy; + vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java index a2f96ef63f..f31c7682ad 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java @@ -15,71 +15,13 @@ */ package androidx.media3.transformer; -import static androidx.media3.common.util.Assertions.checkArgument; -import static androidx.media3.common.util.Assertions.checkStateNotNull; - -import android.content.Context; -import android.opengl.GLES20; -import android.util.Size; -import androidx.media3.common.util.GlProgram; -import androidx.media3.common.util.GlUtil; -import java.io.IOException; - -/** Copies frames from an external texture and applies color transformations for HDR if needed. */ -/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor { - - private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH = - "shaders/vertex_shader_tex_transform_es2.glsl"; - private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH = - "shaders/vertex_shader_tex_transform_es3.glsl"; - private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = - "shaders/fragment_shader_copy_external_es2.glsl"; - private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = - "shaders/fragment_shader_copy_external_yuv_es3.glsl"; - // Color transform coefficients from - // https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f. - private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = { - 1.168f, 1.168f, 1.168f, - 0.0f, -0.188f, 2.148f, - 1.683f, -0.652f, 0.0f, - }; - - private final GlProgram glProgram; - - /** - * Creates a new instance. - * - * @param useHdr Whether to process the input as an HDR signal. - * @throws FrameProcessingException If a problem occurs while reading shader files. - */ - public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException { - String vertexShaderFilePath = - useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH; - String fragmentShaderFilePath = - useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; - try { - glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); - } catch (IOException | GlUtil.GlException e) { - throw new FrameProcessingException(e); - } - // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. - glProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - if (useHdr) { - // In HDR editing mode the decoder output is sampled in YUV. - glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); - } - } - - @Override - public Size configure(int inputWidth, int inputHeight) { - checkArgument(inputWidth > 0, "inputWidth must be positive"); - checkArgument(inputHeight > 0, "inputHeight must be positive"); - - return new Size(inputWidth, inputHeight); - } +/** + * Interface for a {@link GlTextureProcessor} that samples from an external texture. + * + *

Use {@link #setTextureTransformMatrix(float[])} to provide the texture's transformation + * matrix. + */ +/* package */ interface ExternalTextureProcessor extends GlTextureProcessor { /** * Sets the texture transform matrix for converting an external surface texture's coordinates to @@ -88,35 +30,5 @@ import java.io.IOException; * @param textureTransformMatrix The external surface texture's {@linkplain * android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}. */ - public void setTextureTransformMatrix(float[] textureTransformMatrix) { - checkStateNotNull(glProgram); - glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix); - } - - @Override - public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { - checkStateNotNull(glProgram); - try { - glProgram.use(); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - GlUtil.checkGlError(); - } catch (GlUtil.GlException e) { - throw new FrameProcessingException(e, presentationTimeUs); - } - } - - @Override - public void release() throws FrameProcessingException { - super.release(); - if (glProgram != null) { - try { - glProgram.delete(); - } catch (GlUtil.GlException e) { - throw new FrameProcessingException(e); - } - } - } + void setTextureTransformMatrix(float[] textureTransformMatrix); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java index fc91bb3dc5..29bcb3c34d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java @@ -24,6 +24,7 @@ import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; import android.opengl.GLES20; +import android.opengl.Matrix; import android.util.Size; import android.view.Surface; import android.view.SurfaceHolder; @@ -50,7 +51,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; *

This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link * GlTextureProcessor} instances used by {@link FrameProcessor}. */ -/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor { +/* package */ final class FinalMatrixTransformationProcessorWrapper + implements GlTextureProcessor, ExternalTextureProcessor { private static final String TAG = "FinalProcessorWrapper"; @@ -61,7 +63,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final long streamOffsetUs; private final DebugViewProvider debugViewProvider; private final FrameProcessor.Listener frameProcessorListener; + private final boolean sampleFromExternalTexture; private final boolean useHdr; + private final float[] textureTransformMatrix; private int inputWidth; private int inputHeight; @@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, FrameProcessor.Listener frameProcessorListener, DebugViewProvider debugViewProvider, + boolean sampleFromExternalTexture, boolean useHdr) { this.context = context; this.matrixTransformations = matrixTransformations; @@ -94,7 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.streamOffsetUs = streamOffsetUs; this.debugViewProvider = debugViewProvider; this.frameProcessorListener = frameProcessorListener; + this.sampleFromExternalTexture = sampleFromExternalTexture; this.useHdr = useHdr; + + textureTransformMatrix = new float[16]; + Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0); } /** @@ -239,7 +248,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT)); MatrixTransformationProcessor matrixTransformationProcessor = - new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build()); + new MatrixTransformationProcessor( + context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr); + matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight); checkState(outputSize.getWidth() == outputSurfaceInfo.width); checkState(outputSize.getHeight() == outputSurfaceInfo.height); @@ -265,6 +276,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + @Override + public void setTextureTransformMatrix(float[] textureTransformMatrix) { + System.arraycopy( + /* src= */ textureTransformMatrix, + /* srcPos= */ 0, + /* dest= */ this.textureTransformMatrix, + /* destPost= */ 0, + /* length= */ textureTransformMatrix.length); + + if (matrixTransformationProcessor != null) { + matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); + } + } + public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) { this.outputSurfaceInfo = outputSurfaceInfo; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java index a951a9bc2c..3856a6bdee 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java @@ -17,13 +17,13 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static com.google.common.collect.Iterables.getLast; import android.content.Context; import android.graphics.SurfaceTexture; import android.opengl.EGL14; import android.opengl.EGLContext; import android.opengl.EGLDisplay; -import android.util.Pair; import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.WorkerThread; @@ -126,31 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); } - Pair, FinalMatrixTransformationProcessorWrapper> - textureProcessors = - getGlTextureProcessorsForGlEffects( - context, - effects, - eglDisplay, - eglContext, - streamOffsetUs, - listener, - debugViewProvider, - useHdr); - ImmutableList intermediateTextureProcessors = textureProcessors.first; - FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper = - textureProcessors.second; - - ExternalTextureProcessor externalTextureProcessor = - new ExternalTextureProcessor(context, useHdr); + ImmutableList textureProcessors = + getGlTextureProcessorsForGlEffects( + context, + effects, + eglDisplay, + eglContext, + streamOffsetUs, + listener, + debugViewProvider, + useHdr); FrameProcessingTaskExecutor frameProcessingTaskExecutor = new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); - chainTextureProcessorsWithListeners( - externalTextureProcessor, - intermediateTextureProcessors, - finalTextureProcessorWrapper, - frameProcessingTaskExecutor, - listener); + chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener); return new GlEffectsFrameProcessor( eglDisplay, @@ -158,9 +146,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; frameProcessingTaskExecutor, streamOffsetUs, /* inputExternalTextureId= */ GlUtil.createExternalTexture(), - externalTextureProcessor, - intermediateTextureProcessors, - finalTextureProcessorWrapper); + textureProcessors); } /** @@ -168,25 +154,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate * {@link GlTextureProcessor} instances. * - * @return A {@link Pair} containing a list of {@link GlTextureProcessor} instances to apply in - * the given order and a {@link FinalMatrixTransformationProcessorWrapper} to apply after - * them. + * @return A non-empty list of {@link GlTextureProcessor} instances to apply in the given order. + * The first is an {@link ExternalTextureProcessor} and the last is a {@link + * FinalMatrixTransformationProcessorWrapper}. */ - private static Pair, FinalMatrixTransformationProcessorWrapper> - getGlTextureProcessorsForGlEffects( - Context context, - List effects, - EGLDisplay eglDisplay, - EGLContext eglContext, - long streamOffsetUs, - FrameProcessor.Listener listener, - DebugViewProvider debugViewProvider, - boolean useHdr) - throws FrameProcessingException { + private static ImmutableList getGlTextureProcessorsForGlEffects( + Context context, + List effects, + EGLDisplay eglDisplay, + EGLContext eglContext, + long streamOffsetUs, + FrameProcessor.Listener listener, + DebugViewProvider debugViewProvider, + boolean useHdr) + throws FrameProcessingException { ImmutableList.Builder textureProcessorListBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder matrixTransformationListBuilder = new ImmutableList.Builder<>(); + boolean sampleFromExternalTexture = true; for (int i = 0; i < effects.size(); i++) { GlEffect effect = effects.get(i); if (effect instanceof GlMatrixTransformation) { @@ -195,15 +181,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); - if (!matrixTransformations.isEmpty()) { + if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) { textureProcessorListBuilder.add( - new MatrixTransformationProcessor(context, matrixTransformations)); + new MatrixTransformationProcessor( + context, matrixTransformations, sampleFromExternalTexture, useHdr)); matrixTransformationListBuilder = new ImmutableList.Builder<>(); + sampleFromExternalTexture = false; } textureProcessorListBuilder.add(effect.toGlTextureProcessor(context)); } - return Pair.create( - textureProcessorListBuilder.build(), + textureProcessorListBuilder.add( new FinalMatrixTransformationProcessorWrapper( context, eglDisplay, @@ -212,51 +199,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, listener, debugViewProvider, + sampleFromExternalTexture, useHdr)); + return textureProcessorListBuilder.build(); } /** * Chains the given {@link GlTextureProcessor} instances using {@link * ChainingGlTextureProcessorListener} instances. - * - *

The {@link ExternalTextureProcessor} is the first processor in the chain. */ private static void chainTextureProcessorsWithListeners( - ExternalTextureProcessor externalTextureProcessor, - ImmutableList intermediateTextureProcessors, - FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper, + ImmutableList textureProcessors, FrameProcessingTaskExecutor frameProcessingTaskExecutor, FrameProcessor.Listener listener) { - externalTextureProcessor.setListener( - new ChainingGlTextureProcessorListener( - /* previousGlTextureProcessor= */ null, - /* nextGlTextureProcessor= */ intermediateTextureProcessors.size() > 0 - ? intermediateTextureProcessors.get(0) - : finalTextureProcessorWrapper, - frameProcessingTaskExecutor, - listener)); - GlTextureProcessor previousGlTextureProcessor = externalTextureProcessor; - for (int i = 0; i < intermediateTextureProcessors.size(); i++) { - GlTextureProcessor glTextureProcessor = intermediateTextureProcessors.get(i); + for (int i = 0; i < textureProcessors.size(); i++) { + @Nullable + GlTextureProcessor previousGlTextureProcessor = + i - 1 >= 0 ? textureProcessors.get(i - 1) : null; @Nullable GlTextureProcessor nextGlTextureProcessor = - i + 1 < intermediateTextureProcessors.size() - ? intermediateTextureProcessors.get(i + 1) - : finalTextureProcessorWrapper; - glTextureProcessor.setListener( - new ChainingGlTextureProcessorListener( - previousGlTextureProcessor, - nextGlTextureProcessor, - frameProcessingTaskExecutor, - listener)); - previousGlTextureProcessor = glTextureProcessor; + i + 1 < textureProcessors.size() ? textureProcessors.get(i + 1) : null; + textureProcessors + .get(i) + .setListener( + new ChainingGlTextureProcessorListener( + previousGlTextureProcessor, + nextGlTextureProcessor, + frameProcessingTaskExecutor, + listener)); } - finalTextureProcessorWrapper.setListener( - new ChainingGlTextureProcessorListener( - previousGlTextureProcessor, - /* nextGlTextureProcessor= */ null, - frameProcessingTaskExecutor, - listener)); } private static final String TAG = "GlEffectsFrameProcessor"; @@ -280,11 +251,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final float[] inputSurfaceTextureTransformMatrix; private final int inputExternalTextureId; private final ExternalTextureProcessor inputExternalTextureProcessor; - private final ImmutableList intermediateTextureProcessors; private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper; + private final ImmutableList allTextureProcessors; private final ConcurrentLinkedQueue pendingInputFrames; + // Fields accessed on the thread used by the GlEffectsFrameProcessor's caller. private @MonotonicNonNull FrameInfo nextInputFrameInfo; + + // Fields accessed on the frameProcessingTaskExecutor's thread. + private boolean inputTextureInUse; private boolean inputStreamEnded; private GlEffectsFrameProcessor( @@ -293,18 +268,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; FrameProcessingTaskExecutor frameProcessingTaskExecutor, long streamOffsetUs, int inputExternalTextureId, - ExternalTextureProcessor inputExternalTextureProcessor, - ImmutableList intermediateTextureProcessors, - FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper) { + ImmutableList textureProcessors) { this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; this.streamOffsetUs = streamOffsetUs; this.inputExternalTextureId = inputExternalTextureId; - this.inputExternalTextureProcessor = inputExternalTextureProcessor; - this.intermediateTextureProcessors = intermediateTextureProcessors; - this.finalTextureProcessorWrapper = finalTextureProcessorWrapper; + + checkState(!textureProcessors.isEmpty()); + checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor); + checkState(getLast(textureProcessors) instanceof FinalMatrixTransformationProcessorWrapper); + inputExternalTextureProcessor = (ExternalTextureProcessor) textureProcessors.get(0); + finalTextureProcessorWrapper = + (FinalMatrixTransformationProcessorWrapper) getLast(textureProcessors); + allTextureProcessors = textureProcessors; inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId); inputSurface = new Surface(inputSurfaceTexture); @@ -321,7 +299,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void setInputFrameInfo(FrameInfo inputFrameInfo) { - nextInputFrameInfo = inputFrameInfo; + nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo); } @Override @@ -365,36 +343,54 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } /** - * Processes an input frame from the {@linkplain #getInputSurface() external input surface - * texture}. + * Processes an input frame from the {@link #inputSurfaceTexture}. * *

This method must be called on the {@linkplain #THREAD_NAME background thread}. */ @WorkerThread private void processInputFrame() { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); - if (!inputExternalTextureProcessor.acceptsInputFrame()) { + if (inputTextureInUse) { frameProcessingTaskExecutor.submit(this::processInputFrame); // Try again later. return; } + inputTextureInUse = true; inputSurfaceTexture.updateTexImage(); + inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix); + queueInputFrameToTextureProcessors(); + } + + /** + * Queues the input frame to the first texture processor until it is accepted. + * + *

This method must be called on the {@linkplain #THREAD_NAME background thread}. + */ + @WorkerThread + private void queueInputFrameToTextureProcessors() { + checkState(Thread.currentThread().getName().equals(THREAD_NAME)); + checkState(inputTextureInUse); + long inputFrameTimeNs = inputSurfaceTexture.getTimestamp(); // Correct for the stream offset so processors see original media presentation timestamps. long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs; - inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix); inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix); - FrameInfo inputFrameInfo = adjustForPixelWidthHeightRatio(pendingInputFrames.remove()); - checkState( - inputExternalTextureProcessor.maybeQueueInputFrame( - new TextureInfo( - inputExternalTextureId, - /* fboId= */ C.INDEX_UNSET, - inputFrameInfo.width, - inputFrameInfo.height), - presentationTimeUs)); - // After the inputExternalTextureProcessor has produced an output frame, it is processed - // asynchronously by the texture processors chained after it. + FrameInfo inputFrameInfo = checkStateNotNull(pendingInputFrames.peek()); + if (inputExternalTextureProcessor.maybeQueueInputFrame( + new TextureInfo( + inputExternalTextureId, + /* fboId= */ C.INDEX_UNSET, + inputFrameInfo.width, + inputFrameInfo.height), + presentationTimeUs)) { + inputTextureInUse = false; + pendingInputFrames.remove(); + // After the externalTextureProcessor has produced an output frame, it is processed + // asynchronously by the texture processors chained after it. + } else { + // Try again later. + frameProcessingTaskExecutor.submit(this::queueInputFrameToTextureProcessors); + } } /** @@ -442,11 +438,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @WorkerThread private void releaseTextureProcessorsAndDestroyGlContext() throws GlUtil.GlException, FrameProcessingException { - inputExternalTextureProcessor.release(); - for (int i = 0; i < intermediateTextureProcessors.size(); i++) { - intermediateTextureProcessors.get(i).release(); + for (int i = 0; i < allTextureProcessors.size(); i++) { + allTextureProcessors.get(i).release(); } - finalTextureProcessorWrapper.release(); GlUtil.destroyEglContext(eglDisplay, eglContext); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java index 9f551f0a3b..c85bf100f1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java @@ -37,20 +37,36 @@ import java.util.Arrays; * matrices are clipped to the NDC range. * *

The background color of the output frame will be (r=0, g=0, b=0, a=0). + * + *

Can copy frames from an external texture and apply color transformations for HDR if needed. */ @UnstableApi @SuppressWarnings("FunctionalInterfaceClash") // b/228192298 -/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor { +/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor + implements ExternalTextureProcessor { private static final String VERTEX_SHADER_TRANSFORMATION_PATH = "shaders/vertex_shader_transformation_es2.glsl"; - private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; + private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH = + "shaders/vertex_shader_transformation_es3.glsl"; + private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl"; + private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH = + "shaders/fragment_shader_copy_external_es2.glsl"; + private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH = + "shaders/fragment_shader_copy_external_yuv_es3.glsl"; private static final ImmutableList NDC_SQUARE = ImmutableList.of( new float[] {-1, -1, 0, 1}, new float[] {-1, 1, 0, 1}, new float[] {1, 1, 0, 1}, new float[] {1, -1, 0, 1}); + // Color transform coefficients from + // https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f. + private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = { + 1.168f, 1.168f, 1.168f, + 0.0f, -0.188f, 2.148f, + 1.683f, -0.652f, 0.0f, + }; /** The {@link MatrixTransformation MatrixTransformations} to apply. */ private final ImmutableList matrixTransformations; @@ -89,7 +105,11 @@ import java.util.Arrays; */ public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation) throws FrameProcessingException { - this(context, ImmutableList.of(matrixTransformation)); + this( + context, + ImmutableList.of(matrixTransformation), + /* sampleFromExternalTexture= */ false, + /* enableExperimentalHdrEditing= */ false); } /** @@ -102,7 +122,11 @@ import java.util.Arrays; */ public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation) throws FrameProcessingException { - this(context, ImmutableList.of(matrixTransformation)); + this( + context, + ImmutableList.of(matrixTransformation), + /* sampleFromExternalTexture= */ false, + /* enableExperimentalHdrEditing= */ false); } /** @@ -111,10 +135,17 @@ import java.util.Arrays; * @param context The {@link Context}. * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * apply to each frame in order. + * @param sampleFromExternalTexture Whether the input will be provided using an external texture. + * If {@code true}, the caller should use {@link #setTextureTransformMatrix(float[])} to + * provide the transformation matrix associated with the external texture. + * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @throws FrameProcessingException If a problem occurs while reading shader files. */ public MatrixTransformationProcessor( - Context context, ImmutableList matrixTransformations) + Context context, + ImmutableList matrixTransformations, + boolean sampleFromExternalTexture, + boolean enableExperimentalHdrEditing) throws FrameProcessingException { this.matrixTransformations = matrixTransformations; @@ -123,11 +154,41 @@ import java.util.Arrays; tempResultMatrix = new float[16]; Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); visiblePolygon = NDC_SQUARE; + + String vertexShaderFilePath; + String fragmentShaderFilePath; + if (sampleFromExternalTexture) { + vertexShaderFilePath = + enableExperimentalHdrEditing + ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH + : VERTEX_SHADER_TRANSFORMATION_PATH; + fragmentShaderFilePath = + enableExperimentalHdrEditing + ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH + : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; + } else { + vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; + fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH; + } + try { - glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); + glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); } catch (IOException | GlUtil.GlException e) { throw new FrameProcessingException(e); } + + if (enableExperimentalHdrEditing && sampleFromExternalTexture) { + // In HDR editing mode the decoder output is sampled in YUV. + glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM); + } + float[] identityMatrix = new float[16]; + Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0); + glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + } + + @Override + public void setTextureTransformMatrix(float[] textureTransformMatrix) { + glProgram.setFloatsUniform("uTexTransformationMatrix", textureTransformMatrix); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java index b767f973a5..84f05c815b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java @@ -75,16 +75,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso this.listener = listener; } - /** - * Returns whether the {@code SingleFrameGlTextureProcessor} can accept an input frame. - * - *

If this method returns {@code true}, the next call to {@link #maybeQueueInputFrame( - * TextureInfo, long)} will also return {@code true}. - */ - public boolean acceptsInputFrame() { - return !outputTextureInUse; - } - @Override public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { if (outputTextureInUse) {