mirror of
https://github.com/androidx/media.git
synced 2025-04-30 06:46:50 +08:00
Merge MatrixTransformationProcessor and ExternalTextureProcessor.
This saves an intermediate texture copy step for use-cases where matrix transformations are the first or only effects in the chain. PiperOrigin-RevId: 460239403
This commit is contained in:
hschlueter
Rohit Singh
parent
18f4068c06
commit
7dc54efdb9
@ -1,27 +0,0 @@
|
||||
#version 100
|
||||
// Copyright 2022 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ES 2 vertex shader that applies an external surface texture's 4 * 4 texture
|
||||
// transformation matrix to convert the texture coordinates to the sampling
|
||||
// locations.
|
||||
|
||||
attribute vec4 aFramePosition;
|
||||
uniform mat4 uTexTransform;
|
||||
varying vec2 vTexSamplingCoord;
|
||||
void main() {
|
||||
gl_Position = aFramePosition;
|
||||
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
|
||||
vTexSamplingCoord = (uTexTransform * texturePosition).xy;
|
||||
}
|
||||
@ -13,13 +13,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ES 2 vertex shader that applies the 4 * 4 transformation matrix
|
||||
// uTransformationMatrix.
|
||||
// ES 2 vertex shader that applies the 4 * 4 transformation matrices
|
||||
// uTransformationMatrix and the uTexTransformationMatrix.
|
||||
|
||||
attribute vec4 aFramePosition;
|
||||
uniform mat4 uTransformationMatrix;
|
||||
uniform mat4 uTexTransformationMatrix;
|
||||
varying vec2 vTexSamplingCoord;
|
||||
void main() {
|
||||
gl_Position = uTransformationMatrix * aFramePosition;
|
||||
vTexSamplingCoord = vec2(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5);
|
||||
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
|
||||
vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#version 300 es
|
||||
// Copyright 2022 The Android Open Source Project
|
||||
// Copyright 2021 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@ -13,15 +13,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture
|
||||
// transformation matrix to convert the texture coordinates to the sampling
|
||||
// locations.
|
||||
// ES 3 vertex shader that applies the 4 * 4 transformation matrices
|
||||
// uTransformationMatrix and the uTexTransformationMatrix.
|
||||
|
||||
in vec4 aFramePosition;
|
||||
uniform mat4 uTexTransform;
|
||||
uniform mat4 uTransformationMatrix;
|
||||
uniform mat4 uTexTransformationMatrix;
|
||||
out vec2 vTexSamplingCoord;
|
||||
void main() {
|
||||
gl_Position = aFramePosition;
|
||||
gl_Position = uTransformationMatrix * aFramePosition;
|
||||
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
|
||||
vTexSamplingCoord = (uTexTransform * texturePosition).xy;
|
||||
vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
|
||||
}
|
||||
@ -15,71 +15,13 @@
|
||||
*/
|
||||
package androidx.media3.transformer;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
|
||||
import android.content.Context;
|
||||
import android.opengl.GLES20;
|
||||
import android.util.Size;
|
||||
import androidx.media3.common.util.GlProgram;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import java.io.IOException;
|
||||
|
||||
/** Copies frames from an external texture and applies color transformations for HDR if needed. */
|
||||
/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor {
|
||||
|
||||
private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH =
|
||||
"shaders/vertex_shader_tex_transform_es2.glsl";
|
||||
private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH =
|
||||
"shaders/vertex_shader_tex_transform_es3.glsl";
|
||||
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
|
||||
"shaders/fragment_shader_copy_external_es2.glsl";
|
||||
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
|
||||
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
|
||||
// Color transform coefficients from
|
||||
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
|
||||
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
|
||||
1.168f, 1.168f, 1.168f,
|
||||
0.0f, -0.188f, 2.148f,
|
||||
1.683f, -0.652f, 0.0f,
|
||||
};
|
||||
|
||||
private final GlProgram glProgram;
|
||||
|
||||
/**
|
||||
* Creates a new instance.
|
||||
*
|
||||
* @param useHdr Whether to process the input as an HDR signal.
|
||||
* @throws FrameProcessingException If a problem occurs while reading shader files.
|
||||
*/
|
||||
public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException {
|
||||
String vertexShaderFilePath =
|
||||
useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH;
|
||||
String fragmentShaderFilePath =
|
||||
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
|
||||
try {
|
||||
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
|
||||
} catch (IOException | GlUtil.GlException e) {
|
||||
throw new FrameProcessingException(e);
|
||||
}
|
||||
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
|
||||
glProgram.setBufferAttribute(
|
||||
"aFramePosition",
|
||||
GlUtil.getNormalizedCoordinateBounds(),
|
||||
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
|
||||
if (useHdr) {
|
||||
// In HDR editing mode the decoder output is sampled in YUV.
|
||||
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Size configure(int inputWidth, int inputHeight) {
|
||||
checkArgument(inputWidth > 0, "inputWidth must be positive");
|
||||
checkArgument(inputHeight > 0, "inputHeight must be positive");
|
||||
|
||||
return new Size(inputWidth, inputHeight);
|
||||
}
|
||||
/**
|
||||
* Interface for a {@link GlTextureProcessor} that samples from an external texture.
|
||||
*
|
||||
* <p>Use {@link #setTextureTransformMatrix(float[])} to provide the texture's transformation
|
||||
* matrix.
|
||||
*/
|
||||
/* package */ interface ExternalTextureProcessor extends GlTextureProcessor {
|
||||
|
||||
/**
|
||||
* Sets the texture transform matrix for converting an external surface texture's coordinates to
|
||||
@ -88,35 +30,5 @@ import java.io.IOException;
|
||||
* @param textureTransformMatrix The external surface texture's {@linkplain
|
||||
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
|
||||
*/
|
||||
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
|
||||
checkStateNotNull(glProgram);
|
||||
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
|
||||
checkStateNotNull(glProgram);
|
||||
try {
|
||||
glProgram.use();
|
||||
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
|
||||
glProgram.bindAttributesAndUniforms();
|
||||
// The four-vertex triangle strip forms a quad.
|
||||
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
|
||||
GlUtil.checkGlError();
|
||||
} catch (GlUtil.GlException e) {
|
||||
throw new FrameProcessingException(e, presentationTimeUs);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void release() throws FrameProcessingException {
|
||||
super.release();
|
||||
if (glProgram != null) {
|
||||
try {
|
||||
glProgram.delete();
|
||||
} catch (GlUtil.GlException e) {
|
||||
throw new FrameProcessingException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
void setTextureTransformMatrix(float[] textureTransformMatrix);
|
||||
}
|
||||
|
||||
@ -24,6 +24,7 @@ import android.opengl.EGLDisplay;
|
||||
import android.opengl.EGLExt;
|
||||
import android.opengl.EGLSurface;
|
||||
import android.opengl.GLES20;
|
||||
import android.opengl.Matrix;
|
||||
import android.util.Size;
|
||||
import android.view.Surface;
|
||||
import android.view.SurfaceHolder;
|
||||
@ -50,7 +51,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
* <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link
|
||||
* GlTextureProcessor} instances used by {@link FrameProcessor}.
|
||||
*/
|
||||
/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor {
|
||||
/* package */ final class FinalMatrixTransformationProcessorWrapper
|
||||
implements GlTextureProcessor, ExternalTextureProcessor {
|
||||
|
||||
private static final String TAG = "FinalProcessorWrapper";
|
||||
|
||||
@ -61,7 +63,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
private final long streamOffsetUs;
|
||||
private final DebugViewProvider debugViewProvider;
|
||||
private final FrameProcessor.Listener frameProcessorListener;
|
||||
private final boolean sampleFromExternalTexture;
|
||||
private final boolean useHdr;
|
||||
private final float[] textureTransformMatrix;
|
||||
|
||||
private int inputWidth;
|
||||
private int inputHeight;
|
||||
@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
long streamOffsetUs,
|
||||
FrameProcessor.Listener frameProcessorListener,
|
||||
DebugViewProvider debugViewProvider,
|
||||
boolean sampleFromExternalTexture,
|
||||
boolean useHdr) {
|
||||
this.context = context;
|
||||
this.matrixTransformations = matrixTransformations;
|
||||
@ -94,7 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
this.streamOffsetUs = streamOffsetUs;
|
||||
this.debugViewProvider = debugViewProvider;
|
||||
this.frameProcessorListener = frameProcessorListener;
|
||||
this.sampleFromExternalTexture = sampleFromExternalTexture;
|
||||
this.useHdr = useHdr;
|
||||
|
||||
textureTransformMatrix = new float[16];
|
||||
Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -239,7 +248,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
|
||||
|
||||
MatrixTransformationProcessor matrixTransformationProcessor =
|
||||
new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build());
|
||||
new MatrixTransformationProcessor(
|
||||
context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr);
|
||||
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
|
||||
Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight);
|
||||
checkState(outputSize.getWidth() == outputSurfaceInfo.width);
|
||||
checkState(outputSize.getHeight() == outputSurfaceInfo.height);
|
||||
@ -265,6 +276,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
|
||||
System.arraycopy(
|
||||
/* src= */ textureTransformMatrix,
|
||||
/* srcPos= */ 0,
|
||||
/* dest= */ this.textureTransformMatrix,
|
||||
/* destPost= */ 0,
|
||||
/* length= */ textureTransformMatrix.length);
|
||||
|
||||
if (matrixTransformationProcessor != null) {
|
||||
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
|
||||
if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
|
||||
this.outputSurfaceInfo = outputSurfaceInfo;
|
||||
|
||||
@ -17,13 +17,13 @@ package androidx.media3.transformer;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
import static com.google.common.collect.Iterables.getLast;
|
||||
|
||||
import android.content.Context;
|
||||
import android.graphics.SurfaceTexture;
|
||||
import android.opengl.EGL14;
|
||||
import android.opengl.EGLContext;
|
||||
import android.opengl.EGLDisplay;
|
||||
import android.util.Pair;
|
||||
import android.view.Surface;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.annotation.WorkerThread;
|
||||
@ -126,31 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
|
||||
}
|
||||
|
||||
Pair<ImmutableList<GlTextureProcessor>, FinalMatrixTransformationProcessorWrapper>
|
||||
textureProcessors =
|
||||
getGlTextureProcessorsForGlEffects(
|
||||
context,
|
||||
effects,
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
streamOffsetUs,
|
||||
listener,
|
||||
debugViewProvider,
|
||||
useHdr);
|
||||
ImmutableList<GlTextureProcessor> intermediateTextureProcessors = textureProcessors.first;
|
||||
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper =
|
||||
textureProcessors.second;
|
||||
|
||||
ExternalTextureProcessor externalTextureProcessor =
|
||||
new ExternalTextureProcessor(context, useHdr);
|
||||
ImmutableList<GlTextureProcessor> textureProcessors =
|
||||
getGlTextureProcessorsForGlEffects(
|
||||
context,
|
||||
effects,
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
streamOffsetUs,
|
||||
listener,
|
||||
debugViewProvider,
|
||||
useHdr);
|
||||
FrameProcessingTaskExecutor frameProcessingTaskExecutor =
|
||||
new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
|
||||
chainTextureProcessorsWithListeners(
|
||||
externalTextureProcessor,
|
||||
intermediateTextureProcessors,
|
||||
finalTextureProcessorWrapper,
|
||||
frameProcessingTaskExecutor,
|
||||
listener);
|
||||
chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener);
|
||||
|
||||
return new GlEffectsFrameProcessor(
|
||||
eglDisplay,
|
||||
@ -158,9 +146,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
frameProcessingTaskExecutor,
|
||||
streamOffsetUs,
|
||||
/* inputExternalTextureId= */ GlUtil.createExternalTexture(),
|
||||
externalTextureProcessor,
|
||||
intermediateTextureProcessors,
|
||||
finalTextureProcessorWrapper);
|
||||
textureProcessors);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -168,25 +154,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
* MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate
|
||||
* {@link GlTextureProcessor} instances.
|
||||
*
|
||||
* @return A {@link Pair} containing a list of {@link GlTextureProcessor} instances to apply in
|
||||
* the given order and a {@link FinalMatrixTransformationProcessorWrapper} to apply after
|
||||
* them.
|
||||
* @return A non-empty list of {@link GlTextureProcessor} instances to apply in the given order.
|
||||
* The first is an {@link ExternalTextureProcessor} and the last is a {@link
|
||||
* FinalMatrixTransformationProcessorWrapper}.
|
||||
*/
|
||||
private static Pair<ImmutableList<GlTextureProcessor>, FinalMatrixTransformationProcessorWrapper>
|
||||
getGlTextureProcessorsForGlEffects(
|
||||
Context context,
|
||||
List<GlEffect> effects,
|
||||
EGLDisplay eglDisplay,
|
||||
EGLContext eglContext,
|
||||
long streamOffsetUs,
|
||||
FrameProcessor.Listener listener,
|
||||
DebugViewProvider debugViewProvider,
|
||||
boolean useHdr)
|
||||
throws FrameProcessingException {
|
||||
private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffects(
|
||||
Context context,
|
||||
List<GlEffect> effects,
|
||||
EGLDisplay eglDisplay,
|
||||
EGLContext eglContext,
|
||||
long streamOffsetUs,
|
||||
FrameProcessor.Listener listener,
|
||||
DebugViewProvider debugViewProvider,
|
||||
boolean useHdr)
|
||||
throws FrameProcessingException {
|
||||
ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
|
||||
new ImmutableList.Builder<>();
|
||||
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
|
||||
new ImmutableList.Builder<>();
|
||||
boolean sampleFromExternalTexture = true;
|
||||
for (int i = 0; i < effects.size(); i++) {
|
||||
GlEffect effect = effects.get(i);
|
||||
if (effect instanceof GlMatrixTransformation) {
|
||||
@ -195,15 +181,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
}
|
||||
ImmutableList<GlMatrixTransformation> matrixTransformations =
|
||||
matrixTransformationListBuilder.build();
|
||||
if (!matrixTransformations.isEmpty()) {
|
||||
if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) {
|
||||
textureProcessorListBuilder.add(
|
||||
new MatrixTransformationProcessor(context, matrixTransformations));
|
||||
new MatrixTransformationProcessor(
|
||||
context, matrixTransformations, sampleFromExternalTexture, useHdr));
|
||||
matrixTransformationListBuilder = new ImmutableList.Builder<>();
|
||||
sampleFromExternalTexture = false;
|
||||
}
|
||||
textureProcessorListBuilder.add(effect.toGlTextureProcessor(context));
|
||||
}
|
||||
return Pair.create(
|
||||
textureProcessorListBuilder.build(),
|
||||
textureProcessorListBuilder.add(
|
||||
new FinalMatrixTransformationProcessorWrapper(
|
||||
context,
|
||||
eglDisplay,
|
||||
@ -212,51 +199,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
streamOffsetUs,
|
||||
listener,
|
||||
debugViewProvider,
|
||||
sampleFromExternalTexture,
|
||||
useHdr));
|
||||
return textureProcessorListBuilder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Chains the given {@link GlTextureProcessor} instances using {@link
|
||||
* ChainingGlTextureProcessorListener} instances.
|
||||
*
|
||||
* <p>The {@link ExternalTextureProcessor} is the first processor in the chain.
|
||||
*/
|
||||
private static void chainTextureProcessorsWithListeners(
|
||||
ExternalTextureProcessor externalTextureProcessor,
|
||||
ImmutableList<GlTextureProcessor> intermediateTextureProcessors,
|
||||
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper,
|
||||
ImmutableList<GlTextureProcessor> textureProcessors,
|
||||
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
|
||||
FrameProcessor.Listener listener) {
|
||||
externalTextureProcessor.setListener(
|
||||
new ChainingGlTextureProcessorListener(
|
||||
/* previousGlTextureProcessor= */ null,
|
||||
/* nextGlTextureProcessor= */ intermediateTextureProcessors.size() > 0
|
||||
? intermediateTextureProcessors.get(0)
|
||||
: finalTextureProcessorWrapper,
|
||||
frameProcessingTaskExecutor,
|
||||
listener));
|
||||
GlTextureProcessor previousGlTextureProcessor = externalTextureProcessor;
|
||||
for (int i = 0; i < intermediateTextureProcessors.size(); i++) {
|
||||
GlTextureProcessor glTextureProcessor = intermediateTextureProcessors.get(i);
|
||||
for (int i = 0; i < textureProcessors.size(); i++) {
|
||||
@Nullable
|
||||
GlTextureProcessor previousGlTextureProcessor =
|
||||
i - 1 >= 0 ? textureProcessors.get(i - 1) : null;
|
||||
@Nullable
|
||||
GlTextureProcessor nextGlTextureProcessor =
|
||||
i + 1 < intermediateTextureProcessors.size()
|
||||
? intermediateTextureProcessors.get(i + 1)
|
||||
: finalTextureProcessorWrapper;
|
||||
glTextureProcessor.setListener(
|
||||
new ChainingGlTextureProcessorListener(
|
||||
previousGlTextureProcessor,
|
||||
nextGlTextureProcessor,
|
||||
frameProcessingTaskExecutor,
|
||||
listener));
|
||||
previousGlTextureProcessor = glTextureProcessor;
|
||||
i + 1 < textureProcessors.size() ? textureProcessors.get(i + 1) : null;
|
||||
textureProcessors
|
||||
.get(i)
|
||||
.setListener(
|
||||
new ChainingGlTextureProcessorListener(
|
||||
previousGlTextureProcessor,
|
||||
nextGlTextureProcessor,
|
||||
frameProcessingTaskExecutor,
|
||||
listener));
|
||||
}
|
||||
finalTextureProcessorWrapper.setListener(
|
||||
new ChainingGlTextureProcessorListener(
|
||||
previousGlTextureProcessor,
|
||||
/* nextGlTextureProcessor= */ null,
|
||||
frameProcessingTaskExecutor,
|
||||
listener));
|
||||
}
|
||||
|
||||
private static final String TAG = "GlEffectsFrameProcessor";
|
||||
@ -280,11 +251,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
private final float[] inputSurfaceTextureTransformMatrix;
|
||||
private final int inputExternalTextureId;
|
||||
private final ExternalTextureProcessor inputExternalTextureProcessor;
|
||||
private final ImmutableList<GlTextureProcessor> intermediateTextureProcessors;
|
||||
private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper;
|
||||
private final ImmutableList<GlTextureProcessor> allTextureProcessors;
|
||||
private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames;
|
||||
|
||||
// Fields accessed on the thread used by the GlEffectsFrameProcessor's caller.
|
||||
private @MonotonicNonNull FrameInfo nextInputFrameInfo;
|
||||
|
||||
// Fields accessed on the frameProcessingTaskExecutor's thread.
|
||||
private boolean inputTextureInUse;
|
||||
private boolean inputStreamEnded;
|
||||
|
||||
private GlEffectsFrameProcessor(
|
||||
@ -293,18 +268,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
|
||||
long streamOffsetUs,
|
||||
int inputExternalTextureId,
|
||||
ExternalTextureProcessor inputExternalTextureProcessor,
|
||||
ImmutableList<GlTextureProcessor> intermediateTextureProcessors,
|
||||
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper) {
|
||||
ImmutableList<GlTextureProcessor> textureProcessors) {
|
||||
|
||||
this.eglDisplay = eglDisplay;
|
||||
this.eglContext = eglContext;
|
||||
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
|
||||
this.streamOffsetUs = streamOffsetUs;
|
||||
this.inputExternalTextureId = inputExternalTextureId;
|
||||
this.inputExternalTextureProcessor = inputExternalTextureProcessor;
|
||||
this.intermediateTextureProcessors = intermediateTextureProcessors;
|
||||
this.finalTextureProcessorWrapper = finalTextureProcessorWrapper;
|
||||
|
||||
checkState(!textureProcessors.isEmpty());
|
||||
checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor);
|
||||
checkState(getLast(textureProcessors) instanceof FinalMatrixTransformationProcessorWrapper);
|
||||
inputExternalTextureProcessor = (ExternalTextureProcessor) textureProcessors.get(0);
|
||||
finalTextureProcessorWrapper =
|
||||
(FinalMatrixTransformationProcessorWrapper) getLast(textureProcessors);
|
||||
allTextureProcessors = textureProcessors;
|
||||
|
||||
inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId);
|
||||
inputSurface = new Surface(inputSurfaceTexture);
|
||||
@ -321,7 +299,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
@Override
|
||||
public void setInputFrameInfo(FrameInfo inputFrameInfo) {
|
||||
nextInputFrameInfo = inputFrameInfo;
|
||||
nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -365,36 +343,54 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes an input frame from the {@linkplain #getInputSurface() external input surface
|
||||
* texture}.
|
||||
* Processes an input frame from the {@link #inputSurfaceTexture}.
|
||||
*
|
||||
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
|
||||
*/
|
||||
@WorkerThread
|
||||
private void processInputFrame() {
|
||||
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
|
||||
if (!inputExternalTextureProcessor.acceptsInputFrame()) {
|
||||
if (inputTextureInUse) {
|
||||
frameProcessingTaskExecutor.submit(this::processInputFrame); // Try again later.
|
||||
return;
|
||||
}
|
||||
|
||||
inputTextureInUse = true;
|
||||
inputSurfaceTexture.updateTexImage();
|
||||
inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix);
|
||||
queueInputFrameToTextureProcessors();
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues the input frame to the first texture processor until it is accepted.
|
||||
*
|
||||
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
|
||||
*/
|
||||
@WorkerThread
|
||||
private void queueInputFrameToTextureProcessors() {
|
||||
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
|
||||
checkState(inputTextureInUse);
|
||||
|
||||
long inputFrameTimeNs = inputSurfaceTexture.getTimestamp();
|
||||
// Correct for the stream offset so processors see original media presentation timestamps.
|
||||
long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs;
|
||||
inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix);
|
||||
inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix);
|
||||
FrameInfo inputFrameInfo = adjustForPixelWidthHeightRatio(pendingInputFrames.remove());
|
||||
checkState(
|
||||
inputExternalTextureProcessor.maybeQueueInputFrame(
|
||||
new TextureInfo(
|
||||
inputExternalTextureId,
|
||||
/* fboId= */ C.INDEX_UNSET,
|
||||
inputFrameInfo.width,
|
||||
inputFrameInfo.height),
|
||||
presentationTimeUs));
|
||||
// After the inputExternalTextureProcessor has produced an output frame, it is processed
|
||||
// asynchronously by the texture processors chained after it.
|
||||
FrameInfo inputFrameInfo = checkStateNotNull(pendingInputFrames.peek());
|
||||
if (inputExternalTextureProcessor.maybeQueueInputFrame(
|
||||
new TextureInfo(
|
||||
inputExternalTextureId,
|
||||
/* fboId= */ C.INDEX_UNSET,
|
||||
inputFrameInfo.width,
|
||||
inputFrameInfo.height),
|
||||
presentationTimeUs)) {
|
||||
inputTextureInUse = false;
|
||||
pendingInputFrames.remove();
|
||||
// After the externalTextureProcessor has produced an output frame, it is processed
|
||||
// asynchronously by the texture processors chained after it.
|
||||
} else {
|
||||
// Try again later.
|
||||
frameProcessingTaskExecutor.submit(this::queueInputFrameToTextureProcessors);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -442,11 +438,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
@WorkerThread
|
||||
private void releaseTextureProcessorsAndDestroyGlContext()
|
||||
throws GlUtil.GlException, FrameProcessingException {
|
||||
inputExternalTextureProcessor.release();
|
||||
for (int i = 0; i < intermediateTextureProcessors.size(); i++) {
|
||||
intermediateTextureProcessors.get(i).release();
|
||||
for (int i = 0; i < allTextureProcessors.size(); i++) {
|
||||
allTextureProcessors.get(i).release();
|
||||
}
|
||||
finalTextureProcessorWrapper.release();
|
||||
GlUtil.destroyEglContext(eglDisplay, eglContext);
|
||||
}
|
||||
}
|
||||
|
||||
@ -37,20 +37,36 @@ import java.util.Arrays;
|
||||
* matrices are clipped to the NDC range.
|
||||
*
|
||||
* <p>The background color of the output frame will be (r=0, g=0, b=0, a=0).
|
||||
*
|
||||
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
|
||||
*/
|
||||
@UnstableApi
|
||||
@SuppressWarnings("FunctionalInterfaceClash") // b/228192298
|
||||
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor {
|
||||
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor
|
||||
implements ExternalTextureProcessor {
|
||||
|
||||
private static final String VERTEX_SHADER_TRANSFORMATION_PATH =
|
||||
"shaders/vertex_shader_transformation_es2.glsl";
|
||||
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl";
|
||||
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
|
||||
"shaders/vertex_shader_transformation_es3.glsl";
|
||||
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
|
||||
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
|
||||
"shaders/fragment_shader_copy_external_es2.glsl";
|
||||
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
|
||||
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
|
||||
private static final ImmutableList<float[]> NDC_SQUARE =
|
||||
ImmutableList.of(
|
||||
new float[] {-1, -1, 0, 1},
|
||||
new float[] {-1, 1, 0, 1},
|
||||
new float[] {1, 1, 0, 1},
|
||||
new float[] {1, -1, 0, 1});
|
||||
// Color transform coefficients from
|
||||
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
|
||||
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
|
||||
1.168f, 1.168f, 1.168f,
|
||||
0.0f, -0.188f, 2.148f,
|
||||
1.683f, -0.652f, 0.0f,
|
||||
};
|
||||
|
||||
/** The {@link MatrixTransformation MatrixTransformations} to apply. */
|
||||
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
|
||||
@ -89,7 +105,11 @@ import java.util.Arrays;
|
||||
*/
|
||||
public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation)
|
||||
throws FrameProcessingException {
|
||||
this(context, ImmutableList.of(matrixTransformation));
|
||||
this(
|
||||
context,
|
||||
ImmutableList.of(matrixTransformation),
|
||||
/* sampleFromExternalTexture= */ false,
|
||||
/* enableExperimentalHdrEditing= */ false);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -102,7 +122,11 @@ import java.util.Arrays;
|
||||
*/
|
||||
public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation)
|
||||
throws FrameProcessingException {
|
||||
this(context, ImmutableList.of(matrixTransformation));
|
||||
this(
|
||||
context,
|
||||
ImmutableList.of(matrixTransformation),
|
||||
/* sampleFromExternalTexture= */ false,
|
||||
/* enableExperimentalHdrEditing= */ false);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -111,10 +135,17 @@ import java.util.Arrays;
|
||||
* @param context The {@link Context}.
|
||||
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
|
||||
* apply to each frame in order.
|
||||
* @param sampleFromExternalTexture Whether the input will be provided using an external texture.
|
||||
* If {@code true}, the caller should use {@link #setTextureTransformMatrix(float[])} to
|
||||
* provide the transformation matrix associated with the external texture.
|
||||
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
|
||||
* @throws FrameProcessingException If a problem occurs while reading shader files.
|
||||
*/
|
||||
public MatrixTransformationProcessor(
|
||||
Context context, ImmutableList<GlMatrixTransformation> matrixTransformations)
|
||||
Context context,
|
||||
ImmutableList<GlMatrixTransformation> matrixTransformations,
|
||||
boolean sampleFromExternalTexture,
|
||||
boolean enableExperimentalHdrEditing)
|
||||
throws FrameProcessingException {
|
||||
this.matrixTransformations = matrixTransformations;
|
||||
|
||||
@ -123,11 +154,41 @@ import java.util.Arrays;
|
||||
tempResultMatrix = new float[16];
|
||||
Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
|
||||
visiblePolygon = NDC_SQUARE;
|
||||
|
||||
String vertexShaderFilePath;
|
||||
String fragmentShaderFilePath;
|
||||
if (sampleFromExternalTexture) {
|
||||
vertexShaderFilePath =
|
||||
enableExperimentalHdrEditing
|
||||
? VERTEX_SHADER_TRANSFORMATION_ES3_PATH
|
||||
: VERTEX_SHADER_TRANSFORMATION_PATH;
|
||||
fragmentShaderFilePath =
|
||||
enableExperimentalHdrEditing
|
||||
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
|
||||
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
|
||||
} else {
|
||||
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
|
||||
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
|
||||
}
|
||||
|
||||
try {
|
||||
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH);
|
||||
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
|
||||
} catch (IOException | GlUtil.GlException e) {
|
||||
throw new FrameProcessingException(e);
|
||||
}
|
||||
|
||||
if (enableExperimentalHdrEditing && sampleFromExternalTexture) {
|
||||
// In HDR editing mode the decoder output is sampled in YUV.
|
||||
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
|
||||
}
|
||||
float[] identityMatrix = new float[16];
|
||||
Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
|
||||
glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
|
||||
glProgram.setFloatsUniform("uTexTransformationMatrix", textureTransformMatrix);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -75,16 +75,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether the {@code SingleFrameGlTextureProcessor} can accept an input frame.
|
||||
*
|
||||
* <p>If this method returns {@code true}, the next call to {@link #maybeQueueInputFrame(
|
||||
* TextureInfo, long)} will also return {@code true}.
|
||||
*/
|
||||
public boolean acceptsInputFrame() {
|
||||
return !outputTextureInUse;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
||||
if (outputTextureInUse) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user