loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-05 06:30:24 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge MatrixTransformationProcessor and ExternalTextureProcessor.

Browse Source 
This saves an intermediate texture copy step for use-cases
where matrix transformations are the first or only effects
in the chain.

PiperOrigin-RevId: 460239403
(cherry picked from commit 0615922cb64a649a4b01eb6410680f6f754e2005)
This commit is contained in:
hschlueter 2022-07-11 17:04:11 +00:00 committed by microkatz
parent c910750074
commit a4604c722c
8 changed files with 212 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
library/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl
View File

@ -1,27 +0,0 @@
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 vertex shader that applies an external surface texture's 4 * 4 texture
// transformation matrix to convert the texture coordinates to the sampling
// locations.
attribute vec4 aFramePosition;
uniform mat4 uTexTransform;
varying vec2 vTexSamplingCoord;
void main() {
gl_Position = aFramePosition;
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransform * texturePosition).xy;
}

8
library/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl
View File

@ -13,13 +13,15 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// ES 2 vertex shader that applies the 4 * 4 transformation matrix // ES 2 vertex shader that applies the 4 * 4 transformation matrices
// uTransformationMatrix. // uTransformationMatrix and the uTexTransformationMatrix.
attribute vec4 aFramePosition; attribute vec4 aFramePosition;
uniform mat4 uTransformationMatrix; uniform mat4 uTransformationMatrix;
uniform mat4 uTexTransformationMatrix;
varying vec2 vTexSamplingCoord; varying vec2 vTexSamplingCoord;
void main() { void main() {
gl_Position = uTransformationMatrix * aFramePosition; gl_Position = uTransformationMatrix * aFramePosition;
vTexSamplingCoord = vec2(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5); vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
} }

14
library/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl → library/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl
View File

@ -1,5 +1,5 @@
#version 300 es #version 300 es
// Copyright 2022 The Android Open Source Project // Copyright 2021 The Android Open Source Project
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -13,15 +13,15 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture // ES 3 vertex shader that applies the 4 * 4 transformation matrices
// transformation matrix to convert the texture coordinates to the sampling // uTransformationMatrix and the uTexTransformationMatrix.
// locations.
in vec4 aFramePosition; in vec4 aFramePosition;
uniform mat4 uTexTransform; uniform mat4 uTransformationMatrix;
uniform mat4 uTexTransformationMatrix;
out vec2 vTexSamplingCoord; out vec2 vTexSamplingCoord;
void main() { void main() {
gl_Position = aFramePosition; gl_Position = uTransformationMatrix * aFramePosition;
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0); vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransform * texturePosition).xy; vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
} }

104
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalTextureProcessor.java
View File

@ -15,71 +15,13 @@
*/ */
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkArgument; /**
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; * Interface for a {@link GlTextureProcessor} that samples from an external texture.
*
import android.content.Context; * <p>Use {@link #setTextureTransformMatrix(float[])} to provide the texture's transformation
import android.opengl.GLES20; * matrix.
import android.util.Size; */
import com.google.android.exoplayer2.util.GlProgram; /* package */ interface ExternalTextureProcessor extends GlTextureProcessor {
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
/** Copies frames from an external texture and applies color transformations for HDR if needed. */
/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH =
"shaders/vertex_shader_tex_transform_es2.glsl";
private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH =
"shaders/vertex_shader_tex_transform_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
private final GlProgram glProgram;
/**
* Creates a new instance.
*
* @param useHdr Whether to process the input as an HDR signal.
* @throws FrameProcessingException If a problem occurs while reading shader files.
*/
public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException {
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
try {
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
}
@Override
public Size configure(int inputWidth, int inputHeight) {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
return new Size(inputWidth, inputHeight);
}
/** /**
* Sets the texture transform matrix for converting an external surface texture's coordinates to * Sets the texture transform matrix for converting an external surface texture's coordinates to
@ -88,35 +30,5 @@ import java.io.IOException;
* @param textureTransformMatrix The external surface texture's {@linkplain * @param textureTransformMatrix The external surface texture's {@linkplain
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}. * android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
*/ */
public void setTextureTransformMatrix(float[] textureTransformMatrix) { void setTextureTransformMatrix(float[] textureTransformMatrix);
checkStateNotNull(glProgram);
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
}
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
}
}
@Override
public void release() throws FrameProcessingException {
super.release();
if (glProgram != null) {
try {
glProgram.delete();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
}
} }

29
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java
View File

@ -24,6 +24,7 @@ import android.opengl.EGLDisplay;
import android.opengl.EGLExt; import android.opengl.EGLExt;
import android.opengl.EGLSurface; import android.opengl.EGLSurface;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.opengl.Matrix;
import android.util.Size; import android.util.Size;
import android.view.Surface; import android.view.Surface;
import android.view.SurfaceHolder; import android.view.SurfaceHolder;
@ -50,7 +51,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link * <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link
* GlTextureProcessor} instances used by {@link FrameProcessor}. * GlTextureProcessor} instances used by {@link FrameProcessor}.
*/ */
/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor { /* package */ final class FinalMatrixTransformationProcessorWrapper
implements GlTextureProcessor, ExternalTextureProcessor {
private static final String TAG = "FinalProcessorWrapper"; private static final String TAG = "FinalProcessorWrapper";
@ -61,7 +63,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final long streamOffsetUs; private final long streamOffsetUs;
private final DebugViewProvider debugViewProvider; private final DebugViewProvider debugViewProvider;
private final FrameProcessor.Listener frameProcessorListener; private final FrameProcessor.Listener frameProcessorListener;
private final boolean sampleFromExternalTexture;
private final boolean useHdr; private final boolean useHdr;
private final float[] textureTransformMatrix;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs, long streamOffsetUs,
FrameProcessor.Listener frameProcessorListener, FrameProcessor.Listener frameProcessorListener,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean sampleFromExternalTexture,
boolean useHdr) { boolean useHdr) {
this.context = context; this.context = context;
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
@ -94,7 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.streamOffsetUs = streamOffsetUs; this.streamOffsetUs = streamOffsetUs;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
this.frameProcessorListener = frameProcessorListener; this.frameProcessorListener = frameProcessorListener;
this.sampleFromExternalTexture = sampleFromExternalTexture;
this.useHdr = useHdr; this.useHdr = useHdr;
textureTransformMatrix = new float[16];
Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
} }
/** /**
@ -239,7 +248,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT)); outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
MatrixTransformationProcessor matrixTransformationProcessor = MatrixTransformationProcessor matrixTransformationProcessor =
new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build()); new MatrixTransformationProcessor(
context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr);
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight); Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight);
checkState(outputSize.getWidth() == outputSurfaceInfo.width); checkState(outputSize.getWidth() == outputSurfaceInfo.width);
checkState(outputSize.getHeight() == outputSurfaceInfo.height); checkState(outputSize.getHeight() == outputSurfaceInfo.height);
@ -265,6 +276,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
} }
@Override
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
System.arraycopy(
/* src= */ textureTransformMatrix,
/* srcPos= */ 0,
/* dest= */ this.textureTransformMatrix,
/* destPost= */ 0,
/* length= */ textureTransformMatrix.length);
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
}
}
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) { if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
this.outputSurfaceInfo = outputSurfaceInfo; this.outputSurfaceInfo = outputSurfaceInfo;

202
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java
View File

@ -17,13 +17,13 @@ package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import static com.google.common.collect.Iterables.getLast;
import android.content.Context; import android.content.Context;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.opengl.EGL14; import android.opengl.EGL14;
import android.opengl.EGLContext; import android.opengl.EGLContext;
import android.opengl.EGLDisplay; import android.opengl.EGLDisplay;
import android.util.Pair;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.WorkerThread; import androidx.annotation.WorkerThread;
@ -126,31 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
} }
Pair<ImmutableList<GlTextureProcessor>, FinalMatrixTransformationProcessorWrapper> ImmutableList<GlTextureProcessor> textureProcessors =
textureProcessors = getGlTextureProcessorsForGlEffects(
getGlTextureProcessorsForGlEffects( context,
context, effects,
effects, eglDisplay,
eglDisplay, eglContext,
eglContext, streamOffsetUs,
streamOffsetUs, listener,
listener, debugViewProvider,
debugViewProvider, useHdr);
useHdr);
ImmutableList<GlTextureProcessor> intermediateTextureProcessors = textureProcessors.first;
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper =
textureProcessors.second;
ExternalTextureProcessor externalTextureProcessor =
new ExternalTextureProcessor(context, useHdr);
FrameProcessingTaskExecutor frameProcessingTaskExecutor = FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
chainTextureProcessorsWithListeners( chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener);
externalTextureProcessor,
intermediateTextureProcessors,
finalTextureProcessorWrapper,
frameProcessingTaskExecutor,
listener);
return new GlEffectsFrameProcessor( return new GlEffectsFrameProcessor(
eglDisplay, eglDisplay,
@ -158,9 +146,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
frameProcessingTaskExecutor, frameProcessingTaskExecutor,
streamOffsetUs, streamOffsetUs,
/* inputExternalTextureId= */ GlUtil.createExternalTexture(), /* inputExternalTextureId= */ GlUtil.createExternalTexture(),
externalTextureProcessor, textureProcessors);
intermediateTextureProcessors,
finalTextureProcessorWrapper);
} }
/** /**
@ -168,25 +154,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate * MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate
* {@link GlTextureProcessor} instances. * {@link GlTextureProcessor} instances.
* *
* @return A {@link Pair} containing a list of {@link GlTextureProcessor} instances to apply in * @return A non-empty list of {@link GlTextureProcessor} instances to apply in the given order.
* the given order and a {@link FinalMatrixTransformationProcessorWrapper} to apply after * The first is an {@link ExternalTextureProcessor} and the last is a {@link
* them. * FinalMatrixTransformationProcessorWrapper}.
*/ */
private static Pair<ImmutableList<GlTextureProcessor>, FinalMatrixTransformationProcessorWrapper> private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffects(
getGlTextureProcessorsForGlEffects( Context context,
Context context, List<GlEffect> effects,
List<GlEffect> effects, EGLDisplay eglDisplay,
EGLDisplay eglDisplay, EGLContext eglContext,
EGLContext eglContext, long streamOffsetUs,
long streamOffsetUs, FrameProcessor.Listener listener,
FrameProcessor.Listener listener, DebugViewProvider debugViewProvider,
DebugViewProvider debugViewProvider, boolean useHdr)
boolean useHdr) throws FrameProcessingException {
throws FrameProcessingException {
ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder = ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
new ImmutableList.Builder<>(); new ImmutableList.Builder<>();
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder = ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
new ImmutableList.Builder<>(); new ImmutableList.Builder<>();
boolean sampleFromExternalTexture = true;
for (int i = 0; i < effects.size(); i++) { for (int i = 0; i < effects.size(); i++) {
GlEffect effect = effects.get(i); GlEffect effect = effects.get(i);
if (effect instanceof GlMatrixTransformation) { if (effect instanceof GlMatrixTransformation) {
@ -195,15 +181,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
ImmutableList<GlMatrixTransformation> matrixTransformations = ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build(); matrixTransformationListBuilder.build();
if (!matrixTransformations.isEmpty()) { if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) {
textureProcessorListBuilder.add( textureProcessorListBuilder.add(
new MatrixTransformationProcessor(context, matrixTransformations)); new MatrixTransformationProcessor(
context, matrixTransformations, sampleFromExternalTexture, useHdr));
matrixTransformationListBuilder = new ImmutableList.Builder<>(); matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
} }
textureProcessorListBuilder.add(effect.toGlTextureProcessor(context)); textureProcessorListBuilder.add(effect.toGlTextureProcessor(context));
} }
return Pair.create( textureProcessorListBuilder.add(
textureProcessorListBuilder.build(),
new FinalMatrixTransformationProcessorWrapper( new FinalMatrixTransformationProcessorWrapper(
context, context,
eglDisplay, eglDisplay,
@ -212,51 +199,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs, streamOffsetUs,
listener, listener,
debugViewProvider, debugViewProvider,
sampleFromExternalTexture,
useHdr)); useHdr));
return textureProcessorListBuilder.build();
} }
/** /**
* Chains the given {@link GlTextureProcessor} instances using {@link * Chains the given {@link GlTextureProcessor} instances using {@link
* ChainingGlTextureProcessorListener} instances. * ChainingGlTextureProcessorListener} instances.
*
* <p>The {@link ExternalTextureProcessor} is the first processor in the chain.
*/ */
private static void chainTextureProcessorsWithListeners( private static void chainTextureProcessorsWithListeners(
ExternalTextureProcessor externalTextureProcessor, ImmutableList<GlTextureProcessor> textureProcessors,
ImmutableList<GlTextureProcessor> intermediateTextureProcessors,
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper,
FrameProcessingTaskExecutor frameProcessingTaskExecutor, FrameProcessingTaskExecutor frameProcessingTaskExecutor,
FrameProcessor.Listener listener) { FrameProcessor.Listener listener) {
externalTextureProcessor.setListener( for (int i = 0; i < textureProcessors.size(); i++) {
new ChainingGlTextureProcessorListener( @Nullable
/* previousGlTextureProcessor= */ null, GlTextureProcessor previousGlTextureProcessor =
/* nextGlTextureProcessor= */ intermediateTextureProcessors.size() > 0 i - 1 >= 0 ? textureProcessors.get(i - 1) : null;
? intermediateTextureProcessors.get(0)
: finalTextureProcessorWrapper,
frameProcessingTaskExecutor,
listener));
GlTextureProcessor previousGlTextureProcessor = externalTextureProcessor;
for (int i = 0; i < intermediateTextureProcessors.size(); i++) {
GlTextureProcessor glTextureProcessor = intermediateTextureProcessors.get(i);
@Nullable @Nullable
GlTextureProcessor nextGlTextureProcessor = GlTextureProcessor nextGlTextureProcessor =
i + 1 < intermediateTextureProcessors.size() i + 1 < textureProcessors.size() ? textureProcessors.get(i + 1) : null;
? intermediateTextureProcessors.get(i + 1) textureProcessors
: finalTextureProcessorWrapper; .get(i)
glTextureProcessor.setListener( .setListener(
new ChainingGlTextureProcessorListener( new ChainingGlTextureProcessorListener(
previousGlTextureProcessor, previousGlTextureProcessor,
nextGlTextureProcessor, nextGlTextureProcessor,
frameProcessingTaskExecutor, frameProcessingTaskExecutor,
listener)); listener));
previousGlTextureProcessor = glTextureProcessor;
} }
finalTextureProcessorWrapper.setListener(
new ChainingGlTextureProcessorListener(
previousGlTextureProcessor,
/* nextGlTextureProcessor= */ null,
frameProcessingTaskExecutor,
listener));
} }
private static final String TAG = "GlEffectsFrameProcessor"; private static final String TAG = "GlEffectsFrameProcessor";
@ -280,11 +251,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final float[] inputSurfaceTextureTransformMatrix; private final float[] inputSurfaceTextureTransformMatrix;
private final int inputExternalTextureId; private final int inputExternalTextureId;
private final ExternalTextureProcessor inputExternalTextureProcessor; private final ExternalTextureProcessor inputExternalTextureProcessor;
private final ImmutableList<GlTextureProcessor> intermediateTextureProcessors;
private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper; private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper;
private final ImmutableList<GlTextureProcessor> allTextureProcessors;
private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames; private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames;
// Fields accessed on the thread used by the GlEffectsFrameProcessor's caller.
private @MonotonicNonNull FrameInfo nextInputFrameInfo; private @MonotonicNonNull FrameInfo nextInputFrameInfo;
// Fields accessed on the frameProcessingTaskExecutor's thread.
private boolean inputTextureInUse;
private boolean inputStreamEnded; private boolean inputStreamEnded;
private GlEffectsFrameProcessor( private GlEffectsFrameProcessor(
@ -293,18 +268,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
FrameProcessingTaskExecutor frameProcessingTaskExecutor, FrameProcessingTaskExecutor frameProcessingTaskExecutor,
long streamOffsetUs, long streamOffsetUs,
int inputExternalTextureId, int inputExternalTextureId,
ExternalTextureProcessor inputExternalTextureProcessor, ImmutableList<GlTextureProcessor> textureProcessors) {
ImmutableList<GlTextureProcessor> intermediateTextureProcessors,
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper) {
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
this.streamOffsetUs = streamOffsetUs; this.streamOffsetUs = streamOffsetUs;
this.inputExternalTextureId = inputExternalTextureId; this.inputExternalTextureId = inputExternalTextureId;
this.inputExternalTextureProcessor = inputExternalTextureProcessor;
this.intermediateTextureProcessors = intermediateTextureProcessors; checkState(!textureProcessors.isEmpty());
this.finalTextureProcessorWrapper = finalTextureProcessorWrapper; checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor);
checkState(getLast(textureProcessors) instanceof FinalMatrixTransformationProcessorWrapper);
inputExternalTextureProcessor = (ExternalTextureProcessor) textureProcessors.get(0);
finalTextureProcessorWrapper =
(FinalMatrixTransformationProcessorWrapper) getLast(textureProcessors);
allTextureProcessors = textureProcessors;
inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId); inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId);
inputSurface = new Surface(inputSurfaceTexture); inputSurface = new Surface(inputSurfaceTexture);
@ -321,7 +299,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public void setInputFrameInfo(FrameInfo inputFrameInfo) { public void setInputFrameInfo(FrameInfo inputFrameInfo) {
nextInputFrameInfo = inputFrameInfo; nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo);
} }
@Override @Override
@ -365,36 +343,54 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
/** /**
* Processes an input frame from the {@linkplain #getInputSurface() external input surface * Processes an input frame from the {@link #inputSurfaceTexture}.
* texture}.
* *
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}. * <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
*/ */
@WorkerThread @WorkerThread
private void processInputFrame() { private void processInputFrame() {
checkState(Thread.currentThread().getName().equals(THREAD_NAME)); checkState(Thread.currentThread().getName().equals(THREAD_NAME));
if (!inputExternalTextureProcessor.acceptsInputFrame()) { if (inputTextureInUse) {
frameProcessingTaskExecutor.submit(this::processInputFrame); // Try again later. frameProcessingTaskExecutor.submit(this::processInputFrame); // Try again later.
return; return;
} }
inputTextureInUse = true;
inputSurfaceTexture.updateTexImage(); inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix);
queueInputFrameToTextureProcessors();
}
/**
* Queues the input frame to the first texture processor until it is accepted.
*
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
*/
@WorkerThread
private void queueInputFrameToTextureProcessors() {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
checkState(inputTextureInUse);
long inputFrameTimeNs = inputSurfaceTexture.getTimestamp(); long inputFrameTimeNs = inputSurfaceTexture.getTimestamp();
// Correct for the stream offset so processors see original media presentation timestamps. // Correct for the stream offset so processors see original media presentation timestamps.
long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs; long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs;
inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix);
inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix); inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix);
FrameInfo inputFrameInfo = adjustForPixelWidthHeightRatio(pendingInputFrames.remove()); FrameInfo inputFrameInfo = checkStateNotNull(pendingInputFrames.peek());
checkState( if (inputExternalTextureProcessor.maybeQueueInputFrame(
inputExternalTextureProcessor.maybeQueueInputFrame( new TextureInfo(
new TextureInfo( inputExternalTextureId,
inputExternalTextureId, /* fboId= */ C.INDEX_UNSET,
/* fboId= */ C.INDEX_UNSET, inputFrameInfo.width,
inputFrameInfo.width, inputFrameInfo.height),
inputFrameInfo.height), presentationTimeUs)) {
presentationTimeUs)); inputTextureInUse = false;
// After the inputExternalTextureProcessor has produced an output frame, it is processed pendingInputFrames.remove();
// asynchronously by the texture processors chained after it. // After the externalTextureProcessor has produced an output frame, it is processed
// asynchronously by the texture processors chained after it.
} else {
// Try again later.
frameProcessingTaskExecutor.submit(this::queueInputFrameToTextureProcessors);
}
} }
/** /**
@ -442,11 +438,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@WorkerThread @WorkerThread
private void releaseTextureProcessorsAndDestroyGlContext() private void releaseTextureProcessorsAndDestroyGlContext()
throws GlUtil.GlException, FrameProcessingException { throws GlUtil.GlException, FrameProcessingException {
inputExternalTextureProcessor.release(); for (int i = 0; i < allTextureProcessors.size(); i++) {
for (int i = 0; i < intermediateTextureProcessors.size(); i++) { allTextureProcessors.get(i).release();
intermediateTextureProcessors.get(i).release();
} }
finalTextureProcessorWrapper.release();
GlUtil.destroyEglContext(eglDisplay, eglContext); GlUtil.destroyEglContext(eglDisplay, eglContext);
} }
} }

73
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java
View File

@ -36,19 +36,35 @@ import java.util.Arrays;
* matrices are clipped to the NDC range. * matrices are clipped to the NDC range.
* *
* <p>The background color of the output frame will be (r=0, g=0, b=0, a=0). * <p>The background color of the output frame will be (r=0, g=0, b=0, a=0).
*
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
*/ */
@SuppressWarnings("FunctionalInterfaceClash") // b/228192298 @SuppressWarnings("FunctionalInterfaceClash") // b/228192298
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor { /* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor
implements ExternalTextureProcessor {
private static final String VERTEX_SHADER_TRANSFORMATION_PATH = private static final String VERTEX_SHADER_TRANSFORMATION_PATH =
"shaders/vertex_shader_transformation_es2.glsl"; "shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
private static final ImmutableList<float[]> NDC_SQUARE = private static final ImmutableList<float[]> NDC_SQUARE =
ImmutableList.of( ImmutableList.of(
new float[] {-1, -1, 0, 1}, new float[] {-1, -1, 0, 1},
new float[] {-1, 1, 0, 1}, new float[] {-1, 1, 0, 1},
new float[] {1, 1, 0, 1}, new float[] {1, 1, 0, 1},
new float[] {1, -1, 0, 1}); new float[] {1, -1, 0, 1});
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
/** The {@link MatrixTransformation MatrixTransformations} to apply. */ /** The {@link MatrixTransformation MatrixTransformations} to apply. */
private final ImmutableList<GlMatrixTransformation> matrixTransformations; private final ImmutableList<GlMatrixTransformation> matrixTransformations;
@ -87,7 +103,11 @@ import java.util.Arrays;
*/ */
public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation) public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this(context, ImmutableList.of(matrixTransformation)); this(
context,
ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false,
/* enableExperimentalHdrEditing= */ false);
} }
/** /**
@ -100,7 +120,11 @@ import java.util.Arrays;
*/ */
public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation) public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this(context, ImmutableList.of(matrixTransformation)); this(
context,
ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false,
/* enableExperimentalHdrEditing= */ false);
} }
/** /**
@ -109,10 +133,17 @@ import java.util.Arrays;
* @param context The {@link Context}. * @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. * apply to each frame in order.
* @param sampleFromExternalTexture Whether the input will be provided using an external texture.
* If {@code true}, the caller should use {@link #setTextureTransformMatrix(float[])} to
* provide the transformation matrix associated with the external texture.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor( public MatrixTransformationProcessor(
Context context, ImmutableList<GlMatrixTransformation> matrixTransformations) Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean sampleFromExternalTexture,
boolean enableExperimentalHdrEditing)
throws FrameProcessingException { throws FrameProcessingException {
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
@ -121,11 +152,41 @@ import java.util.Arrays;
tempResultMatrix = new float[16]; tempResultMatrix = new float[16];
Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
visiblePolygon = NDC_SQUARE; visiblePolygon = NDC_SQUARE;
String vertexShaderFilePath;
String fragmentShaderFilePath;
if (sampleFromExternalTexture) {
vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TRANSFORMATION_ES3_PATH
: VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath =
enableExperimentalHdrEditing
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
} else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
}
try { try {
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) { } catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e); throw new FrameProcessingException(e);
} }
if (enableExperimentalHdrEditing && sampleFromExternalTexture) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
float[] identityMatrix = new float[16];
Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
}
@Override
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
glProgram.setFloatsUniform("uTexTransformationMatrix", textureTransformMatrix);
} }
@Override @Override

10
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java
View File

@ -73,16 +73,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
this.listener = listener; this.listener = listener;
} }
/**
* Returns whether the {@code SingleFrameGlTextureProcessor} can accept an input frame.
*
* <p>If this method returns {@code true}, the next call to {@link #maybeQueueInputFrame(
* TextureInfo, long)} will also return {@code true}.
*/
public boolean acceptsInputFrame() {
return !outputTextureInUse;
}
@Override @Override
public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
if (outputTextureInUse) { if (outputTextureInUse) {