loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge RgbProcessor and MatrixTransformation.

Browse Source 
PiperOrigin-RevId: 472325145
(cherry picked from commit 672405afdff53ee746a4bcd237170de3b9feb1f7)
This commit is contained in:
leonwind 2022-09-05 21:51:11 +00:00 committed by microkatz
parent 1bf431566f
commit 8da65d01d2
15 changed files with 207 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java
View File

@ -28,7 +28,6 @@ import android.opengl.EGLSurface;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlUtil;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
@ -97,8 +96,7 @@ public final class MatrixTransformationProcessorPixelTest {
Matrix identityMatrix = new Matrix();
MatrixTransformation noEditsTransformation = (long presentationTimeUs) -> identityMatrix;
matrixTransformationFrameProcessor =
MatrixTransformationProcessor.create(
context, ImmutableList.of(noEditsTransformation), /* useHdr= */ false);
noEditsTransformation.toGlTextureProcessor(context, /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
@ -123,8 +121,7 @@ public final class MatrixTransformationProcessorPixelTest {
MatrixTransformation translateRightTransformation =
(long presentationTimeUs) -> translateRightMatrix;
matrixTransformationFrameProcessor =
MatrixTransformationProcessor.create(
context, ImmutableList.of(translateRightTransformation), /* useHdr= */ false);
translateRightTransformation.toGlTextureProcessor(context, /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
@ -148,8 +145,7 @@ public final class MatrixTransformationProcessorPixelTest {
scaleNarrowMatrix.postScale(.5f, 1.2f);
MatrixTransformation scaleNarrowTransformation = (long presentationTimeUs) -> scaleNarrowMatrix;
matrixTransformationFrameProcessor =
MatrixTransformationProcessor.create(
context, ImmutableList.of(scaleNarrowTransformation), /* useHdr= */ false);
scaleNarrowTransformation.toGlTextureProcessor(context, /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH);
@ -173,8 +169,7 @@ public final class MatrixTransformationProcessorPixelTest {
rotate90Matrix.postRotate(/* degrees= */ 90);
MatrixTransformation rotate90Transformation = (long presentationTimeUs) -> rotate90Matrix;
matrixTransformationFrameProcessor =
MatrixTransformationProcessor.create(
context, ImmutableList.of(rotate90Transformation), /* useHdr= */ false);
rotate90Transformation.toGlTextureProcessor(context, /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH);

86
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java
View File

@ -62,7 +62,7 @@ public final class RgbAdjustmentPixelTest {
private @MonotonicNonNull EGLDisplay eglDisplay;
private @MonotonicNonNull EGLContext eglContext;
private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor;
private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor;
private @MonotonicNonNull EGLSurface placeholderEglSurface;
private int inputTexId;
private int outputTexId;
@ -94,8 +94,8 @@ public final class RgbAdjustmentPixelTest {
@After
public void release() throws GlUtil.GlException, FrameProcessingException {
if (rgbMatrixProcessor != null) {
rgbMatrixProcessor.release();
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
@ -104,11 +104,13 @@ public final class RgbAdjustmentPixelTest {
public void drawFrame_identityMatrix_leavesFrameUnchanged() throws Exception {
String testId = "drawFrame_identityMatrix";
RgbMatrix identityMatrix = new RgbAdjustment.Builder().build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, identityMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
identityMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -126,13 +128,15 @@ public final class RgbAdjustmentPixelTest {
String testId = "drawFrame_removeColors";
RgbMatrix removeColorMatrix =
new RgbAdjustment.Builder().setRedScale(0).setGreenScale(0).setBlueScale(0).build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, removeColorMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
removeColorMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap =
BitmapTestUtil.createArgb8888BitmapWithSolidColor(
outputSize.first, outputSize.second, Color.BLACK);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -149,11 +153,13 @@ public final class RgbAdjustmentPixelTest {
public void drawFrame_redOnlyFilter_removeBlueAndGreenValues() throws Exception {
String testId = "drawFrame_redOnlyFilter";
RgbMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
redOnlyMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -170,11 +176,13 @@ public final class RgbAdjustmentPixelTest {
public void drawFrame_increaseRedChannel_producesBrighterAndRedderFrame() throws Exception {
String testId = "drawFrame_increaseRedChannel";
RgbMatrix increaseRedMatrix = new RgbAdjustment.Builder().setRedScale(5).build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, increaseRedMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
increaseRedMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_RED_CHANNEL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -192,12 +200,13 @@ public final class RgbAdjustmentPixelTest {
String testId = "drawFrame_increaseBrightness";
RgbMatrix increaseBrightnessMatrix =
new RgbAdjustment.Builder().setRedScale(5).setGreenScale(5).setBlueScale(5).build();
rgbMatrixProcessor =
new RgbMatrixProcessor(context, increaseBrightnessMatrix, /* useHdr = */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
increaseBrightnessMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -216,15 +225,19 @@ public final class RgbAdjustmentPixelTest {
RgbMatrix noRed = new RgbAdjustment.Builder().setRedScale(0).build();
RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build();
RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build();
rgbMatrixProcessor =
new RgbMatrixProcessor(
context, ImmutableList.of(noRed, noGreen, noBlue), /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(noRed, noGreen, noBlue),
/* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap =
BitmapTestUtil.createArgb8888BitmapWithSolidColor(
outputSize.first, outputSize.second, Color.BLACK);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -242,12 +255,17 @@ public final class RgbAdjustmentPixelTest {
String testId = "drawFrame_removeBlueAndGreenValuesInAChain";
RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build();
RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build();
rgbMatrixProcessor =
new RgbMatrixProcessor(context, ImmutableList.of(noGreen, noBlue), /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(noGreen, noBlue),
/* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -267,15 +285,17 @@ public final class RgbAdjustmentPixelTest {
RgbMatrix scaleRedMatrix = new RgbAdjustment.Builder().setRedScale(redScale).build();
RgbMatrix scaleRedByInverseMatrix =
new RgbAdjustment.Builder().setRedScale(1 / redScale).build();
rgbMatrixProcessor =
new RgbMatrixProcessor(
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context,
ImmutableList.of(scaleRedMatrix, scaleRedByInverseMatrix),
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(scaleRedMatrix, scaleRedByInverseMatrix),
/* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);

22
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java
View File

@ -58,7 +58,7 @@ public final class RgbFilterPixelTest {
private @MonotonicNonNull EGLDisplay eglDisplay;
private @MonotonicNonNull EGLContext eglContext;
private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor;
private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor;
private @MonotonicNonNull EGLSurface placeholderEglSurface;
private int inputTexId;
private int outputTexId;
@ -90,8 +90,8 @@ public final class RgbFilterPixelTest {
@After
public void release() throws GlUtil.GlException, FrameProcessingException {
if (rgbMatrixProcessor != null) {
rgbMatrixProcessor.release();
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
@ -100,11 +100,13 @@ public final class RgbFilterPixelTest {
public void drawFrame_grayscale_producesGrayscaleImage() throws Exception {
String testId = "drawFrame_grayscale";
RgbMatrix grayscaleMatrix = RgbFilter.createGrayscaleFilter();
rgbMatrixProcessor = new RgbMatrixProcessor(context, grayscaleMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
grayscaleMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -121,11 +123,13 @@ public final class RgbFilterPixelTest {
public void drawFrame_inverted_producesInvertedFrame() throws Exception {
String testId = "drawFrame_inverted";
RgbMatrix invertedMatrix = RgbFilter.createInvertedFilter();
rgbMatrixProcessor = new RgbMatrixProcessor(context, invertedMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
invertedMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INVERT_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);

24
libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl
View File

@ -1,24 +0,0 @@
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that samples from a (non-external) texture with uTexSampler,
// copying from this texture to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
varying vec2 vTexSamplingCoord;
void main() {
gl_FragColor = texture2D(uTexSampler, vTexSamplingCoord);
}

7
libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl
View File

@ -15,7 +15,8 @@
// ES 3 fragment shader that:
// 1. samples optical linear BT.2020 RGB from a (non-external) texture with
// uTexSampler,
// uTexSampler, and applies a 4x4 RGB color matrix to change the pixel
// colors,
// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB,
// and
// 3. copies this converted texture color to the current output.
@ -28,6 +29,7 @@ out vec4 outColor;
// Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed.
uniform int uOetfColorTransfer;
uniform mat3 uColorTransform;
uniform mat4 uRgbMatrix;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization.
@ -83,5 +85,6 @@ highp vec3 getElectricalColor(highp vec3 linearColor) {
void main() {
vec4 inputColor = texture(uTexSampler, vTexSamplingCoord);
outColor = vec4(getElectricalColor(inputColor.rgb), inputColor.a);
vec4 transformedColors = uRgbMatrix * vec4(inputColor.rgb, 1);
outColor = vec4(getElectricalColor(transformedColors.rgb), inputColor.a);
}

4
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl
View File

@ -19,11 +19,11 @@
precision mediump float;
uniform sampler2D uTexSampler;
uniform mat4 uColorMatrix;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;
void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
gl_FragColor = uColorMatrix * vec4(inputColor.rgb, 1);
gl_FragColor = uRgbMatrix * vec4(inputColor.rgb, 1);
gl_FragColor.a = inputColor.a;
}

9
libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl → libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_es2.glsl
View File

@ -14,12 +14,17 @@
// limitations under the License.
// ES 2 fragment shader that samples from an external texture with uTexSampler,
// copying from this texture to the current output.
// copying from this texture to the current output while applying a 4x4 RGB
// color matrix to change the pixel colors.
#extension GL_OES_EGL_image_external : require
precision mediump float;
uniform samplerExternalOES uTexSampler;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;
void main() {
gl_FragColor = texture2D(uTexSampler, vTexSamplingCoord);
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
gl_FragColor = uRgbMatrix * vec4(inputColor.rgb, 1);
gl_FragColor.a = inputColor.a;
}

6
libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl → libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl
View File

@ -23,13 +23,15 @@
// 3. If uEotfColorTransfer is COLOR_TRANSFER_NO_VALUE, outputs electrical
// (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for
// intermediate shaders by applying the HLG or PQ EOTF.
// 4. Copies this converted texture color to the current output, with alpha = 1.
// 4. Copies this converted texture color to the current output, with alpha = 1,
// while applying a 4x4 RGB color matrix to change the pixel colors.
#extension GL_OES_EGL_image_external : require
#extension GL_EXT_YUV_target : require
precision mediump float;
uniform __samplerExternal2DY2YEXT uTexSampler;
uniform mat3 uYuvToRgbColorTransform;
uniform mat4 uRgbMatrix;
// C.java#ColorTransfer value.
uniform int uEotfColorTransfer;
in vec2 vTexSamplingCoord;
@ -101,5 +103,5 @@ vec3 yuvToRgb(vec3 yuv) {
void main() {
vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
vec3 rgb = yuvToRgb(srcYuv);
outColor = vec4(getOpticalColor(rgb), 1.0);
outColor = uRgbMatrix * vec4(getOpticalColor(rgb), 1.0);
}

7
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
View File

@ -66,6 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Context context;
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
private final ImmutableList<RgbMatrix> rgbMatrices;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final DebugViewProvider debugViewProvider;
@ -100,6 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EGLDisplay eglDisplay,
EGLContext eglContext,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
FrameProcessor.Listener frameProcessorListener,
DebugViewProvider debugViewProvider,
boolean sampleFromExternalTexture,
@ -107,6 +109,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
boolean releaseFramesAutomatically) {
this.context = context;
this.matrixTransformations = matrixTransformations;
this.rgbMatrices = rgbMatrices;
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.debugViewProvider = debugViewProvider;
@ -374,11 +377,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (sampleFromExternalTexture) {
matrixTransformationProcessor =
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotfThenOetf(
context, expandedMatrixTransformations, colorInfo);
context, expandedMatrixTransformations, rgbMatrices, colorInfo);
} else {
matrixTransformationProcessor =
MatrixTransformationProcessor.createApplyingOetf(
context, expandedMatrixTransformations, colorInfo);
context, expandedMatrixTransformations, rgbMatrices, colorInfo);
}
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);

40
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
View File

@ -184,8 +184,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
new ImmutableList.Builder<>();
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
new ImmutableList.Builder<>();
ImmutableList.Builder<RgbMatrix> rgbaMatrixTransformationListBuilder =
new ImmutableList.Builder<>();
ImmutableList.Builder<RgbMatrix> rgbMatrixListBuilder = new ImmutableList.Builder<>();
boolean sampleFromExternalTexture = true;
for (int i = 0; i < effects.size(); i++) {
Effect effect = effects.get(i);
@ -200,62 +199,39 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
continue;
}
if (glEffect instanceof RgbMatrix) {
rgbaMatrixTransformationListBuilder.add((RgbMatrix) glEffect);
rgbMatrixListBuilder.add((RgbMatrix) glEffect);
continue;
}
ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build();
if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) {
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromExternalTexture) {
MatrixTransformationProcessor matrixTransformationProcessor;
if (sampleFromExternalTexture) {
matrixTransformationProcessor =
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf(
context, matrixTransformations, colorInfo);
context, matrixTransformations, rgbMatrices, colorInfo);
} else {
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context, matrixTransformations, ColorInfo.isTransferHdr(colorInfo));
context, matrixTransformations, rgbMatrices, ColorInfo.isTransferHdr(colorInfo));
}
textureProcessorListBuilder.add(matrixTransformationProcessor);
matrixTransformationListBuilder = new ImmutableList.Builder<>();
rgbMatrixListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
}
ImmutableList<RgbMatrix> rgbaMatrixTransformations =
rgbaMatrixTransformationListBuilder.build();
if (!rgbaMatrixTransformations.isEmpty()) {
textureProcessorListBuilder.add(
new RgbMatrixProcessor(
context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo)));
rgbaMatrixTransformationListBuilder = new ImmutableList.Builder<>();
}
textureProcessorListBuilder.add(
glEffect.toGlTextureProcessor(context, ColorInfo.isTransferHdr(colorInfo)));
}
ImmutableList<RgbMatrix> rgbaMatrixTransformations =
rgbaMatrixTransformationListBuilder.build();
if (!rgbaMatrixTransformations.isEmpty()) {
// Add a MatrixTransformationProcessor if none yet exists for sampling from an external
// texture.
if (sampleFromExternalTexture) {
// TODO(b/239757183): Remove the unnecessary MatrixTransformationProcessor after it got
// merged with RgbMatrixProcessor.
textureProcessorListBuilder.add(
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf(
context, /* matrixTransformations= */ ImmutableList.of(), colorInfo));
sampleFromExternalTexture = false;
}
textureProcessorListBuilder.add(
new RgbMatrixProcessor(
context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo)));
}
textureProcessorListBuilder.add(
new FinalMatrixTransformationProcessorWrapper(
context,
eglDisplay,
eglContext,
matrixTransformationListBuilder.build(),
rgbMatrixListBuilder.build(),
listener,
debugViewProvider,
sampleFromExternalTexture,

5
libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java
View File

@ -56,6 +56,9 @@ public interface GlMatrixTransformation extends GlEffect {
default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException {
return MatrixTransformationProcessor.create(
context, /* matrixTransformations= */ ImmutableList.of(this), useHdr);
context,
/* matrixTransformations= */ ImmutableList.of(this),
/* rgbMatrices= */ ImmutableList.of(),
useHdr);
}
}

113
libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
View File

@ -34,17 +34,22 @@ import java.io.IOException;
import java.util.Arrays;
/**
* Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into
* an output frame based on their locations after applying the sequence of transformation matrices.
* Applies a sequence of {@link MatrixTransformation MatrixTransformations} in the vertex shader and
* a sequence of {@link RgbMatrix RgbMatrices} in the fragment shader. Copies input pixels into an
* output frame based on their locations after applying the sequence of transformation matrices.
*
* <p>Operations are done on normalized device coordinates (-1 to 1 on x, y, and z axes).
* Transformed vertices that are moved outside of this range after any of the transformation
* matrices are clipped to the NDC range.
* <p>{@link MatrixTransformation} operations are done on normalized device coordinates (-1 to 1 on
* x, y, and z axes). Transformed vertices that are moved outside of this range after any of the
* transformation matrices are clipped to the NDC range.
*
* <p>After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the
* color space (e.g. BT.709 for SDR). Intermediate results are not clamped.
*
* <p>The background color of the output frame will be (r=0, g=0, b=0, a=0).
*
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
*/
// TODO(b/239757183): Rename Matrix to a more generic name.
@UnstableApi
@SuppressWarnings("FunctionalInterfaceClash") // b/228192298
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor
@ -54,13 +59,14 @@ import java.util.Arrays;
"shaders/vertex_shader_transformation_es2.glsl";
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_PATH =
"shaders/fragment_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_OETF_ES3_PATH =
"shaders/fragment_shader_oetf_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH =
"shaders/fragment_shader_transformation_external_es2.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_transformation_external_yuv_es3.glsl";
private static final ImmutableList<float[]> NDC_SQUARE =
ImmutableList.of(
new float[] {-1, -1, 0, 1},
@ -84,6 +90,10 @@ import java.util.Arrays;
/** The {@link MatrixTransformation MatrixTransformations} to apply. */
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
/** The {@link RgbMatrix RgbMatrices} to apply. */
private final ImmutableList<RgbMatrix> rgbMatrices;
/** Whether the frame is in HDR or not. */
private final boolean useHdr;
/**
* The transformation matrices provided by the {@link MatrixTransformation MatrixTransformations}
* for the most recent frame.
@ -117,19 +127,25 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param useHdr Whether input and output colors are HDR.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public static MatrixTransformationProcessor create(
Context context, ImmutableList<GlMatrixTransformation> matrixTransformations, boolean useHdr)
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
boolean useHdr)
throws FrameProcessingException {
GlProgram glProgram =
createGlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_COPY_PATH);
createGlProgram(
context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH);
// No transfer functions needed, because input and output are both optical colors.
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr);
}
/**
@ -148,7 +164,9 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
@ -156,13 +174,16 @@ import java.util.Arrays;
public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
useHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
@ -184,7 +205,7 @@ import java.util.Arrays;
glProgram.setIntUniform("uEotfColorTransfer", colorTransfer);
}
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr);
}
/**
@ -199,7 +220,9 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
@ -207,13 +230,14 @@ import java.util.Arrays;
public static MatrixTransformationProcessor createApplyingOetf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH;
useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
@ -224,7 +248,7 @@ import java.util.Arrays;
glProgram.setIntUniform("uOetfColorTransfer", colorTransfer);
}
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr);
}
/**
@ -239,7 +263,9 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
@ -247,13 +273,16 @@ import java.util.Arrays;
public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotfThenOetf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
useHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
@ -273,7 +302,7 @@ import java.util.Arrays;
glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE);
}
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr);
}
/**
@ -281,17 +310,22 @@ import java.util.Arrays;
*
* @param glProgram The {@link GlProgram}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
* EXT_YUV_target} OpenGL extension.
*/
private MatrixTransformationProcessor(
GlProgram glProgram,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
boolean useHdr) {
super(useHdr);
this.glProgram = glProgram;
this.matrixTransformations = matrixTransformations;
this.rgbMatrices = rgbMatrices;
this.useHdr = useHdr;
transformationMatrixCache = new float[matrixTransformations.size()][16];
compositeTransformationMatrix = new float[16];
@ -333,11 +367,13 @@ import java.util.Arrays;
if (visiblePolygon.size() < 3) {
return; // Need at least three visible vertices for a triangle.
}
float[] compositeRgbMatrix =
createCompositeRgbaMatrixArray(rgbMatrices, useHdr, presentationTimeUs);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrix);
glProgram.setFloatsUniform("uRgbMatrix", compositeRgbMatrix);
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.createVertexBuffer(visiblePolygon),
@ -425,4 +461,31 @@ import java.util.Arrays;
}
return matrixChanged;
}
// TODO(b/239757183): Add caching for RgbMatrix and refactor RgbMatrix and MatrixTransformation
// composing.
private static float[] createCompositeRgbaMatrixArray(
ImmutableList<RgbMatrix> rgbMatrices, boolean useHdr, long presentationTimeUs) {
float[] tempResultMatrix = new float[16];
float[] compositeRgbaMatrix = new float[16];
Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0);
for (int i = 0; i < rgbMatrices.size(); i++) {
Matrix.multiplyMM(
/* result= */ tempResultMatrix,
/* resultOffset= */ 0,
/* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs, useHdr),
/* lhsOffset= */ 0,
/* rhs= */ compositeRgbaMatrix,
/* rhsOffset= */ 0);
System.arraycopy(
/* src= */ tempResultMatrix,
/* srcPos= */ 0,
/* dest= */ compositeRgbaMatrix,
/* destPost= */ 0,
/* length= */ tempResultMatrix.length);
}
return compositeRgbaMatrix;
}
}

9
libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java
View File

@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Provides common color filters. */
@ -91,9 +92,13 @@ public class RgbFilter implements RgbMatrix {
}
@Override
public RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr)
public MatrixTransformationProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException {
checkForConsistentHdrSetting(useHdr);
return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr);
return MatrixTransformationProcessor.create(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(this),
useHdr);
}
}

9
libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java
View File

@ -19,6 +19,7 @@ package androidx.media3.effect;
import android.content.Context;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
/**
* Specifies a 4x4 RGB color transformation matrix to apply to each frame in the fragment shader.
@ -39,8 +40,12 @@ public interface RgbMatrix extends GlEffect {
float[] getMatrix(long presentationTimeUs, boolean useHdr);
@Override
default RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr)
default MatrixTransformationProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException {
return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr);
return MatrixTransformationProcessor.create(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(this),
useHdr);
}
}

151
libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java
View File

@ -1,151 +0,0 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.effect;
import android.content.Context;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.util.Pair;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
/**
* Applies a sequence of {@link RgbMatrix} to each frame.
*
* <p>After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the
* color space. Intermediate results are not clamped.
*/
/* package */ final class RgbMatrixProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH =
"shaders/fragment_shader_transformation_es2.glsl";
private final GlProgram glProgram;
private final ImmutableList<RgbMatrix> rgbMatrices;
private final boolean useHdr;
// TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor.
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param rgbMatrix The {@link RgbMatrix} to apply to each frame.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public RgbMatrixProcessor(Context context, RgbMatrix rgbMatrix, boolean useHdr)
throws FrameProcessingException {
this(context, ImmutableList.of(rgbMatrix), useHdr);
}
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param rgbMatrices The {@link RgbMatrix} to apply to each frame.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public RgbMatrixProcessor(Context context, ImmutableList<RgbMatrix> rgbMatrices, boolean useHdr)
throws FrameProcessingException {
super(useHdr);
this.rgbMatrices = rgbMatrices;
this.useHdr = useHdr;
try {
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
float[] identityMatrix = new float[16];
Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix);
glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
}
@Override
public Pair<Integer, Integer> configure(int inputWidth, int inputHeight) {
return Pair.create(inputWidth, inputHeight);
}
private static float[] createCompositeRgbaMatrixArray(
ImmutableList<RgbMatrix> rgbMatrices, boolean useHdr, long presentationTimeUs) {
float[] tempResultMatrix = new float[16];
float[] compositeRgbaMatrix = new float[16];
Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0);
for (int i = 0; i < rgbMatrices.size(); i++) {
Matrix.multiplyMM(
/* result= */ tempResultMatrix,
/* resultOffset= */ 0,
/* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs, useHdr),
/* lhsOffset= */ 0,
/* rhs= */ compositeRgbaMatrix,
/* rhsOffset= */ 0);
System.arraycopy(
/* src= */ tempResultMatrix,
/* srcPos= */ 0,
/* dest= */ compositeRgbaMatrix,
/* destPost= */ 0,
/* length= */ tempResultMatrix.length);
}
return compositeRgbaMatrix;
}
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
// TODO(b/239431666): Add caching for compacting Matrices.
float[] rgbMatrixArray =
createCompositeRgbaMatrixArray(rgbMatrices, useHdr, presentationTimeUs);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uColorMatrix", rgbMatrixArray);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
}
}
@Override
public void release() throws FrameProcessingException {
super.release();
try {
glProgram.delete();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
}