loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor RgbaMatrix to RgbMatrix.

Browse Source 
* Rename all Rgba instances to Rgb.
* Remove alpha value from the RGBA Matrices and apply the 4x4 matrix
only to the R, G, B channels.
* Restore the alpha from the input unchanged.

PiperOrigin-RevId: 467208888
This commit is contained in:
leonwind 2022-08-12 14:51:25 +00:00 committed by Marc Baechinger
parent d7bf1ed2d7
commit 8760ee48b9
5 changed files with 56 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java
View File

@ -39,7 +39,7 @@ import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
/** /**
* Pixel tests for {@link RgbaMatrix}. * Pixel tests for {@link RgbMatrix}.
* *
* <p>Expected images are taken from an emulator, so tests on different emulators or physical * <p>Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link * devices may fail. To test on other devices, please increase the {@link
@ -63,7 +63,7 @@ public final class RgbAdjustmentPixelTest {
private @MonotonicNonNull EGLDisplay eglDisplay; private @MonotonicNonNull EGLDisplay eglDisplay;
private @MonotonicNonNull EGLContext eglContext; private @MonotonicNonNull EGLContext eglContext;
private @MonotonicNonNull SingleFrameGlTextureProcessor rgbaMatrixProcessor; private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor;
private @MonotonicNonNull EGLSurface placeholderEglSurface; private @MonotonicNonNull EGLSurface placeholderEglSurface;
private int inputTexId; private int inputTexId;
private int outputTexId; private int outputTexId;
@ -95,27 +95,27 @@ public final class RgbAdjustmentPixelTest {
@After @After
public void release() throws GlUtil.GlException, FrameProcessingException { public void release() throws GlUtil.GlException, FrameProcessingException {
if (rgbaMatrixProcessor != null) { if (rgbMatrixProcessor != null) {
rgbaMatrixProcessor.release(); rgbMatrixProcessor.release();
} }
GlUtil.destroyEglContext(eglDisplay, eglContext); GlUtil.destroyEglContext(eglDisplay, eglContext);
} }
private static RgbaMatrixProcessor createRgbaMatrixProcessor(Context context, float[] rgbaMatrix) private static RgbMatrixProcessor createRgbMatrixProcessor(Context context, float[] rgbMatrix)
throws FrameProcessingException { throws FrameProcessingException {
return ((RgbaMatrix) presentationTimeUs -> rgbaMatrix) return ((RgbMatrix) presentationTimeUs -> rgbMatrix)
.toGlTextureProcessor(context, /* useHdr= */ false); .toGlTextureProcessor(context, /* useHdr= */ false);
} }
@Test @Test
public void drawFrame_identityMatrix_leavesFrameUnchanged() throws Exception { public void drawFrame_identityMatrix_leavesFrameUnchanged() throws Exception {
String testId = "drawFrame_identityMatrix"; String testId = "drawFrame_identityMatrix";
RgbaMatrix identityMatrix = new RgbAdjustment.Builder().build(); RgbMatrix identityMatrix = new RgbAdjustment.Builder().build();
rgbaMatrixProcessor = new RgbaMatrixProcessor(context, identityMatrix, /* useHdr= */ false); rgbMatrixProcessor = new RgbMatrixProcessor(context, identityMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second); outputSize.first, outputSize.second);
@ -131,15 +131,15 @@ public final class RgbAdjustmentPixelTest {
@Test @Test
public void drawFrame_removeColors_producesBlackFrame() throws Exception { public void drawFrame_removeColors_producesBlackFrame() throws Exception {
String testId = "drawFrame_removeColors"; String testId = "drawFrame_removeColors";
RgbaMatrix removeColorMatrix = RgbMatrix removeColorMatrix =
new RgbAdjustment.Builder().setRedScale(0).setGreenScale(0).setBlueScale(0).build(); new RgbAdjustment.Builder().setRedScale(0).setGreenScale(0).setBlueScale(0).build();
rgbaMatrixProcessor = new RgbaMatrixProcessor(context, removeColorMatrix, /* useHdr= */ false); rgbMatrixProcessor = new RgbMatrixProcessor(context, removeColorMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = Bitmap expectedBitmap =
BitmapTestUtil.createArgb8888BitmapWithSolidColor( BitmapTestUtil.createArgb8888BitmapWithSolidColor(
outputSize.first, outputSize.second, Color.BLACK); outputSize.first, outputSize.second, Color.BLACK);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second); outputSize.first, outputSize.second);
@ -155,12 +155,12 @@ public final class RgbAdjustmentPixelTest {
@Test @Test
public void drawFrame_redOnlyFilter_setsBlueAndGreenValuesToZero() throws Exception { public void drawFrame_redOnlyFilter_setsBlueAndGreenValuesToZero() throws Exception {
String testId = "drawFrame_redOnlyFilter"; String testId = "drawFrame_redOnlyFilter";
RgbaMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build(); RgbMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build();
rgbaMatrixProcessor = new RgbaMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false); rgbMatrixProcessor = new RgbMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second); outputSize.first, outputSize.second);
@ -176,12 +176,12 @@ public final class RgbAdjustmentPixelTest {
@Test @Test
public void drawFrame_increaseRedChannel_producesBrighterAndRedderFrame() throws Exception { public void drawFrame_increaseRedChannel_producesBrighterAndRedderFrame() throws Exception {
String testId = "drawFrame_increaseRedChannel"; String testId = "drawFrame_increaseRedChannel";
RgbaMatrix increaseRedMatrix = new RgbAdjustment.Builder().setRedScale(5).build(); RgbMatrix increaseRedMatrix = new RgbAdjustment.Builder().setRedScale(5).build();
rgbaMatrixProcessor = new RgbaMatrixProcessor(context, increaseRedMatrix, /* useHdr= */ false); rgbMatrixProcessor = new RgbMatrixProcessor(context, increaseRedMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_RED_CHANNEL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_RED_CHANNEL_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second); outputSize.first, outputSize.second);
@ -197,14 +197,14 @@ public final class RgbAdjustmentPixelTest {
@Test @Test
public void drawFrame_increaseBrightness_increasesAllValues() throws Exception { public void drawFrame_increaseBrightness_increasesAllValues() throws Exception {
String testId = "drawFrame_increaseBrightness"; String testId = "drawFrame_increaseBrightness";
RgbaMatrix increaseBrightnessMatrix = RgbMatrix increaseBrightnessMatrix =
new RgbAdjustment.Builder().setRedScale(5).setGreenScale(5).setBlueScale(5).build(); new RgbAdjustment.Builder().setRedScale(5).setGreenScale(5).setBlueScale(5).build();
rgbaMatrixProcessor = rgbMatrixProcessor =
new RgbaMatrixProcessor(context, increaseBrightnessMatrix, /* useHdr = */ false); new RgbMatrixProcessor(context, increaseBrightnessMatrix, /* useHdr = */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second); outputSize.first, outputSize.second);
@ -227,11 +227,11 @@ public final class RgbAdjustmentPixelTest {
0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0, 0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0,
0, 0, 1 0, 0, 1
}; };
rgbaMatrixProcessor = createRgbaMatrixProcessor(/* context= */ context, grayscaleMatrix); rgbMatrixProcessor = createRgbMatrixProcessor(/* context= */ context, grayscaleMatrix);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight); Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second); outputSize.first, outputSize.second);

5
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl
View File

@ -15,7 +15,7 @@
// ES 2 fragment shader that samples from a (non-external) texture with // ES 2 fragment shader that samples from a (non-external) texture with
// uTexSampler, copying from this texture to the current output while // uTexSampler, copying from this texture to the current output while
// applying a 4x4 RGBA color matrix to change the pixel colors. // applying a 4x4 RGB color matrix to change the pixel colors.
precision mediump float; precision mediump float;
uniform sampler2D uTexSampler; uniform sampler2D uTexSampler;
@ -24,5 +24,6 @@ varying vec2 vTexSamplingCoord;
void main() { void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
gl_FragColor = uColorMatrix * inputColor; gl_FragColor = uColorMatrix * vec4(inputColor.rgb, 1);
gl_FragColor.a = inputColor.a;
} }

22
libraries/effect/src/main/java/androidx/media3/effect/RgbAdjustment.java
View File

@ -24,7 +24,7 @@ import com.google.errorprone.annotations.CanIgnoreReturnValue;
/** Scales the red, green, and blue color channels of a frame. */ /** Scales the red, green, and blue color channels of a frame. */
@UnstableApi @UnstableApi
public final class RgbAdjustment implements RgbaMatrix { public final class RgbAdjustment implements RgbMatrix {
/** A builder for {@link RgbAdjustment} instances. */ /** A builder for {@link RgbAdjustment} instances. */
public static final class Builder { public static final class Builder {
@ -80,27 +80,23 @@ public final class RgbAdjustment implements RgbaMatrix {
/** Creates a new {@link RgbAdjustment} instance. */ /** Creates a new {@link RgbAdjustment} instance. */
public RgbAdjustment build() { public RgbAdjustment build() {
float[] rgbaMatrix = new float[16]; float[] rgbMatrix = new float[16];
Matrix.setIdentityM(rgbaMatrix, /* smOffset= */ 0); Matrix.setIdentityM(rgbMatrix, /* smOffset= */ 0);
Matrix.scaleM( Matrix.scaleM(
rgbaMatrix, rgbMatrix, /* smOffset= */ 0, /* x= */ redScale, /* y= */ greenScale, /* z= */ blueScale);
/* smOffset= */ 0,
/* x= */ redScale,
/* y= */ greenScale,
/* z= */ blueScale);
return new RgbAdjustment(rgbaMatrix); return new RgbAdjustment(rgbMatrix);
} }
} }
private final float[] rgbaMatrix; private final float[] rgbMatrix;
private RgbAdjustment(float[] rgbaMatrix) { private RgbAdjustment(float[] rgbMatrix) {
this.rgbaMatrix = rgbaMatrix; this.rgbMatrix = rgbMatrix;
} }
@Override @Override
public float[] getMatrix(long presentationTimeUs) { public float[] getMatrix(long presentationTimeUs) {
return rgbaMatrix; return rgbMatrix;
} }
} }

10
libraries/effect/src/main/java/androidx/media3/effect/RgbaMatrix.java → libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java
View File

@ -21,20 +21,20 @@ import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
/** /**
* Specifies a 4x4 RGBA color transformation matrix to apply to each frame in the fragment shader. * Specifies a 4x4 RGB color transformation matrix to apply to each frame in the fragment shader.
*/ */
@UnstableApi @UnstableApi
public interface RgbaMatrix extends GlEffect { public interface RgbMatrix extends GlEffect {
/** /**
* Returns the 4x4 RGBA transformation {@linkplain android.opengl.Matrix matrix} to apply to the * Returns the 4x4 RGB transformation {@linkplain android.opengl.Matrix matrix} to apply to the
* color values of each pixel in the frame with the given timestamp. * color values of each pixel in the frame with the given timestamp.
*/ */
float[] getMatrix(long presentationTimeUs); float[] getMatrix(long presentationTimeUs);
@Override @Override
default RgbaMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr) default RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
return new RgbaMatrixProcessor(context, /* rgbaMatrix= */ this, useHdr); return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr);
} }
} }

18
libraries/effect/src/main/java/androidx/media3/effect/RgbaMatrixProcessor.java → libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java
View File

@ -25,21 +25,21 @@ import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import java.io.IOException; import java.io.IOException;
/** Applies an {@link RgbaMatrix} to each frame. */ /** Applies an {@link RgbMatrix} to each frame. */
/* package */ final class RgbaMatrixProcessor extends SingleFrameGlTextureProcessor { /* package */ final class RgbMatrixProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = private static final String FRAGMENT_SHADER_PATH =
"shaders/fragment_shader_transformation_es2.glsl"; "shaders/fragment_shader_transformation_es2.glsl";
private final GlProgram glProgram; private final GlProgram glProgram;
private final RgbaMatrix rgbaMatrix; private final RgbMatrix rgbMatrix;
// TODO(b/239431666): Support chaining multiple RgbaMatrix instances in RgbaMatrixProcessor. // TODO(b/239431666): Support chaining multiple RgbMatrix instances in RgbMatrixProcessor.
// TODO(b/239757183): Merge RgbaMatrixProcessor with MatrixTransformationProcessor. // TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor.
public RgbaMatrixProcessor(Context context, RgbaMatrix rgbaMatrix, boolean useHdr) public RgbMatrixProcessor(Context context, RgbMatrix rgbMatrix, boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
super(useHdr); super(useHdr);
this.rgbaMatrix = rgbaMatrix; this.rgbMatrix = rgbMatrix;
try { try {
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
@ -66,11 +66,11 @@ import java.io.IOException;
@Override @Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
float[] rgbaMatrixArray = rgbaMatrix.getMatrix(presentationTimeUs); float[] rgbMatrixArray = rgbMatrix.getMatrix(presentationTimeUs);
try { try {
glProgram.use(); glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uColorMatrix", rgbaMatrixArray); glProgram.setFloatsUniform("uColorMatrix", rgbMatrixArray);
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad. // The four-vertex triangle strip forms a quad.