loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor RgbaMatrix to RgbMatrix.

Browse Source 
* Rename all Rgba instances to Rgb.
* Remove alpha value from the RGBA Matrices and apply the 4x4 matrix
only to the R, G, B channels.
* Restore the alpha from the input unchanged.

PiperOrigin-RevId: 467208888
This commit is contained in:
leonwind 2022-08-12 14:51:25 +00:00 committed by Marc Baechinger
parent d7bf1ed2d7
commit 8760ee48b9
5 changed files with 56 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java
View File

@ -39,7 +39,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Pixel tests for {@link RgbaMatrix}.
* Pixel tests for {@link RgbMatrix}.
*
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link
@ -63,7 +63,7 @@ public final class RgbAdjustmentPixelTest {
private @MonotonicNonNull EGLDisplay eglDisplay;
private @MonotonicNonNull EGLContext eglContext;
private @MonotonicNonNull SingleFrameGlTextureProcessor rgbaMatrixProcessor;
private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor;
private @MonotonicNonNull EGLSurface placeholderEglSurface;
private int inputTexId;
private int outputTexId;
@ -95,27 +95,27 @@ public final class RgbAdjustmentPixelTest {
@After
public void release() throws GlUtil.GlException, FrameProcessingException {
if (rgbaMatrixProcessor != null) {
rgbaMatrixProcessor.release();
if (rgbMatrixProcessor != null) {
rgbMatrixProcessor.release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
private static RgbaMatrixProcessor createRgbaMatrixProcessor(Context context, float[] rgbaMatrix)
private static RgbMatrixProcessor createRgbMatrixProcessor(Context context, float[] rgbMatrix)
throws FrameProcessingException {
return ((RgbaMatrix) presentationTimeUs -> rgbaMatrix)
return ((RgbMatrix) presentationTimeUs -> rgbMatrix)
.toGlTextureProcessor(context, /* useHdr= */ false);
}
@Test
public void drawFrame_identityMatrix_leavesFrameUnchanged() throws Exception {
String testId = "drawFrame_identityMatrix";
RgbaMatrix identityMatrix = new RgbAdjustment.Builder().build();
rgbaMatrixProcessor = new RgbaMatrixProcessor(context, identityMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight);
RgbMatrix identityMatrix = new RgbAdjustment.Builder().build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, identityMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -131,15 +131,15 @@ public final class RgbAdjustmentPixelTest {
@Test
public void drawFrame_removeColors_producesBlackFrame() throws Exception {
String testId = "drawFrame_removeColors";
RgbaMatrix removeColorMatrix =
RgbMatrix removeColorMatrix =
new RgbAdjustment.Builder().setRedScale(0).setGreenScale(0).setBlueScale(0).build();
rgbaMatrixProcessor = new RgbaMatrixProcessor(context, removeColorMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight);
rgbMatrixProcessor = new RgbMatrixProcessor(context, removeColorMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap =
BitmapTestUtil.createArgb8888BitmapWithSolidColor(
outputSize.first, outputSize.second, Color.BLACK);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -155,12 +155,12 @@ public final class RgbAdjustmentPixelTest {
@Test
public void drawFrame_redOnlyFilter_setsBlueAndGreenValuesToZero() throws Exception {
String testId = "drawFrame_redOnlyFilter";
RgbaMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build();
rgbaMatrixProcessor = new RgbaMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight);
RgbMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -176,12 +176,12 @@ public final class RgbAdjustmentPixelTest {
@Test
public void drawFrame_increaseRedChannel_producesBrighterAndRedderFrame() throws Exception {
String testId = "drawFrame_increaseRedChannel";
RgbaMatrix increaseRedMatrix = new RgbAdjustment.Builder().setRedScale(5).build();
rgbaMatrixProcessor = new RgbaMatrixProcessor(context, increaseRedMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight);
RgbMatrix increaseRedMatrix = new RgbAdjustment.Builder().setRedScale(5).build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, increaseRedMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_RED_CHANNEL_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -197,14 +197,14 @@ public final class RgbAdjustmentPixelTest {
@Test
public void drawFrame_increaseBrightness_increasesAllValues() throws Exception {
String testId = "drawFrame_increaseBrightness";
RgbaMatrix increaseBrightnessMatrix =
RgbMatrix increaseBrightnessMatrix =
new RgbAdjustment.Builder().setRedScale(5).setGreenScale(5).setBlueScale(5).build();
rgbaMatrixProcessor =
new RgbaMatrixProcessor(context, increaseBrightnessMatrix, /* useHdr = */ false);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight);
rgbMatrixProcessor =
new RgbMatrixProcessor(context, increaseBrightnessMatrix, /* useHdr = */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
@ -227,11 +227,11 @@ public final class RgbAdjustmentPixelTest {
0.2126f, 0.2126f, 0.2126f, 0, 0.7152f, 0.7152f, 0.7152f, 0, 0.0722f, 0.0722f, 0.0722f, 0, 0,
0, 0, 1
};
rgbaMatrixProcessor = createRgbaMatrixProcessor(/* context= */ context, grayscaleMatrix);
Pair<Integer, Integer> outputSize = rgbaMatrixProcessor.configure(inputWidth, inputHeight);
rgbMatrixProcessor = createRgbMatrixProcessor(/* context= */ context, grayscaleMatrix);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH);
rgbaMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);

5
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl
View File

@ -15,7 +15,7 @@
// ES 2 fragment shader that samples from a (non-external) texture with
// uTexSampler, copying from this texture to the current output while
// applying a 4x4 RGBA color matrix to change the pixel colors.
// applying a 4x4 RGB color matrix to change the pixel colors.
precision mediump float;
uniform sampler2D uTexSampler;
@ -24,5 +24,6 @@ varying vec2 vTexSamplingCoord;
void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
gl_FragColor = uColorMatrix * inputColor;
gl_FragColor = uColorMatrix * vec4(inputColor.rgb, 1);
gl_FragColor.a = inputColor.a;
}

22
libraries/effect/src/main/java/androidx/media3/effect/RgbAdjustment.java
View File

@ -24,7 +24,7 @@ import com.google.errorprone.annotations.CanIgnoreReturnValue;
/** Scales the red, green, and blue color channels of a frame. */
@UnstableApi
public final class RgbAdjustment implements RgbaMatrix {
public final class RgbAdjustment implements RgbMatrix {
/** A builder for {@link RgbAdjustment} instances. */
public static final class Builder {
@ -80,27 +80,23 @@ public final class RgbAdjustment implements RgbaMatrix {
/** Creates a new {@link RgbAdjustment} instance. */
public RgbAdjustment build() {
float[] rgbaMatrix = new float[16];
Matrix.setIdentityM(rgbaMatrix, /* smOffset= */ 0);
float[] rgbMatrix = new float[16];
Matrix.setIdentityM(rgbMatrix, /* smOffset= */ 0);
Matrix.scaleM(
rgbaMatrix,
/* smOffset= */ 0,
/* x= */ redScale,
/* y= */ greenScale,
/* z= */ blueScale);
rgbMatrix, /* smOffset= */ 0, /* x= */ redScale, /* y= */ greenScale, /* z= */ blueScale);
return new RgbAdjustment(rgbaMatrix);
return new RgbAdjustment(rgbMatrix);
}
}
private final float[] rgbaMatrix;
private final float[] rgbMatrix;
private RgbAdjustment(float[] rgbaMatrix) {
this.rgbaMatrix = rgbaMatrix;
private RgbAdjustment(float[] rgbMatrix) {
this.rgbMatrix = rgbMatrix;
}
@Override
public float[] getMatrix(long presentationTimeUs) {
return rgbaMatrix;
return rgbMatrix;
}
}

10
libraries/effect/src/main/java/androidx/media3/effect/RgbaMatrix.java → libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java
View File

@ -21,20 +21,20 @@ import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.UnstableApi;
/**
* Specifies a 4x4 RGBA color transformation matrix to apply to each frame in the fragment shader.
* Specifies a 4x4 RGB color transformation matrix to apply to each frame in the fragment shader.
*/
@UnstableApi
public interface RgbaMatrix extends GlEffect {
public interface RgbMatrix extends GlEffect {
/**
* Returns the 4x4 RGBA transformation {@linkplain android.opengl.Matrix matrix} to apply to the
* Returns the 4x4 RGB transformation {@linkplain android.opengl.Matrix matrix} to apply to the
* color values of each pixel in the frame with the given timestamp.
*/
float[] getMatrix(long presentationTimeUs);
@Override
default RgbaMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr)
default RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException {
return new RgbaMatrixProcessor(context, /* rgbaMatrix= */ this, useHdr);
return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr);
}
}

18
libraries/effect/src/main/java/androidx/media3/effect/RgbaMatrixProcessor.java → libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java
View File

@ -25,21 +25,21 @@ import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
/** Applies an {@link RgbaMatrix} to each frame. */
/* package */ final class RgbaMatrixProcessor extends SingleFrameGlTextureProcessor {
/** Applies an {@link RgbMatrix} to each frame. */
/* package */ final class RgbMatrixProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH =
"shaders/fragment_shader_transformation_es2.glsl";
private final GlProgram glProgram;
private final RgbaMatrix rgbaMatrix;
private final RgbMatrix rgbMatrix;
// TODO(b/239431666): Support chaining multiple RgbaMatrix instances in RgbaMatrixProcessor.
// TODO(b/239757183): Merge RgbaMatrixProcessor with MatrixTransformationProcessor.
public RgbaMatrixProcessor(Context context, RgbaMatrix rgbaMatrix, boolean useHdr)
// TODO(b/239431666): Support chaining multiple RgbMatrix instances in RgbMatrixProcessor.
// TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor.
public RgbMatrixProcessor(Context context, RgbMatrix rgbMatrix, boolean useHdr)
throws FrameProcessingException {
super(useHdr);
this.rgbaMatrix = rgbaMatrix;
this.rgbMatrix = rgbMatrix;
try {
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
@ -66,11 +66,11 @@ import java.io.IOException;
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
float[] rgbaMatrixArray = rgbaMatrix.getMatrix(presentationTimeUs);
float[] rgbMatrixArray = rgbMatrix.getMatrix(presentationTimeUs);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uColorMatrix", rgbaMatrixArray);
glProgram.setFloatsUniform("uColorMatrix", rgbMatrixArray);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.