Expected images are taken from an emulator, so tests on different emulators or physical + * devices may fail. To test on other devices, please increase the {@link + * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps + * as recommended in {@link GlEffectsFrameProcessorPixelTest}. + */ +@RunWith(AndroidJUnit4.class) +public class ContrastProcessorPixelTest { + private static final String EXOPLAYER_LOGO_PNG_ASSET_PATH = + "media/bitmap/exoplayer_logo/original.png"; + // TODO(b/239005261): Migrate png to an emulator generated picture. + private static final String MAXIMUM_CONTRAST_PNG_ASSET_PATH = + "media/bitmap/exoplayer_logo/maximum_contrast.png"; + + // OpenGL uses floats in [0, 1] and maps 0.5f to 128 = 256 / 2. + private static final int OPENGL_NEUTRAL_RGB_VALUE = 128; + + private final Context context = getApplicationContext(); + + private @MonotonicNonNull EGLDisplay eglDisplay; + private @MonotonicNonNull EGLContext eglContext; + private @MonotonicNonNull EGLSurface placeholderEglSurface; + private @MonotonicNonNull SingleFrameGlTextureProcessor contrastProcessor; + private int inputTexId; + private int outputTexId; + private int inputWidth; + private int inputHeight; + + @Before + public void createGlObjects() throws Exception { + eglDisplay = GlUtil.createEglDisplay(); + eglContext = GlUtil.createEglContext(eglDisplay); + + Bitmap inputBitmap = BitmapTestUtil.readBitmap(EXOPLAYER_LOGO_PNG_ASSET_PATH); + inputWidth = inputBitmap.getWidth(); + inputHeight = inputBitmap.getHeight(); + + placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay); + GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, inputWidth, inputHeight); + inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap); + } + + @After + public void release() throws GlUtil.GlException, FrameProcessingException { + if (contrastProcessor != null) { + contrastProcessor.release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + } + + @Test + public void drawFrame_noContrastChange_leavesFrameUnchanged() throws Exception { + String testId = "drawFrame_noContrastChange"; + contrastProcessor = + new Contrast(/* contrast= */ 0.0f).toGlTextureProcessor(context, /* useHdr= */ false); + Size outputSize = contrastProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(EXOPLAYER_LOGO_PNG_ASSET_PATH); + + contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.getWidth(), outputSize.getHeight()); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_minimumContrast_producesAllGrayFrame() throws Exception { + String testId = "drawFrame_minimumContrast"; + contrastProcessor = + new Contrast(/* contrast= */ -1.0f).toGlTextureProcessor(context, /* useHdr= */ false); + Size outputSize = contrastProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = + BitmapTestUtil.createArgb8888BitmapWithSolidColor( + inputWidth, + inputHeight, + Color.rgb( + OPENGL_NEUTRAL_RGB_VALUE, OPENGL_NEUTRAL_RGB_VALUE, OPENGL_NEUTRAL_RGB_VALUE)); + + contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.getWidth(), outputSize.getHeight()); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_decreaseContrast_decreasesPixelsGreaterEqual128IncreasesBelow() + throws Exception { + String testId = "drawFrame_decreaseContrast"; + contrastProcessor = + new Contrast(/* contrast= */ -0.75f).toGlTextureProcessor(context, /* useHdr= */ false); + Size outputSize = contrastProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap originalBitmap = BitmapTestUtil.readBitmap(EXOPLAYER_LOGO_PNG_ASSET_PATH); + + contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.getWidth(), outputSize.getHeight()); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + assertIncreasedOrDecreasedContrast(originalBitmap, actualBitmap, /* increased= */ false); + } + + @Test + public void drawFrame_increaseContrast_increasesPixelsGreaterEqual128DecreasesBelow() + throws Exception { + String testId = "drawFrame_increaseContrast"; + contrastProcessor = + new Contrast(/* contrast= */ 0.75f).toGlTextureProcessor(context, /* useHdr= */ false); + Size outputSize = contrastProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap originalBitmap = BitmapTestUtil.readBitmap(EXOPLAYER_LOGO_PNG_ASSET_PATH); + + contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.getWidth(), outputSize.getHeight()); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + assertIncreasedOrDecreasedContrast(originalBitmap, actualBitmap, /* increased= */ true); + } + + @Test + public void drawFrame_maximumContrast_pixelEither0or255() throws Exception { + String testId = "drawFrame_maximumContrast"; + contrastProcessor = + new Contrast(/* contrast= */ 1.0f).toGlTextureProcessor(context, /* useHdr= */ false); + Size outputSize = contrastProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(MAXIMUM_CONTRAST_PNG_ASSET_PATH); + + contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.getWidth(), outputSize.getHeight()); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + private static void assertIncreasedOrDecreasedContrast( + Bitmap originalBitmap, Bitmap actualBitmap, boolean increased) { + + for (int y = 0; y < actualBitmap.getHeight(); y++) { + for (int x = 0; x < actualBitmap.getWidth(); x++) { + int originalColor = originalBitmap.getPixel(x, y); + int actualColor = actualBitmap.getPixel(x, y); + + int redDifference = Color.red(actualColor) - Color.red(originalColor); + int greenDifference = Color.green(actualColor) - Color.green(originalColor); + int blueDifference = Color.blue(actualColor) - Color.blue(originalColor); + + // If the contrast increases, all pixels with a value greater or equal to + // OPENGL_NEUTRAL_RGB_VALUE must increase (diff is greater or equal to 0) and all pixels + // below OPENGL_NEUTRAL_RGB_VALUE must decrease (diff is smaller or equal to 0). + // If the contrast decreases, all pixels with a value greater or equal to + // OPENGL_NEUTRAL_RGB_VALUE must decrease (diff is smaller or equal to 0) and all pixels + // below OPENGL_NEUTRAL_RGB_VALUE must increase (diff is greater or equal to 0). + // The interval limits 0 and 255 stay unchanged for either contrast in- or decrease. + + if (Color.red(originalColor) >= OPENGL_NEUTRAL_RGB_VALUE) { + assertThat(increased ? redDifference : -redDifference).isAtLeast(0); + } else { + assertThat(increased ? redDifference : -redDifference).isAtMost(0); + } + + if (Color.green(originalColor) >= OPENGL_NEUTRAL_RGB_VALUE) { + assertThat(increased ? greenDifference : -greenDifference).isAtLeast(0); + } else { + assertThat(increased ? greenDifference : -greenDifference).isAtMost(0); + } + + if (Color.blue(originalColor) >= OPENGL_NEUTRAL_RGB_VALUE) { + assertThat(increased ? blueDifference : -blueDifference).isAtLeast(0); + } else { + assertThat(increased ? blueDifference : -blueDifference).isAtMost(0); + } + } + } + } + + private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { + outputTexId = + GlUtil.createTexture( + outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false); + int frameBuffer = GlUtil.createFboForTexture(outputTexId); + GlUtil.focusFramebuffer( + checkNotNull(eglDisplay), + checkNotNull(eglContext), + checkNotNull(placeholderEglSurface), + frameBuffer, + outputWidth, + outputHeight); + } +} diff --git a/libraries/transformer/src/main/assets/shaders/fragment_shader_contrast_es2.glsl b/libraries/transformer/src/main/assets/shaders/fragment_shader_contrast_es2.glsl new file mode 100644 index 0000000000..6420451d85 --- /dev/null +++ b/libraries/transformer/src/main/assets/shaders/fragment_shader_contrast_es2.glsl @@ -0,0 +1,33 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 fragment shader that samples from a (non-external) texture with +// uTexSampler, copying from this texture to the current output +// while adjusting contrast based on uContrastFactor. + +precision mediump float; +uniform sampler2D uTexSampler; +uniform float uContrastFactor; +varying vec2 vTexSamplingCoord; + +void main() { + vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); + + gl_FragColor = vec4( + clamp(uContrastFactor * (inputColor.r - 0.5) + 0.5, 0.0, 1.0), + clamp(uContrastFactor * (inputColor.g - 0.5) + 0.5, 0.0, 1.0), + clamp(uContrastFactor * (inputColor.b - 0.5) + 0.5, 0.0, 1.0), + inputColor.a); +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Contrast.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Contrast.java new file mode 100644 index 0000000000..06db7a3a3a --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Contrast.java @@ -0,0 +1,47 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkArgument; + +import android.content.Context; +import androidx.media3.common.util.UnstableApi; + +/** A {@link GlEffect} to control the contrast of video frames. */ +@UnstableApi +public class Contrast implements GlEffect { + + /** Adjusts the contrast of video frames in the interval [-1, 1]. */ + public final float contrast; + + /** + * Creates a new instance for the given contrast value. + * + *
Contrast values range from -1 (all gray pixels) to 1 (maximum difference of colors). 0 means + * to add no contrast and leaves the frames unchanged. + */ + public Contrast(float contrast) { + checkArgument(-1 <= contrast && contrast <= 1, "Contrast needs to be in the interval [-1, 1]."); + this.contrast = contrast; + } + + @Override + public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr) + throws FrameProcessingException { + return new ContrastProcessor(context, this, useHdr); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ContrastProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ContrastProcessor.java new file mode 100644 index 0000000000..44c9b89cac --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ContrastProcessor.java @@ -0,0 +1,78 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import android.content.Context; +import android.opengl.GLES20; +import android.opengl.Matrix; +import android.util.Size; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import java.io.IOException; + +/** Contrast processor to apply a {@link Contrast} to each frame. */ +/* package */ final class ContrastProcessor extends SingleFrameGlTextureProcessor { + private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_contrast_es2.glsl"; + + private final GlProgram glProgram; + private final float contrastFactor; + + public ContrastProcessor(Context context, Contrast contrastEffect, boolean useHdr) + throws FrameProcessingException { + super(useHdr); + // Use 1.0001f to avoid division by zero issues. + contrastFactor = (1 + contrastEffect.contrast) / (1.0001f - contrastEffect.contrast); + + try { + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + } catch (IOException | GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + + float[] identityMatrix = new float[16]; + Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0); + glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix); + glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + } + + @Override + public Size configure(int inputWidth, int inputHeight) { + return new Size(inputWidth, inputHeight); + } + + @Override + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { + try { + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + glProgram.setFloatUniform("uContrastFactor", contrastFactor); + glProgram.bindAttributesAndUniforms(); + + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e, presentationTimeUs); + } + } +}