diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/SingleColorLutPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/SingleColorLutPixelTest.java new file mode 100644 index 0000000000..a459ade723 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/SingleColorLutPixelTest.java @@ -0,0 +1,325 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.effect.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Color; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLSurface; +import android.util.Pair; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.util.GlUtil; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Pixel test for Lookup Tables via {@link ColorLutProcessor}. + * + *

Expected images are taken from an emulator, so tests on different emulators or physical + * devices may fail. To test on other devices, please increase the {@link + * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps + * as recommended in {@link GlEffectsFrameProcessorPixelTest}. + */ +@RunWith(AndroidJUnit4.class) +public class SingleColorLutPixelTest { + public static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/original.png"; + public static final String LUT_MAP_WHITE_TO_GREEN_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/lut_map_white_to_green.png"; + public static final String GRAYSCALE_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/grayscale.png"; + public static final String INVERT_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/invert.png"; + public static final String VERTICAL_HALD_IDENTITY_LUT = "media/bitmap/lut/identity.png"; + public static final String VERTICAL_HALD_GRAYSCALE_LUT = "media/bitmap/lut/grayscale.png"; + public static final String VERTICAL_HALD_INVERTED_LUT = "media/bitmap/lut/inverted.png"; + + private final Context context = getApplicationContext(); + + private @MonotonicNonNull EGLDisplay eglDisplay; + private @MonotonicNonNull EGLContext eglContext; + private @MonotonicNonNull EGLSurface placeholderEglSurface; + private @MonotonicNonNull SingleFrameGlTextureProcessor colorLutProcessor; + private int inputTexId; + private int outputTexId; + private int inputWidth; + private int inputHeight; + + @Before + public void createGlObjects() throws Exception { + eglDisplay = GlUtil.createEglDisplay(); + eglContext = GlUtil.createEglContext(eglDisplay); + + Bitmap inputBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + inputWidth = inputBitmap.getWidth(); + inputHeight = inputBitmap.getHeight(); + + placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay); + GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, inputWidth, inputHeight); + inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap); + } + + @After + public void release() throws GlUtil.GlException, FrameProcessingException { + if (colorLutProcessor != null) { + colorLutProcessor.release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + } + + @Test + public void drawFrame_identityCubeLutSize2_leavesFrameUnchanged() throws Exception { + String testId = "drawFrame_identityLutCubeSize2"; + int[][][] cubeIdentityLut = createIdentityLutCube(/* length= */ 2); + colorLutProcessor = + SingleColorLut.createFromCube(cubeIdentityLut) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = colorLutProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + + colorLutProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_identityCubeLutSize64_leavesFrameUnchanged() throws Exception { + String testId = "drawFrame_identityLutCubeSize64"; + int[][][] cubeIdentityLut = createIdentityLutCube(/* length= */ 64); + colorLutProcessor = + SingleColorLut.createFromCube(cubeIdentityLut) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = colorLutProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + + colorLutProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_identityBitmapLutSize2_leavesFrameUnchanged() throws Exception { + String testId = "drawFrame_identityBitmapLutSize2"; + Bitmap bitmapLut = createIdentityLutBitmap(/* length= */ 2); + colorLutProcessor = + SingleColorLut.createFromBitmap(bitmapLut) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = colorLutProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + + colorLutProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_identityBitmapLutSize64_leavesFrameUnchanged() throws Exception { + String testId = "drawFrame_identityBitmapLutSize64"; + Bitmap bitmapLut = createIdentityLutBitmap(/* length= */ 64); + colorLutProcessor = + SingleColorLut.createFromBitmap(bitmapLut) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = colorLutProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + + colorLutProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_identityLutFromHaldImage_leavesFrameUnchanged() throws Exception { + String testId = "drawFrame_identityLutFromHaldImage"; + Bitmap bitmapLut = BitmapTestUtil.readBitmap(VERTICAL_HALD_IDENTITY_LUT); + colorLutProcessor = + SingleColorLut.createFromBitmap(bitmapLut) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = colorLutProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + + colorLutProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_mapWhiteToGreen_producesGreenHighlights() throws Exception { + String testId = "drawFrame_mapWhiteToGreen"; + int length = 3; + int[][][] mapWhiteToGreen = createIdentityLutCube(length); + mapWhiteToGreen[length - 1][length - 1][length - 1] = Color.GREEN; + colorLutProcessor = + SingleColorLut.createFromCube(mapWhiteToGreen) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = colorLutProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(LUT_MAP_WHITE_TO_GREEN_ASSET_PATH); + + colorLutProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_applyInvertedLut_producesInvertedFrame() throws Exception { + String testId = "drawFrame_applyInvertedLut"; + Bitmap invertedLutBitmap = BitmapTestUtil.readBitmap(VERTICAL_HALD_INVERTED_LUT); + colorLutProcessor = + SingleColorLut.createFromBitmap(invertedLutBitmap) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = colorLutProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INVERT_PNG_ASSET_PATH); + + colorLutProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_applyGrayscaleLut_producesGrayscaleFrame() throws Exception { + String testId = "drawFrame_applyGrayscaleLut"; + Bitmap grayscaleLutBitmap = BitmapTestUtil.readBitmap(VERTICAL_HALD_GRAYSCALE_LUT); + colorLutProcessor = + SingleColorLut.createFromBitmap(grayscaleLutBitmap) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = colorLutProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH); + + colorLutProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer( + outputSize.first, outputSize.second); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap); + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { + outputTexId = + GlUtil.createTexture( + outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false); + int frameBuffer = GlUtil.createFboForTexture(outputTexId); + GlUtil.focusFramebuffer( + checkNotNull(eglDisplay), + checkNotNull(eglContext), + checkNotNull(placeholderEglSurface), + frameBuffer, + outputWidth, + outputHeight); + } + + private static int[][][] createIdentityLutCube(int length) { + int[][][] lut = new int[length][length][length]; + float scale = 1f / (length - 1); + for (int r = 0; r < length; r++) { + for (int g = 0; g < length; g++) { + for (int b = 0; b < length; b++) { + lut[r][g][b] = + Color.rgb(/* red= */ r * scale, /* green= */ g * scale, /* blue= */ b * scale); + } + } + } + return lut; + } + + private static Bitmap createIdentityLutBitmap(int length) { + int[][][] lutCube = createIdentityLutCube(length); + int[] colors = new int[length * length * length]; + + for (int r = 0; r < length; r++) { + for (int g = 0; g < length; g++) { + for (int b = 0; b < length; b++) { + int color = lutCube[r][g][b]; + int planePosition = b + length * (g + length * r); + colors[planePosition] = color; + } + } + } + return Bitmap.createBitmap( + colors, /* width= */ length, /* height= */ length * length, Bitmap.Config.ARGB_8888); + } +} diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_lut_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_lut_es2.glsl new file mode 100644 index 0000000000..7200e43e7b --- /dev/null +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_lut_es2.glsl @@ -0,0 +1,99 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES2 fragment shader that samples from a (non-external) texture with +// uTexSampler, copying from this texture to the current output while +// applying a 3D color lookup table to change the pixel colors. + +precision highp float; +uniform sampler2D uTexSampler; +// The uColorLut texture is a N x N^2 2D texture where each z-plane of the 3D +// LUT is vertically stacked on top of each other. The red channel of the input +// color (z-axis in LUT[R][G][B] = LUT[z][y][x]) points to the plane to sample +// from. For more information check the +// androidx/media3/effect/SingleColorLut.java class, especially the function +// #transformCubeIntoBitmap with a provided example. +uniform sampler2D uColorLut; +uniform float uColorLutLength; +varying vec2 vTexSamplingCoord; + +// Applies the color lookup using uLut based on the input colors. +vec3 applyLookup(vec3 color) { + // Reminder: Inside OpenGL vector.xyz is the same as vector.rgb. + // Here we use mentions of x and y coordinates to references to + // the position to sample from inside the 2D LUT plane and + // rgb to create the 3D coordinates based on the input colors. + + // To sample from the 3D LUT we interpolate bilinearly twice in the 2D LUT + // to replicate the trilinear interpolation in a 3D LUT. Thus we sample + // from the plane of position redCoordLow and on the plane above. + // redCoordLow points to the lower plane to sample from. + float redCoord = color.r * (uColorLutLength - 1.0); + // Clamping to uColorLutLength - 2 is only needed if redCoord points to the + // most upper plane. In this case there would not be any plane above + // available to sample from. + float redCoordLow = clamp(floor(redCoord), 0.0, uColorLutLength - 2.0); + + // lowerY is indexed in two steps. First redCoordLow defines the plane to + // sample from. Next the green color component is added to index the row in + // the found plane. As described in the NVIDIA blog article about LUTs + // https://developer.nvidia.com/gpugems/gpugems2/part-iii-high-quality-rendering/chapter-24-using-lookup-tables-accelerate-color + // (Section 24.2), we sample from color * scale + offset, where offset is + // defined by 1 / (2 * uColorLutLength) and the scale is defined by + // (uColorLutLength - 1.0) / uColorLutLength. + + // The following derives the equation of lowerY. For this let + // N = uColorLutLenght. The general formula to sample at row y + // is defined as y = N * r + g. + // Using the offset and scale as described in NVIDIA's blog article we get: + // y = offset + (N * r + g) * scale + // y = 1 / (2 * N) + (N * r + g) * (N - 1) / N + // y = 1 / (2 * N) + N * r * (N - 1) / N + g * (N - 1) / N + // We have defined redCoord as r * (N - 1) if we excluded the clamping for + // now, giving us: + // y = 1 / (2 * N) + N * redCoord / N + g * (N - 1) / N + // This simplifies to: + // y = 0.5 / N + (N * redCoord + g * (N - 1)) / N + // y = (0.5 + N * redCoord + g * (N - 1)) / N + // This formula now assumes a coordinate system in the range of [0, N] but + // OpenGL uses a [0, 1] unit coordinate system internally. Thus dividing + // by N gives us the final formula for y: + // y = ((0.5 + N * redCoord + g * (N - 1)) / N) / N + // y = (0.5 + redCoord * N + g * (N - 1)) / (N * N) + float lowerY = + (0.5 + + redCoordLow * uColorLutLength + + color.g * (uColorLutLength - 1.0)) + / (uColorLutLength * uColorLutLength); + // The upperY is the same position moved up by one LUT plane. + float upperY = lowerY + 1.0 / uColorLutLength; + + // The x position is the blue color channel (x-axis in LUT[R][G][B]). + float x = (0.5 + color.b * (uColorLutLength - 1.0)) / uColorLutLength; + + vec3 lowerRgb = texture2D(uColorLut, vec2(x, lowerY)).rgb; + vec3 upperRgb = texture2D(uColorLut, vec2(x, upperY)).rgb; + + // Linearly interpolate between lowerRgb and upperRgb based on the + // distance of the actual in the plane and the lower sampling position. + return mix(lowerRgb, upperRgb, redCoord - redCoordLow); +} + +void main() { + vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); + + gl_FragColor.rgb = applyLookup(inputColor.rgb); + gl_FragColor.a = inputColor.a; +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ColorLut.java b/libraries/effect/src/main/java/androidx/media3/effect/ColorLut.java new file mode 100644 index 0000000000..311b6a3e87 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ColorLut.java @@ -0,0 +1,48 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import android.content.Context; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; + +/** + * Specifies color transformations using color lookup tables to apply to each frame in the fragment + * shader. + */ +@UnstableApi +public interface ColorLut extends GlEffect { + + /** + * Returns the OpenGL texture ID of the LUT to apply to the pixels of the frame with the given + * timestamp. + */ + int getLutTextureId(long presentationTimeUs); + + /** Returns the length N of the 3D N x N x N LUT cube with the given timestamp. */ + int getLength(long presentationTimeUs); + + /** Releases the OpenGL texture of the LUT. */ + void release() throws GlUtil.GlException; + + @Override + default ColorLutProcessor toGlTextureProcessor(Context context, boolean useHdr) + throws FrameProcessingException { + return new ColorLutProcessor(context, /* colorLut= */ this, useHdr); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ColorLutProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/ColorLutProcessor.java new file mode 100644 index 0000000000..f6d772dc16 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ColorLutProcessor.java @@ -0,0 +1,103 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkArgument; + +import android.content.Context; +import android.opengl.GLES20; +import android.opengl.Matrix; +import android.util.Pair; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import java.io.IOException; + +/** Applies a {@link ColorLut} to each frame in the fragment shader. */ +/* package */ final class ColorLutProcessor extends SingleFrameGlTextureProcessor { + private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_lut_es2.glsl"; + + private final GlProgram glProgram; + private final ColorLut colorLut; + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param colorLut The {@link ColorLut} to apply to each frame in order. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709. + * @throws FrameProcessingException If a problem occurs while reading shader files. + */ + public ColorLutProcessor(Context context, ColorLut colorLut, boolean useHdr) + throws FrameProcessingException { + super(useHdr); + // TODO(b/246315245): Add HDR support. + checkArgument(!useHdr, "LutProcessor does not support HDR colors."); + this.colorLut = colorLut; + + try { + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + } catch (IOException | GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + + float[] identityMatrix = new float[16]; + Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0); + glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix); + glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + } + + @Override + public Pair configure(int inputWidth, int inputHeight) { + return Pair.create(inputWidth, inputHeight); + } + + @Override + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { + try { + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + glProgram.setSamplerTexIdUniform( + "uColorLut", colorLut.getLutTextureId(presentationTimeUs), /* texUnitIndex= */ 1); + glProgram.setFloatUniform("uColorLutLength", colorLut.getLength(presentationTimeUs)); + glProgram.bindAttributesAndUniforms(); + + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + } + + @Override + public void release() throws FrameProcessingException { + super.release(); + try { + colorLut.release(); + glProgram.delete(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleColorLut.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleColorLut.java new file mode 100644 index 0000000000..ece986bf48 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleColorLut.java @@ -0,0 +1,160 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkState; + +import android.content.Context; +import android.graphics.Bitmap; +import android.opengl.GLES20; +import android.opengl.GLUtils; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; + +/** Transforms the colors of a frame by applying the same color lookup table to each frame. */ +@UnstableApi +public class SingleColorLut implements ColorLut { + private final int lutTextureId; + private final int length; + + /** + * Creates a new instance. + * + *

{@code lutCube} needs to be a {@code N x N x N} cube and each element is an integer + * representing a color using the {@link Bitmap.Config#ARGB_8888} format. + */ + public static SingleColorLut createFromCube(int[][][] lutCube) throws GlUtil.GlException { + checkArgument( + lutCube.length > 0 && lutCube[0].length > 0 && lutCube[0][0].length > 0, + "LUT must have three dimensions."); + checkArgument( + lutCube.length == lutCube[0].length && lutCube.length == lutCube[0][0].length, + Util.formatInvariant( + "All three dimensions of a LUT must match, received %d x %d x %d.", + lutCube.length, lutCube[0].length, lutCube[0][0].length)); + + return new SingleColorLut(transformCubeIntoBitmap(lutCube)); + } + + /** + * Creates a new instance. + * + *

LUT needs to be a Bitmap of a flattened HALD image of width {@code N} and height {@code + * N^2}. Each element must be an integer representing a color using the {@link + * Bitmap.Config#ARGB_8888} format. + */ + public static SingleColorLut createFromBitmap(Bitmap lut) throws GlUtil.GlException { + checkArgument( + lut.getWidth() * lut.getWidth() == lut.getHeight(), + Util.formatInvariant( + "LUT needs to be in a N x N^2 format, received %d x %d.", + lut.getWidth(), lut.getHeight())); + checkArgument( + lut.getConfig() == Bitmap.Config.ARGB_8888, "Color representation needs to be ARGB_8888."); + + return new SingleColorLut(lut); + } + + private SingleColorLut(Bitmap lut) throws GlUtil.GlException { + length = lut.getWidth(); + lutTextureId = storeLutAsTexture(lut); + } + + private static int storeLutAsTexture(Bitmap bitmap) throws GlUtil.GlException { + int lutTextureId = + GlUtil.createTexture( + bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false); + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0); + GlUtil.checkGlError(); + return lutTextureId; + } + + /** + * Transforms the N x N x N {@code cube} into a N x N^2 {@code bitmap}. + * + * @param cube The 3D Color Lut which gets indexed using {@code cube[R][G][B]}. + * @return A {@link Bitmap} of size {@code N x N^2}, where the {@code cube[R][G][B]} color can be + * indexed at {@code bitmap.getColor(B, N * R + G)}. + */ + private static Bitmap transformCubeIntoBitmap(int[][][] cube) { + // The support for 3D textures starts in OpenGL 3.0 and the Android API 8, Version 2.2 + // uses OpenGL 2.0 which only supports 2D textures. Thus we need to transform the 3D LUT + // into 2D to support all Android SDKs. + + // The cube consists of N planes on the z-direction in the coordinate system where each plane + // has a size of N x N. To transform the cube into a 2D bitmap we stack each N x N plane + // vertically on top of each other. This gives us a bitmap of width N and height N^2. + // + // As an example, lets take the following 3D identity LUT of size 2x2x2: + // cube = [ + // [[(0, 0, 0), (0, 0, 1)], + // [(0, 1, 0), (0, 1, 1)]], + // [[(1, 0, 0), (1, 0, 1)], + // [(1, 1, 0), (1, 1, 1)]] + // ]; + // If we transform this cube now into a 2x2^2 = 2x4 bitmap we yield the following 2D plane: + // bitmap = [[(0, 0, 0), (0, 0, 1)], + // [(0, 1, 0), (0, 1, 1)], + // [(1, 0, 0), (1, 0, 1)], + // [(1, 1, 0), (1, 1, 1)]]; + // media/bitmap/lut/identity.png is an example of how a 32x32x32 3D LUT looks like as an + // 32x32^2 bitmap. + int length = cube.length; + int[] bitmapColorsArray = new int[length * length * length]; + + for (int r = 0; r < length; r++) { + for (int g = 0; g < length; g++) { + for (int b = 0; b < length; b++) { + int color = cube[r][g][b]; + int planePosition = b + length * (g + length * r); + bitmapColorsArray[planePosition] = color; + } + } + } + + return Bitmap.createBitmap( + bitmapColorsArray, + /* width= */ length, + /* height= */ length * length, + Bitmap.Config.ARGB_8888); + } + + @Override + public int getLutTextureId(long presentationTimeUs) { + return lutTextureId; + } + + @Override + public int getLength(long presentationTimeUs) { + return length; + } + + @Override + public void release() throws GlUtil.GlException { + GlUtil.deleteTexture(lutTextureId); + } + + @Override + public ColorLutProcessor toGlTextureProcessor(Context context, boolean useHdr) + throws FrameProcessingException { + checkState(!useHdr, "HDR is currently not supported."); + return new ColorLutProcessor(context, /* colorLut= */ this, useHdr); + } +} diff --git a/libraries/test_data/src/test/assets/media/bitmap/lut/grayscale.png b/libraries/test_data/src/test/assets/media/bitmap/lut/grayscale.png new file mode 100644 index 0000000000..680bab838e Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/lut/grayscale.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/lut/identity.png b/libraries/test_data/src/test/assets/media/bitmap/lut/identity.png new file mode 100644 index 0000000000..19e5fde615 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/lut/identity.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/lut/inverted.png b/libraries/test_data/src/test/assets/media/bitmap/lut/inverted.png new file mode 100644 index 0000000000..d03e6cbb40 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/lut/inverted.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/lut_map_white_to_green.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/lut_map_white_to_green.png new file mode 100644 index 0000000000..bcd6322354 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/lut_map_white_to_green.png differ