Possible values are: + * + *
The algorithms are ordered by increasing visual quality and computational cost. + */ + @UnstableApi + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef({TEXTURE_MIN_FILTER_LINEAR, TEXTURE_MIN_FILTER_LINEAR_MIPMAP_LINEAR}) + public @interface TextureMinFilter {} + + /** + * Returns the weighted average of the four texture elements that are closest to the specified + * texture coordinates. + */ + @UnstableApi public static final int TEXTURE_MIN_FILTER_LINEAR = GLES20.GL_LINEAR; + + /** + * Chooses the two mipmaps that most closely match the size of the pixel being textured and uses + * the {@link C#TEXTURE_MIN_FILTER_LINEAR} criterion (a weighted average of the texture elements + * that are closest to the specified texture coordinates) to produce a texture value from each + * mipmap. The final texture value is a weighted average of those two values. + */ + @UnstableApi + public static final int TEXTURE_MIN_FILTER_LINEAR_MIPMAP_LINEAR = GLES20.GL_LINEAR_MIPMAP_LINEAR; + /** * @deprecated Use {@link Util#usToMs(long)}. */ diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java b/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java index 674c3e64b8..410e8ce701 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java @@ -15,12 +15,14 @@ */ package androidx.media3.common.util; +import static androidx.media3.common.C.TEXTURE_MIN_FILTER_LINEAR_MIPMAP_LINEAR; import static androidx.media3.common.util.Assertions.checkNotNull; import android.content.Context; import android.opengl.GLES11Ext; import android.opengl.GLES20; import androidx.annotation.Nullable; +import androidx.media3.common.C; import java.io.IOException; import java.nio.Buffer; import java.util.HashMap; @@ -186,6 +188,22 @@ public final class GlProgram { checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, texUnitIndex); } + /** + * Sets a texture sampler type uniform. + * + * @param name The uniform's name. + * @param texId The texture identifier. + * @param texUnitIndex The texture unit index. Use a different index (0, 1, 2, ...) for each + * texture sampler in the program. + * @param texMinFilter The {@link C.TextureMinFilter}. + */ + public void setSamplerTexIdUniform( + String name, int texId, int texUnitIndex, @C.TextureMinFilter int texMinFilter) { + Uniform texUniform = checkNotNull(uniformByName.get(name)); + texUniform.setSamplerTexId(texId, texUnitIndex); + texUniform.setTexMinFilter(texMinFilter); + } + /** Sets an {@code int} type uniform. */ public void setIntUniform(String name, int value) { checkNotNull(uniformByName.get(name)).setInt(value); @@ -365,6 +383,7 @@ public final class GlProgram { private int texIdValue; private int texUnitIndex; + private @C.TextureMinFilter int texMinFilter; private Uniform(String name, int location, int type) { this.name = name; @@ -372,6 +391,7 @@ public final class GlProgram { this.type = type; this.floatValue = new float[16]; // Allocate 16 for mat4 this.intValue = new int[4]; // Allocate 4 for ivec4 + this.texMinFilter = C.TEXTURE_MIN_FILTER_LINEAR; } /** @@ -386,6 +406,19 @@ public final class GlProgram { this.texUnitIndex = texUnitIndex; } + /** + * Configures {@link #bind(boolean)} to use the specified texture minification filter for this + * sampler uniform. + * + *
Only has effect for {@linkplain GLES20#GL_SAMPLER_2D internal texture} type. External + * texture sampling is controlled via the parameter passed to {@link #bind(boolean)}. + * + * @param texMinFilter The {@link C.TextureMinFilter}. + */ + public void setTexMinFilter(@C.TextureMinFilter int texMinFilter) { + this.texMinFilter = texMinFilter; + } + /** Configures {@link #bind(boolean)} to use the specified {@code int} {@code value}. */ public void setInt(int value) { this.intValue[0] = value; @@ -476,6 +509,15 @@ public final class GlProgram { type == GLES20.GL_SAMPLER_2D || !externalTexturesRequireNearestSampling ? GLES20.GL_LINEAR : GLES20.GL_NEAREST); + if (type == GLES20.GL_SAMPLER_2D) { + if (texMinFilter == TEXTURE_MIN_FILTER_LINEAR_MIPMAP_LINEAR) { + GLES20.glGenerateMipmap(GLES20.GL_TEXTURE_2D); + GlUtil.checkGlError(); + } + GLES20.glTexParameteri( + GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, texMinFilter); + GlUtil.checkGlError(); + } GLES20.glUniform1i(location, texUnitIndex); GlUtil.checkGlError(); break; diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/PresentationPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/PresentationPixelTest.java index c922f97ae0..b07f45933c 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/PresentationPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/PresentationPixelTest.java @@ -22,6 +22,8 @@ import static androidx.media3.test.utils.BitmapPixelTestUtil.createGlTextureFrom import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.media3.test.utils.TestUtil.PSNR_THRESHOLD; +import static androidx.media3.test.utils.TestUtil.assertBitmapsAreSimilar; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; @@ -71,6 +73,9 @@ public final class PresentationPixelTest { "test-generated-goldens/sample_mp4_first_frame/electrical_colors/aspect_ratio_stretch_to_fit_narrow.png"; private static final String ASPECT_RATIO_STRETCH_TO_FIT_WIDE_PNG_ASSET_PATH = "test-generated-goldens/sample_mp4_first_frame/electrical_colors/aspect_ratio_stretch_to_fit_wide.png"; + private static final String HIGH_RESOLUTION_JPG_ASSET_PATH = "media/jpeg/ultraHDR.jpg"; + private static final String DOWNSCALED_6X_PNG_ASSET_PATH = + "test-generated-goldens/PresentationPixelTest/ultraHDR_mipmap_512x680.png"; private final Context context = getApplicationContext(); @@ -254,6 +259,29 @@ public final class PresentationPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void drawFrame_downscaleWithLinearMipmap_matchesGoldenFile() throws Exception { + Bitmap inputBitmap = readBitmap(HIGH_RESOLUTION_JPG_ASSET_PATH); + inputWidth = inputBitmap.getWidth(); + inputHeight = inputBitmap.getHeight(); + inputTexId = createGlTextureFromBitmap(inputBitmap); + presentationShaderProgram = + Presentation.createForWidthAndHeight( + inputWidth / 6, inputHeight / 6, Presentation.LAYOUT_SCALE_TO_FIT) + .copyWithTextureMinFilter(C.TEXTURE_MIN_FILTER_LINEAR_MIPMAP_LINEAR) + .toGlShaderProgram(context, /* useHdr= */ false); + Size outputSize = presentationShaderProgram.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = readBitmap(DOWNSCALED_6X_PNG_ASSET_PATH); + + presentationShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD); + } + private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { int outputTexId = GlUtil.createTexture( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java index 15da706830..1c8c0c0bfc 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java @@ -21,6 +21,7 @@ import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR; +import static java.lang.Math.max; import android.content.Context; import android.graphics.Gainmap; @@ -140,6 +141,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** Matrix for storing an intermediate calculation result. */ private final float[] tempResultMatrix; + /** The texture minification filter to use when sampling from the input texture. */ + private final @C.TextureMinFilter int textureMinFilter; + /** * A polygon in the input space chosen such that no additional clipping is needed to keep vertices * inside the NDC range when applying each of the {@link #matrixTransformations}. @@ -473,6 +477,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; tempResultMatrix = new float[16]; visiblePolygon = NDC_SQUARE; gainmapTexId = C.INDEX_UNSET; + + // When multiple matrix transformations are applied in a single shader program, use the highest + // quality resampling algorithm requested. + @C.TextureMinFilter int textureMinFilter = C.TEXTURE_MIN_FILTER_LINEAR; + for (int i = 0; i < matrixTransformations.size(); i++) { + textureMinFilter = + max(textureMinFilter, matrixTransformations.get(i).getGlTextureMinFilter()); + } + this.textureMinFilter = textureMinFilter; } private static GlProgram createGlProgram( @@ -518,7 +531,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; try { glProgram.use(); setGainmapSamplerAndUniforms(); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + glProgram.setSamplerTexIdUniform( + "uTexSampler", inputTexId, /* texUnitIndex= */ 0, textureMinFilter); glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrixArray); glProgram.setFloatsUniformIfPresent("uRgbMatrix", compositeRgbMatrixArray); glProgram.setBufferAttribute( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java b/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java index 5eec26cb5b..858d34a725 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java @@ -15,8 +15,11 @@ */ package androidx.media3.effect; +import static androidx.media3.common.C.TEXTURE_MIN_FILTER_LINEAR; + import android.content.Context; import android.opengl.Matrix; +import androidx.media3.common.C; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; @@ -47,6 +50,14 @@ public interface GlMatrixTransformation extends GlEffect { return new Size(inputWidth, inputHeight); } + /** + * Returns the {@linkplain C.TextureMinFilter texture minification filter} to use for sampling the + * input texture when applying this matrix transformation. + */ + default @C.TextureMinFilter int getGlTextureMinFilter() { + return TEXTURE_MIN_FILTER_LINEAR; + } + /** * Returns the 4x4 transformation {@link Matrix} to apply to the frame with the given timestamp. */ diff --git a/libraries/effect/src/main/java/androidx/media3/effect/Presentation.java b/libraries/effect/src/main/java/androidx/media3/effect/Presentation.java index cf49c7d5bf..5228508264 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/Presentation.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/Presentation.java @@ -15,6 +15,8 @@ */ package androidx.media3.effect; +import static androidx.media3.common.C.TEXTURE_MIN_FILTER_LINEAR; +import static androidx.media3.common.C.TEXTURE_MIN_FILTER_LINEAR_MIPMAP_LINEAR; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static java.lang.annotation.ElementType.TYPE_USE; @@ -125,7 +127,11 @@ public final class Presentation implements MatrixTransformation { checkArgument(aspectRatio > 0, "aspect ratio " + aspectRatio + " must be positive"); checkLayout(layout); return new Presentation( - /* width= */ C.LENGTH_UNSET, /* height= */ C.LENGTH_UNSET, aspectRatio, layout); + /* width= */ C.LENGTH_UNSET, + /* height= */ C.LENGTH_UNSET, + aspectRatio, + layout, + TEXTURE_MIN_FILTER_LINEAR); } /** @@ -138,7 +144,11 @@ public final class Presentation implements MatrixTransformation { */ public static Presentation createForHeight(int height) { return new Presentation( - /* width= */ C.LENGTH_UNSET, height, ASPECT_RATIO_UNSET, LAYOUT_SCALE_TO_FIT); + /* width= */ C.LENGTH_UNSET, + height, + ASPECT_RATIO_UNSET, + LAYOUT_SCALE_TO_FIT, + TEXTURE_MIN_FILTER_LINEAR); } /** @@ -156,19 +166,25 @@ public final class Presentation implements MatrixTransformation { checkArgument(width > 0, "width " + width + " must be positive"); checkArgument(height > 0, "height " + height + " must be positive"); checkLayout(layout); - return new Presentation(width, height, ASPECT_RATIO_UNSET, layout); + return new Presentation(width, height, ASPECT_RATIO_UNSET, layout, TEXTURE_MIN_FILTER_LINEAR); } private final int requestedWidthPixels; private final int requestedHeightPixels; private float requestedAspectRatio; private final @Layout int layout; + private final @C.TextureMinFilter int textureMinFilter; private float outputWidth; private float outputHeight; private @MonotonicNonNull Matrix transformationMatrix; - private Presentation(int width, int height, float aspectRatio, @Layout int layout) { + private Presentation( + int width, + int height, + float aspectRatio, + @Layout int layout, + @C.TextureMinFilter int textureMinFilter) { checkArgument( (aspectRatio == ASPECT_RATIO_UNSET) || (width == C.LENGTH_UNSET), "width and aspect ratio should not both be set"); @@ -177,12 +193,35 @@ public final class Presentation implements MatrixTransformation { this.requestedHeightPixels = height; this.requestedAspectRatio = aspectRatio; this.layout = layout; + this.textureMinFilter = textureMinFilter; outputWidth = C.LENGTH_UNSET; outputHeight = C.LENGTH_UNSET; transformationMatrix = new Matrix(); } + /** + * Returns a copy with the specified texture minification filter. + * + * @param textureMinFilter The {@link C.TextureMinFilter}. + */ + public Presentation copyWithTextureMinFilter(@C.TextureMinFilter int textureMinFilter) { + checkArgument( + textureMinFilter == TEXTURE_MIN_FILTER_LINEAR + || textureMinFilter == TEXTURE_MIN_FILTER_LINEAR_MIPMAP_LINEAR); + return new Presentation( + requestedWidthPixels, + requestedHeightPixels, + requestedAspectRatio, + layout, + textureMinFilter); + } + + @Override + public @C.TextureMinFilter int getGlTextureMinFilter() { + return textureMinFilter; + } + @Override public Size configure(int inputWidth, int inputHeight) { checkArgument(inputWidth > 0, "inputWidth must be positive"); diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/PresentationPixelTest/ultraHDR_mipmap_512x680.png b/libraries/test_data/src/test/assets/test-generated-goldens/PresentationPixelTest/ultraHDR_mipmap_512x680.png new file mode 100644 index 0000000000..baf4ead10e Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/PresentationPixelTest/ultraHDR_mipmap_512x680.png differ