Support Separable Convolutions with negative coefficients

fragment_shader_separable_convolution_es2.glsl had optimizations that assumed
all convolution coefficients are positive. Support negative coefficients,
and add tests.

PiperOrigin-RevId: 640104741

libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java

@@ -96,4 +96,43 @@ public class GaussianBlurTest {
     assertThat(actualPresentationTimesUs).containsExactly(22_000L);
     getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH);
   }
+
+  @Test
+  public void gaussianBlur_withNegativeCoefficients_blursFrame() throws Exception {
+    GaussianFunction gaussianFunction =
+        new GaussianFunction(/* sigma= */ 5f, /* numStandardDeviations= */ 2f);
+    ImmutableList<Long> frameTimesUs = ImmutableList.of(22_000L);
+    ImmutableList<Long> actualPresentationTimesUs =
+        generateAndProcessFrames(
+            BLANK_FRAME_WIDTH,
+            BLANK_FRAME_HEIGHT,
+            frameTimesUs,
+            new SeparableConvolution() {
+              @Override
+              public ConvolutionFunction1D getConvolution(long presentationTimeUs) {
+                return new ConvolutionFunction1D() {
+
+                  @Override
+                  public float domainStart() {
+                    return gaussianFunction.domainStart();
+                  }
+
+                  @Override
+                  public float domainEnd() {
+                    return gaussianFunction.domainEnd();
+                  }
+
+                  @Override
+                  public float value(float samplePosition) {
+                    return -gaussianFunction.value(samplePosition);
+                  }
+                };
+              }
+            },
+            textureBitmapReader,
+            TEXT_SPAN_CONSUMER);
+
+    assertThat(actualPresentationTimesUs).containsExactly(22_000L);
+    getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH);
+  }
 }

libraries/effect/src/androidTest/java/androidx/media3/effect/LanczosResampleTest.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapFromFocusedGlFramebuffer;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.createGlTextureFromBitmap;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
+import static androidx.media3.test.utils.TestUtil.PSNR_THRESHOLD;
+import static androidx.media3.test.utils.TestUtil.assertBitmapsAreSimilar;
+import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.opengl.EGLContext;
+import android.opengl.EGLDisplay;
+import android.opengl.EGLSurface;
+import androidx.media3.common.C;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.Size;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+/**
+ * Tests for {@link LanczosResample}.
+ *
+ * <p>Expected images are generated by ffmpeg, using {@code -vf scale=WxH:flags=lanczos:param0=3}.
+ */
+@RunWith(AndroidJUnit4.class)
+public class LanczosResampleTest {
+  @Rule public final TestName testName = new TestName();
+
+  private static final String ORIGINAL_JPG_ASSET_PATH = "media/jpeg/ultraHDR.jpg";
+  private static final String DOWNSCALED_6X_PNG_ASSET_PATH =
+      "test-generated-goldens/LanczosResampleTest/ultraHDR_512x680.png";
+
+  private final Context context = getApplicationContext();
+
+  private String testId;
+  private @MonotonicNonNull EGLDisplay eglDisplay;
+  private @MonotonicNonNull EGLContext eglContext;
+  private @MonotonicNonNull EGLSurface placeholderEglSurface;
+  private @MonotonicNonNull GlShaderProgram lanczosShaderProgram;
+  private int inputTexId;
+  private int inputWidth;
+  private int inputHeight;
+
+  @Before
+  public void createGlObjects() throws Exception {
+    eglDisplay = GlUtil.getDefaultEglDisplay();
+    eglContext = GlUtil.createEglContext(eglDisplay);
+    placeholderEglSurface = GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay);
+
+    Bitmap inputBitmap = readBitmap(ORIGINAL_JPG_ASSET_PATH);
+    inputWidth = inputBitmap.getWidth();
+    inputHeight = inputBitmap.getHeight();
+    inputTexId = createGlTextureFromBitmap(inputBitmap);
+  }
+
+  @Before
+  public void setUpTestId() {
+    testId = testName.getMethodName();
+  }
+
+  @After
+  public void release() throws GlUtil.GlException, VideoFrameProcessingException {
+    if (lanczosShaderProgram != null) {
+      lanczosShaderProgram.release();
+    }
+    GlUtil.destroyEglContext(eglDisplay, eglContext);
+  }
+
+  @Test
+  public void queueInputFrame_with6xDownscale_matchesGoldenFile() throws Exception {
+    float scale = 1f / 6;
+    lanczosShaderProgram =
+        new LanczosResample(/* radius= */ 3, scale).toGlShaderProgram(context, /* useHdr= */ false);
+    Size outputSize = new Size((int) (inputWidth * scale), (int) (inputHeight * scale));
+    setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
+    Bitmap expectedBitmap = readBitmap(DOWNSCALED_6X_PNG_ASSET_PATH);
+    GlTextureInfo inputTextureInfo =
+        new GlTextureInfo(
+            inputTexId,
+            /* fboId= */ C.INDEX_UNSET,
+            /* rboId= */ C.INDEX_UNSET,
+            inputWidth,
+            inputHeight);
+
+    lanczosShaderProgram.queueInputFrame(
+        new DefaultGlObjectsProvider(eglContext), inputTextureInfo, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
+
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD);
+  }
+
+  private void setupOutputTexture(int outputWidth, int outputHeight) throws Exception {
+    int outputTexId =
+        GlUtil.createTexture(
+            outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false);
+    int frameBuffer = GlUtil.createFboForTexture(outputTexId);
+    GlUtil.focusFramebuffer(
+        checkNotNull(eglDisplay),
+        checkNotNull(eglContext),
+        checkNotNull(placeholderEglSurface),
+        frameBuffer,
+        outputWidth,
+        outputHeight);
+  }
+}

libraries/effect/src/main/assets/shaders/fragment_shader_separable_convolution_es2.glsl

@@ -81,14 +81,14 @@ void main() {
     float sample1Weight = texture2D(uFunctionLookupSampler, function1Coord).x;
     float totalSampleWeight = sample0Weight + sample1Weight;
 
-    // Skip samples with very low weight to avoid unnecessary lookups and
+    // Collapse adjacent taps and reduce two lookups to one when sample weights
-    // avoid dividing by 0.
+    // have the same sign. https://vec3.ca/bicubic-filtering-in-fewer-taps/
-    if (abs(totalSampleWeight) > epsilon) {
+    if (sample0Weight * sample1Weight > epsilon &&
+        abs(totalSampleWeight) > epsilon) {
       // Select a coordinate so that a linear sample at that location
       // intrinsically includes the relative sampling weights.
-      float sampleOffsetTexels = (sample0OffsetTexels * sample0Weight +
+      float sampleOffsetTexels =
-                                  sample1OffsetTexels * sample1Weight) /
+          sample0OffsetTexels + sample1Weight / totalSampleWeight;
-                                 totalSampleWeight;
 
       vec2 textureSamplePos =
           vTexSamplingCoord + sampleOffsetTexels * singleTexelStep;
@@ -96,10 +96,20 @@ void main() {
       vec4 textureSampleColor = texture2D(uTexSampler, textureSamplePos);
       accumulatedRgba += textureSampleColor * totalSampleWeight;
       accumulatedWeight += totalSampleWeight;
+    } else {
+      vec2 textureSample0Pos =
+          vTexSamplingCoord + sample0OffsetTexels * singleTexelStep;
+      vec4 textureSample0Color = texture2D(uTexSampler, textureSample0Pos);
+      accumulatedRgba += textureSample0Color * sample0Weight;
+      vec2 textureSample1Pos =
+          vTexSamplingCoord + sample1OffsetTexels * singleTexelStep;
+      vec4 textureSample1Color = texture2D(uTexSampler, textureSample1Pos);
+      accumulatedRgba += textureSample1Color * sample1Weight;
+      accumulatedWeight += totalSampleWeight;
     }
   }
 
-  if (accumulatedWeight > 0.0) {
+  if (abs(accumulatedWeight) > epsilon) {
     gl_FragColor = accumulatedRgba / accumulatedWeight;
   }
 }

libraries/effect/src/main/java/androidx/media3/effect/LanczosResample.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+/**
+ * A {@link SeparableConvolution} that applies a Lanczos-windowed sinc function when resampling an
+ * image. See Filters for Common Resampling Tasks, Ken Turkowski.
+ *
+ * <p>The filter rescales images in both dimensions with the same scaling factor.
+ */
+/* package */ final class LanczosResample extends SeparableConvolution {
+
+  private final float radius;
+  private final float scale;
+
+  /**
+   * Creates an instance.
+   *
+   * @param radius The non-zero radius of the Lanczos reconstruction kernel.
+   * @param scale The scaling factor to be applied when scaling the input image.
+   */
+  public LanczosResample(float radius, float scale) {
+    super(scale, scale);
+    this.radius = radius;
+    this.scale = scale;
+  }
+
+  @Override
+  public ConvolutionFunction1D getConvolution(long presentationTimeUs) {
+    return new ScaledLanczosFunction(radius, scale);
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/ScaledLanczosFunction.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static java.lang.Math.PI;
+import static java.lang.Math.abs;
+import static java.lang.Math.sin;
+
+import androidx.annotation.Nullable;
+import java.util.Objects;
+
+/**
+ * Implementation of a scaled Lanczos window function.
+ *
+ * <p>The function input is multiplied by {@code scale} before applying the textbook Lanczos window
+ * function.
+ */
+/* package */ final class ScaledLanczosFunction implements ConvolutionFunction1D {
+  private final float radius;
+  private final float scale;
+
+  /**
+   * Creates an instance.
+   *
+   * @param radius The radius parameter of the Lanczos window function.
+   * @param scale The scaling factor applied to inputs.
+   */
+  public ScaledLanczosFunction(float radius, float scale) {
+    this.radius = radius;
+    this.scale = scale;
+  }
+
+  @Override
+  public float domainStart() {
+    return -radius / scale;
+  }
+
+  @Override
+  public float domainEnd() {
+    return radius / scale;
+  }
+
+  @Override
+  public float value(float samplePosition) {
+    float x = samplePosition * scale;
+    if (abs(x) < 1e-5) {
+      return 1.0f;
+    }
+    if (abs(x) > radius) {
+      return 0.0f;
+    }
+    return (float) (radius * sin(PI * x) * sin(PI * x / radius) / (PI * PI * x * x));
+  }
+
+  @Override
+  public boolean equals(@Nullable Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof ScaledLanczosFunction)) {
+      return false;
+    }
+    ScaledLanczosFunction that = (ScaledLanczosFunction) o;
+    return Float.compare(that.radius, radius) == 0 && Float.compare(that.scale, scale) == 0;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(radius, scale);
+  }
+}

libraries/test_utils/src/main/java/androidx/media3/test/utils/TestUtil.java

@@ -80,6 +80,12 @@ import java.util.concurrent.ExecutionException;
 /** Utility methods for tests. */
 @UnstableApi
 public class TestUtil {
+  /**
+   * Luma PSNR values between 30 and 50 are considered good for lossy compression (See <a
+   * href="https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio#Quality_estimation_with_PSNR">Quality
+   * estimation with PSNR</a> ).
+   */
+  public static final float PSNR_THRESHOLD = 35f;
 
   private TestUtil() {}