From ae563dbab07e9eb37f1b149bcfdf3598e8331d70 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 24 Mar 2025 04:06:10 -0700 Subject: [PATCH] Update Lanczos Effect to support different orientations PiperOrigin-RevId: 739884019 --- .../CompositionPreviewActivity.java | 3 +- .../demo/transformer/TransformerActivity.java | 2 +- .../media3/effect/LanczosResampleTest.java | 77 ++++++++++++- .../media3/effect/LanczosResample.java | 105 +++++++++++++++--- 4 files changed, 165 insertions(+), 22 deletions(-) diff --git a/demos/composition/src/main/java/androidx/media3/demo/composition/CompositionPreviewActivity.java b/demos/composition/src/main/java/androidx/media3/demo/composition/CompositionPreviewActivity.java index 82436fa3db..d6c4884e21 100644 --- a/demos/composition/src/main/java/androidx/media3/demo/composition/CompositionPreviewActivity.java +++ b/demos/composition/src/main/java/androidx/media3/demo/composition/CompositionPreviewActivity.java @@ -217,7 +217,8 @@ public final class CompositionPreviewActivity extends AppCompatActivity { String selectedResolutionHeight = String.valueOf(resolutionHeightSpinner.getSelectedItem()); if (!SAME_AS_INPUT_OPTION.equals(selectedResolutionHeight)) { int resolutionHeight = Integer.parseInt(selectedResolutionHeight); - videoEffectsBuilder.add(LanczosResample.scaleToFit(10000, resolutionHeight)); + videoEffectsBuilder.add( + LanczosResample.scaleToFitWithFlexibleOrientation(10000, resolutionHeight)); videoEffectsBuilder.add(Presentation.createForShortSide(resolutionHeight)); } ImmutableList videoEffects = videoEffectsBuilder.build(); diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index c1214a79fa..d1cae9990c 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -677,7 +677,7 @@ public final class TransformerActivity extends AppCompatActivity { int resolutionHeight = bundle.getInt(ConfigurationActivity.RESOLUTION_HEIGHT, /* defaultValue= */ C.LENGTH_UNSET); if (resolutionHeight != C.LENGTH_UNSET) { - effects.add(LanczosResample.scaleToFit(10000, resolutionHeight)); + effects.add(LanczosResample.scaleToFitWithFlexibleOrientation(10000, resolutionHeight)); effects.add(Presentation.createForShortSide(resolutionHeight)); } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/LanczosResampleTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/LanczosResampleTest.java index 3af28b211b..a62c1ccd49 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/LanczosResampleTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/LanczosResampleTest.java @@ -24,6 +24,7 @@ import static androidx.media3.test.utils.TestUtil.PSNR_THRESHOLD; import static androidx.media3.test.utils.TestUtil.assertBitmapsAreSimilar; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; +import static java.lang.Math.round; import android.content.Context; import android.graphics.Bitmap; @@ -93,7 +94,7 @@ public class LanczosResampleTest { GlTextureInfo inputTextureInfo = setupInputTexture(ORIGINAL_JPG_ASSET_PATH); float scale = 1f / 6; Size outputSize = - new Size((int) (inputTextureInfo.width * scale), (int) (inputTextureInfo.height * scale)); + new Size(round(inputTextureInfo.width * scale), round(inputTextureInfo.height * scale)); lanczosShaderProgram = LanczosResample.scaleToFit(outputSize.getWidth(), outputSize.getHeight()) .toGlShaderProgram(context, /* useHdr= */ false); @@ -109,12 +110,35 @@ public class LanczosResampleTest { assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD); } + @Test + public void queueInputFrame_with6xDownscaleFlexibleOrientation_matchesGoldenFile() + throws Exception { + GlTextureInfo inputTextureInfo = setupInputTexture(ORIGINAL_JPG_ASSET_PATH); + float scale = 1f / 6; + Size outputSize = + new Size(round(inputTextureInfo.width * scale), round(inputTextureInfo.height * scale)); + lanczosShaderProgram = + LanczosResample.scaleToFitWithFlexibleOrientation( + outputSize.getHeight(), outputSize.getWidth()) + .toGlShaderProgram(context, /* useHdr= */ false); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = readBitmap(DOWNSCALED_6X_PNG_ASSET_PATH); + + lanczosShaderProgram.queueInputFrame( + new DefaultGlObjectsProvider(eglContext), inputTextureInfo, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD); + } + @Test public void queueInputFrame_with3xUpscale_matchesGoldenFile() throws Exception { GlTextureInfo inputTextureInfo = setupInputTexture(SMALLER_JPG_ASSET_PATH); float scale = 3; Size outputSize = - new Size((int) (inputTextureInfo.width * scale), (int) (inputTextureInfo.height * scale)); + new Size(round(inputTextureInfo.width * scale), round(inputTextureInfo.height * scale)); lanczosShaderProgram = LanczosResample.scaleToFit(outputSize.getWidth(), outputSize.getHeight()) .toGlShaderProgram(context, /* useHdr= */ false); @@ -130,6 +154,29 @@ public class LanczosResampleTest { assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD); } + @Test + public void queueInputFrame_with3xUpscaleFlexibleOrientation_matchesGoldenFile() + throws Exception { + GlTextureInfo inputTextureInfo = setupInputTexture(SMALLER_JPG_ASSET_PATH); + float scale = 3; + Size outputSize = + new Size((int) (inputTextureInfo.width * scale), (int) (inputTextureInfo.height * scale)); + lanczosShaderProgram = + LanczosResample.scaleToFitWithFlexibleOrientation( + outputSize.getWidth(), outputSize.getHeight()) + .toGlShaderProgram(context, /* useHdr= */ false); + setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); + Bitmap expectedBitmap = readBitmap(UPSCALED_3X_PNG_ASSET_PATH); + + lanczosShaderProgram.queueInputFrame( + new DefaultGlObjectsProvider(eglContext), inputTextureInfo, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromFocusedGlFramebuffer(outputSize.getWidth(), outputSize.getHeight()); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD); + } + @Test public void isNoOp_whenSizeDoesntChange_returnsTrue() { LanczosResample lanczosResample = LanczosResample.scaleToFit(720, 1280); @@ -137,6 +184,14 @@ public class LanczosResampleTest { assertThat(lanczosResample.isNoOp(720, 1280)).isTrue(); } + @Test + public void isNoOp_whenSizeDoesntChangeFlexibleOrientation_returnsTrue() { + LanczosResample lanczosResample = LanczosResample.scaleToFitWithFlexibleOrientation(720, 1280); + + assertThat(lanczosResample.isNoOp(720, 1280)).isTrue(); + assertThat(lanczosResample.isNoOp(1280, 720)).isTrue(); + } + @Test public void isNoOp_forSmallScalingFactors_returnsTrue() { LanczosResample lanczosResample = LanczosResample.scaleToFit(1920, 1072); @@ -145,12 +200,28 @@ public class LanczosResampleTest { } @Test - public void isNoOp_forLargeScalingFactors_returnsTrue() { + public void isNoOp_forSmallScalingFactorsFlexibleOrientation_returnsTrue() { + LanczosResample lanczosResample = LanczosResample.scaleToFitWithFlexibleOrientation(1920, 1072); + + assertThat(lanczosResample.isNoOp(1920, 1080)).isTrue(); + assertThat(lanczosResample.isNoOp(1080, 1920)).isTrue(); + } + + @Test + public void isNoOp_forLargeScalingFactors_returnsFalse() { LanczosResample lanczosResample = LanczosResample.scaleToFit(1920, 1068); assertThat(lanczosResample.isNoOp(1920, 1080)).isFalse(); } + @Test + public void isNoOp_forLargeScalingFactorsFlexibleOrientation_returnsFalse() { + LanczosResample lanczosResample = LanczosResample.scaleToFitWithFlexibleOrientation(1920, 1068); + + assertThat(lanczosResample.isNoOp(1920, 1080)).isFalse(); + assertThat(lanczosResample.isNoOp(1080, 1920)).isFalse(); + } + private static GlTextureInfo setupInputTexture(String path) throws Exception { Bitmap inputBitmap = readBitmap(path); return new GlTextureInfo( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/LanczosResample.java b/libraries/effect/src/main/java/androidx/media3/effect/LanczosResample.java index 21000a4b44..2bca0e32ca 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/LanczosResample.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/LanczosResample.java @@ -43,8 +43,9 @@ public final class LanczosResample implements GlEffect { private static final float NO_OP_THRESHOLD = 0.01f; private final float radius; - private final int width; - private final int height; + private final int longSide; + private final int shortSide; + private final boolean assumeLandscapeOrientation; /** * Creates an instance. @@ -56,20 +57,56 @@ public final class LanczosResample implements GlEffect { @IntRange(from = 1) int width, @IntRange(from = 1) int height) { checkArgument(width > 0); checkArgument(height > 0); - return new LanczosResample(DEFAULT_RADIUS, width, height); + return new LanczosResample( + DEFAULT_RADIUS, width, height, /* assumeLandscapeOrientation= */ true); } - private LanczosResample(float radius, int width, int height) { + /** + * Creates an instance. + * + *

The output resolution will be either {@code firstDimension} x {@code secondDimension} or + * {@code secondDimension} x {@code firstDimension}. The longer of {@code firstDimension} or + * {@code secondDimension} will have the same orientation as the longer side of the {@link Size} + * passed in to {@link LanczosResampleScaledFunctionProvider#configure}. + * + * @param firstDimension The first dimension of the output contents. + * @param secondDimension The second dimension of the output contents. + */ + public static LanczosResample scaleToFitWithFlexibleOrientation( + @IntRange(from = 1) int firstDimension, @IntRange(from = 1) int secondDimension) { + checkArgument(firstDimension > 0); + checkArgument(secondDimension > 0); + if (firstDimension > secondDimension) { + return new LanczosResample( + DEFAULT_RADIUS, + /* longSide= */ firstDimension, + /* shortSide= */ secondDimension, + /* assumeLandscapeOrientation= */ false); + } else { + return new LanczosResample( + DEFAULT_RADIUS, + /* longSide= */ secondDimension, + /* shortSide= */ firstDimension, + /* assumeLandscapeOrientation= */ false); + } + } + + private LanczosResample( + float radius, int longSide, int shortSide, boolean assumeLandscapeOrientation) { this.radius = radius; - this.width = width; - this.height = height; + this.longSide = longSide; + this.shortSide = shortSide; + this.assumeLandscapeOrientation = assumeLandscapeOrientation; } @Override public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) throws VideoFrameProcessingException { return new SeparableConvolutionShaderProgram( - context, useHdr, new LanczosResampleScaledFunctionProvider(radius, width, height)); + context, + useHdr, + new LanczosResampleScaledFunctionProvider( + radius, longSide, shortSide, assumeLandscapeOrientation)); } /** @@ -80,7 +117,13 @@ public final class LanczosResample implements GlEffect { */ @Override public boolean isNoOp(int inputWidth, int inputHeight) { - return abs(scalingFactorToFit(inputWidth, inputHeight, width, height) - 1f) < NO_OP_THRESHOLD; + Size targetSize = + getTargetSize(inputWidth, inputHeight, longSide, shortSide, assumeLandscapeOrientation); + return abs( + scalingFactorToFit( + inputWidth, inputHeight, targetSize.getWidth(), targetSize.getHeight()) + - 1f) + < NO_OP_THRESHOLD; } /** @@ -108,21 +151,24 @@ public final class LanczosResample implements GlEffect { // Note: We deliberately don't use Float.MIN_VALUE because it's positive & very close to zero. private static final float SCALE_UNSET = -Float.MAX_VALUE; private final float radius; - private final int width; - private final int height; + private final int longSide; + private final int shortSide; + private final boolean assumeLandscapeOrientation; private float scale; private LanczosResampleScaledFunctionProvider( @FloatRange(from = 0, fromInclusive = false) float radius, - @IntRange(from = 1) int width, - @IntRange(from = 1) int height) { + @IntRange(from = 1) int longSide, + @IntRange(from = 1) int shortSide, + boolean assumeLandscapeOrientation) { checkArgument(radius > 0); - checkArgument(width > 0); - checkArgument(height > 0); + checkArgument(longSide > 0); + checkArgument(shortSide > 0); this.radius = radius; - this.width = width; - this.height = height; + this.longSide = longSide; + this.shortSide = shortSide; + this.assumeLandscapeOrientation = assumeLandscapeOrientation; scale = SCALE_UNSET; } @@ -136,8 +182,33 @@ public final class LanczosResample implements GlEffect { @Override public Size configure(Size inputSize) { - scale = scalingFactorToFit(inputSize.getWidth(), inputSize.getHeight(), width, height); + Size targetSize = + LanczosResample.getTargetSize( + inputSize.getWidth(), + inputSize.getHeight(), + longSide, + shortSide, + assumeLandscapeOrientation); + scale = + scalingFactorToFit( + inputSize.getWidth(), + inputSize.getHeight(), + targetSize.getWidth(), + targetSize.getHeight()); return new Size(round(inputSize.getWidth() * scale), round(inputSize.getHeight() * scale)); } } + + private static Size getTargetSize( + int inputWidth, + int inputHeight, + int longSide, + int shortSide, + boolean assumeLandscapeOrientation) { + if (assumeLandscapeOrientation || inputWidth > inputHeight) { + return new Size(longSide, shortSide); + } else { + return new Size(shortSide, longSide); + } + } }