diff --git a/libraries/effect/src/main/java/androidx/media3/effect/Crop.java b/libraries/effect/src/main/java/androidx/media3/effect/Crop.java index 61736e91e3..509497c2dd 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/Crop.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/Crop.java @@ -93,4 +93,12 @@ public final class Crop implements MatrixTransformation { public Matrix getMatrix(long presentationTimeUs) { return checkStateNotNull(transformationMatrix, "configure must be called first"); } + + @Override + public boolean isNoOp(int inputWidth, int inputHeight) { + Size outputSize = configure(inputWidth, inputHeight); + return checkStateNotNull(transformationMatrix).isIdentity() + && inputWidth == outputSize.getWidth() + && inputHeight == outputSize.getHeight(); + } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java index 3d67934e38..817a66febc 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java @@ -40,4 +40,19 @@ public interface GlEffect extends Effect { */ GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) throws FrameProcessingException; + + /** + * Returns whether a {@link GlEffect} applies no change at every timestamp. + * + *

This can be used as a hint to skip this instance. + * + * @param inputWidth The input frame width, in pixels. + * @param inputHeight The input frame height, in pixels. + */ + default boolean isNoOp(int inputWidth, int inputHeight) { + // TODO(b/265927935): Generalize this logic by implementing this method on all + // subclasses, and deleting the default implementation here. Otherwise, some no-op effects may + // not be properly detected or handled. + return false; + } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/Presentation.java b/libraries/effect/src/main/java/androidx/media3/effect/Presentation.java index a1ef5d611a..2547036e4b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/Presentation.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/Presentation.java @@ -214,6 +214,14 @@ public final class Presentation implements MatrixTransformation { return checkStateNotNull(transformationMatrix, "configure must be called first"); } + @Override + public boolean isNoOp(int inputWidth, int inputHeight) { + configure(inputWidth, inputHeight); + return checkStateNotNull(transformationMatrix).isIdentity() + && inputWidth == Math.round(outputWidth) + && inputHeight == Math.round(outputHeight); + } + @RequiresNonNull("transformationMatrix") private void applyAspectRatio() { float inputAspectRatio = outputWidth / outputHeight; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ScaleToFitTransformation.java b/libraries/effect/src/main/java/androidx/media3/effect/ScaleToFitTransformation.java index ff0bfa7d0f..75872ba16c 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ScaleToFitTransformation.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ScaleToFitTransformation.java @@ -92,13 +92,6 @@ public final class ScaleToFitTransformation implements MatrixTransformation { } } - /** The multiplier by which the frame will scale horizontally, along the x-axis. */ - public final float scaleX; - /** The multiplier by which the frame will scale vertically, along the y-axis. */ - public final float scaleY; - /** How much to rotate the frame counterclockwise, in degrees. */ - public final float rotationDegrees; - private final Matrix transformationMatrix; private @MonotonicNonNull Matrix adjustedTransformationMatrix; @@ -110,9 +103,6 @@ public final class ScaleToFitTransformation implements MatrixTransformation { * @param rotationDegrees How much to rotate the frame counterclockwise, in degrees. */ private ScaleToFitTransformation(float scaleX, float scaleY, float rotationDegrees) { - this.scaleX = scaleX; - this.scaleY = scaleY; - this.rotationDegrees = rotationDegrees; transformationMatrix = new Matrix(); transformationMatrix.postScale(scaleX, scaleY); transformationMatrix.postRotate(rotationDegrees); @@ -162,4 +152,12 @@ public final class ScaleToFitTransformation implements MatrixTransformation { public Matrix getMatrix(long presentationTimeUs) { return checkStateNotNull(adjustedTransformationMatrix, "configure must be called first"); } + + @Override + public boolean isNoOp(int inputWidth, int inputHeight) { + Size outputSize = configure(inputWidth, inputHeight); + return checkStateNotNull(adjustedTransformationMatrix).isIdentity() + && inputWidth == outputSize.getWidth() + && inputHeight == outputSize.getHeight(); + } } diff --git a/libraries/effect/src/test/java/androidx/media3/effect/CropTest.java b/libraries/effect/src/test/java/androidx/media3/effect/CropTest.java index d6dbd6e309..347cc0af91 100644 --- a/libraries/effect/src/test/java/androidx/media3/effect/CropTest.java +++ b/libraries/effect/src/test/java/androidx/media3/effect/CropTest.java @@ -37,7 +37,9 @@ public final class CropTest { Crop crop = new Crop(/* left= */ -1, /* right= */ 1, /* bottom= */ -1, /* top= */ 1); Size outputSize = crop.configure(inputWidth, inputHeight); + boolean isNoOp = crop.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -53,7 +55,9 @@ public final class CropTest { Crop crop = new Crop(left, right, bottom, top); Size outputSize = crop.configure(inputWidth, inputHeight); + boolean isNoOp = crop.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC); int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC); assertThat(outputSize.getWidth()).isEqualTo(expectedPostCropWidth); diff --git a/libraries/effect/src/test/java/androidx/media3/effect/PresentationTest.java b/libraries/effect/src/test/java/androidx/media3/effect/PresentationTest.java index 40e5019443..3cecedc6fe 100644 --- a/libraries/effect/src/test/java/androidx/media3/effect/PresentationTest.java +++ b/libraries/effect/src/test/java/androidx/media3/effect/PresentationTest.java @@ -37,7 +37,9 @@ public final class PresentationTest { Presentation presentation = Presentation.createForHeight(C.LENGTH_UNSET); Size outputSize = presentation.configure(inputWidth, inputHeight); + boolean isNoOp = presentation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -50,7 +52,9 @@ public final class PresentationTest { Presentation presentation = Presentation.createForHeight(requestedHeight); Size outputSize = presentation.configure(inputWidth, inputHeight); + boolean isNoOp = presentation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight); assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); } @@ -64,7 +68,9 @@ public final class PresentationTest { Presentation.createForAspectRatio(aspectRatio, Presentation.LAYOUT_SCALE_TO_FIT); Size outputSize = presentation.configure(inputWidth, inputHeight); + boolean isNoOp = presentation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * inputHeight)); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -80,7 +86,9 @@ public final class PresentationTest { requestedWidth, requestedHeight, Presentation.LAYOUT_SCALE_TO_FIT); Size outputSize = presentation.configure(inputWidth, inputHeight); + boolean isNoOp = presentation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(requestedWidth); assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); } diff --git a/libraries/effect/src/test/java/androidx/media3/effect/ScaleToFitTransformationTest.java b/libraries/effect/src/test/java/androidx/media3/effect/ScaleToFitTransformationTest.java index 8bd172de5e..c3666014cf 100644 --- a/libraries/effect/src/test/java/androidx/media3/effect/ScaleToFitTransformationTest.java +++ b/libraries/effect/src/test/java/androidx/media3/effect/ScaleToFitTransformationTest.java @@ -39,7 +39,9 @@ public final class ScaleToFitTransformationTest { new ScaleToFitTransformation.Builder().build(); Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight); + boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -54,7 +56,9 @@ public final class ScaleToFitTransformationTest { .build(); Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight); + boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f)); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -67,7 +71,9 @@ public final class ScaleToFitTransformationTest { new ScaleToFitTransformation.Builder().setScale(/* scaleX= */ 2f, /* scaleY= */ 1f).build(); Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight); + boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -80,7 +86,9 @@ public final class ScaleToFitTransformationTest { new ScaleToFitTransformation.Builder().setScale(/* scaleX= */ 1f, /* scaleY= */ 2f).build(); Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight); + boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2); } @@ -93,7 +101,9 @@ public final class ScaleToFitTransformationTest { new ScaleToFitTransformation.Builder().setRotationDegrees(90).build(); Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight); + boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputWidth); } @@ -107,7 +117,9 @@ public final class ScaleToFitTransformationTest { long expectedOutputWidthHeight = 247; Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight); + boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight); + assertThat(isNoOp).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight); assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 379a196b9d..468105ab0c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -39,9 +39,7 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Clock; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.HandlerWrapper; -import androidx.media3.common.util.Size; -import androidx.media3.effect.Presentation; -import androidx.media3.effect.ScaleToFitTransformation; +import androidx.media3.effect.GlEffect; import androidx.media3.extractor.metadata.mp4.SlowMotionData; import com.google.common.collect.ImmutableList; import java.lang.annotation.Documented; @@ -565,35 +563,34 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; if (inputFormat.pixelWidthHeightRatio != 1f) { return true; } + if (!areVideoEffectsAllNoOp(firstEditedMediaItem.effects.videoEffects, inputFormat)) { + return true; + } + return false; + } - // TODO(b/265927935): consider generalizing this logic. - for (int i = 0; i < firstEditedMediaItem.effects.videoEffects.size(); i++) { - Effect videoEffect = firstEditedMediaItem.effects.videoEffects.get(i); - if (videoEffect instanceof Presentation) { - Presentation presentation = (Presentation) videoEffect; - // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. - int decodedWidth = - (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height; - int decodedHeight = - (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; - Size outputSize = presentation.configure(decodedWidth, decodedHeight); - if (outputSize.getWidth() != decodedWidth || outputSize.getHeight() != decodedHeight) { - return true; - } - } else if (videoEffect instanceof ScaleToFitTransformation) { - ScaleToFitTransformation scaleToFitTransformation = - (ScaleToFitTransformation) videoEffect; - if (scaleToFitTransformation.scaleX != 1f - || scaleToFitTransformation.scaleY != 1f - || scaleToFitTransformation.rotationDegrees != 0f) { - return true; - } - } else { - return true; + /** + * Returns whether all {@code videoEffects} are {@linkplain GlEffect#isNoOp(int, int) no-ops}, + * given an input {@link Format}. + */ + private boolean areVideoEffectsAllNoOp(ImmutableList videoEffects, Format inputFormat) { + int decodedWidth = + (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height; + int decodedHeight = + (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; + for (int i = 0; i < videoEffects.size(); i++) { + Effect videoEffect = videoEffects.get(i); + if (!(videoEffect instanceof GlEffect)) { + // We cannot confirm whether Effect instances that are not GlEffect instances are + // no-ops. + return false; + } + GlEffect glEffect = (GlEffect) videoEffect; + if (!glEffect.isNoOp(decodedWidth, decodedHeight)) { + return false; } } - - return false; + return true; } } }