loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Effect: Create GlEffect.isNoOp to simplify Transformer.

Browse Source 
This will also allow us to skip transcodes in the future for other effects, like
cropping, color effects, etc.

PiperOrigin-RevId: 507765618
This commit is contained in:
huangdarwin 2023-02-07 14:43:44 +00:00 committed by microkatz
parent c434cc0c9f
commit 0c0c972ea0
8 changed files with 89 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libraries/effect/src/main/java/androidx/media3/effect/Crop.java
View File

@ -93,4 +93,12 @@ public final class Crop implements MatrixTransformation {
public Matrix getMatrix(long presentationTimeUs) {
return checkStateNotNull(transformationMatrix, "configure must be called first");
}
@Override
public boolean isNoOp(int inputWidth, int inputHeight) {
Size outputSize = configure(inputWidth, inputHeight);
return checkStateNotNull(transformationMatrix).isIdentity()
&& inputWidth == outputSize.getWidth()
&& inputHeight == outputSize.getHeight();
}
}

15
libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java
View File

@ -40,4 +40,19 @@ public interface GlEffect extends Effect {
*/
GlShaderProgram toGlShaderProgram(Context context, boolean useHdr)
throws FrameProcessingException;
/**
* Returns whether a {@link GlEffect} applies no change at every timestamp.
*
* <p>This can be used as a hint to skip this instance.
*
* @param inputWidth The input frame width, in pixels.
* @param inputHeight The input frame height, in pixels.
*/
default boolean isNoOp(int inputWidth, int inputHeight) {
// TODO(b/265927935): Generalize this logic by implementing this method on all
// subclasses, and deleting the default implementation here. Otherwise, some no-op effects may
// not be properly detected or handled.
return false;
}
}

8
libraries/effect/src/main/java/androidx/media3/effect/Presentation.java
View File

@ -214,6 +214,14 @@ public final class Presentation implements MatrixTransformation {
return checkStateNotNull(transformationMatrix, "configure must be called first");
}
@Override
public boolean isNoOp(int inputWidth, int inputHeight) {
configure(inputWidth, inputHeight);
return checkStateNotNull(transformationMatrix).isIdentity()
&& inputWidth == Math.round(outputWidth)
&& inputHeight == Math.round(outputHeight);
}
@RequiresNonNull("transformationMatrix")
private void applyAspectRatio() {
float inputAspectRatio = outputWidth / outputHeight;

18
libraries/effect/src/main/java/androidx/media3/effect/ScaleToFitTransformation.java
View File

@ -92,13 +92,6 @@ public final class ScaleToFitTransformation implements MatrixTransformation {
}
}
/** The multiplier by which the frame will scale horizontally, along the x-axis. */
public final float scaleX;
/** The multiplier by which the frame will scale vertically, along the y-axis. */
public final float scaleY;
/** How much to rotate the frame counterclockwise, in degrees. */
public final float rotationDegrees;
private final Matrix transformationMatrix;
private @MonotonicNonNull Matrix adjustedTransformationMatrix;
@ -110,9 +103,6 @@ public final class ScaleToFitTransformation implements MatrixTransformation {
* @param rotationDegrees How much to rotate the frame counterclockwise, in degrees.
*/
private ScaleToFitTransformation(float scaleX, float scaleY, float rotationDegrees) {
this.scaleX = scaleX;
this.scaleY = scaleY;
this.rotationDegrees = rotationDegrees;
transformationMatrix = new Matrix();
transformationMatrix.postScale(scaleX, scaleY);
transformationMatrix.postRotate(rotationDegrees);
@ -162,4 +152,12 @@ public final class ScaleToFitTransformation implements MatrixTransformation {
public Matrix getMatrix(long presentationTimeUs) {
return checkStateNotNull(adjustedTransformationMatrix, "configure must be called first");
}
@Override
public boolean isNoOp(int inputWidth, int inputHeight) {
Size outputSize = configure(inputWidth, inputHeight);
return checkStateNotNull(adjustedTransformationMatrix).isIdentity()
&& inputWidth == outputSize.getWidth()
&& inputHeight == outputSize.getHeight();
}
}

4
libraries/effect/src/test/java/androidx/media3/effect/CropTest.java
View File

@ -37,7 +37,9 @@ public final class CropTest {
Crop crop = new Crop(/* left= */ -1, /* right= */ 1, /* bottom= */ -1, /* top= */ 1);
Size outputSize = crop.configure(inputWidth, inputHeight);
boolean isNoOp = crop.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@ -53,7 +55,9 @@ public final class CropTest {
Crop crop = new Crop(left, right, bottom, top);
Size outputSize = crop.configure(inputWidth, inputHeight);
boolean isNoOp = crop.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
assertThat(outputSize.getWidth()).isEqualTo(expectedPostCropWidth);

8
libraries/effect/src/test/java/androidx/media3/effect/PresentationTest.java
View File

@ -37,7 +37,9 @@ public final class PresentationTest {
Presentation presentation = Presentation.createForHeight(C.LENGTH_UNSET);
Size outputSize = presentation.configure(inputWidth, inputHeight);
boolean isNoOp = presentation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@ -50,7 +52,9 @@ public final class PresentationTest {
Presentation presentation = Presentation.createForHeight(requestedHeight);
Size outputSize = presentation.configure(inputWidth, inputHeight);
boolean isNoOp = presentation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
}
@ -64,7 +68,9 @@ public final class PresentationTest {
Presentation.createForAspectRatio(aspectRatio, Presentation.LAYOUT_SCALE_TO_FIT);
Size outputSize = presentation.configure(inputWidth, inputHeight);
boolean isNoOp = presentation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * inputHeight));
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@ -80,7 +86,9 @@ public final class PresentationTest {
requestedWidth, requestedHeight, Presentation.LAYOUT_SCALE_TO_FIT);
Size outputSize = presentation.configure(inputWidth, inputHeight);
boolean isNoOp = presentation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(requestedWidth);
assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
}

12
libraries/effect/src/test/java/androidx/media3/effect/ScaleToFitTransformationTest.java
View File

@ -39,7 +39,9 @@ public final class ScaleToFitTransformationTest {
new ScaleToFitTransformation.Builder().build();
Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight);
boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@ -54,7 +56,9 @@ public final class ScaleToFitTransformationTest {
.build();
Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight);
boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f));
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@ -67,7 +71,9 @@ public final class ScaleToFitTransformationTest {
new ScaleToFitTransformation.Builder().setScale(/* scaleX= */ 2f, /* scaleY= */ 1f).build();
Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight);
boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@ -80,7 +86,9 @@ public final class ScaleToFitTransformationTest {
new ScaleToFitTransformation.Builder().setScale(/* scaleX= */ 1f, /* scaleY= */ 2f).build();
Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight);
boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2);
}
@ -93,7 +101,9 @@ public final class ScaleToFitTransformationTest {
new ScaleToFitTransformation.Builder().setRotationDegrees(90).build();
Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight);
boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
}
@ -107,7 +117,9 @@ public final class ScaleToFitTransformationTest {
long expectedOutputWidthHeight = 247;
Size outputSize = scaleToFitTransformation.configure(inputWidth, inputHeight);
boolean isNoOp = scaleToFitTransformation.isNoOp(inputWidth, inputHeight);
assertThat(isNoOp).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight);
assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight);
}

47
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -39,9 +39,7 @@ import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.ConditionVariable;
import androidx.media3.common.util.HandlerWrapper;
import androidx.media3.common.util.Size;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.ScaleToFitTransformation;
import androidx.media3.effect.GlEffect;
import androidx.media3.extractor.metadata.mp4.SlowMotionData;
import com.google.common.collect.ImmutableList;
import java.lang.annotation.Documented;
@ -565,35 +563,34 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (inputFormat.pixelWidthHeightRatio != 1f) {
return true;
}
if (!areVideoEffectsAllNoOp(firstEditedMediaItem.effects.videoEffects, inputFormat)) {
return true;
}
return false;
}
// TODO(b/265927935): consider generalizing this logic.
for (int i = 0; i < firstEditedMediaItem.effects.videoEffects.size(); i++) {
Effect videoEffect = firstEditedMediaItem.effects.videoEffects.get(i);
if (videoEffect instanceof Presentation) {
Presentation presentation = (Presentation) videoEffect;
// The decoder rotates encoded frames for display by inputFormat.rotationDegrees.
/**
* Returns whether all {@code videoEffects} are {@linkplain GlEffect#isNoOp(int, int) no-ops},
* given an input {@link Format}.
*/
private boolean areVideoEffectsAllNoOp(ImmutableList<Effect> videoEffects, Format inputFormat) {
int decodedWidth =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
Size outputSize = presentation.configure(decodedWidth, decodedHeight);
if (outputSize.getWidth() != decodedWidth || outputSize.getHeight() != decodedHeight) {
return true;
for (int i = 0; i < videoEffects.size(); i++) {
Effect videoEffect = videoEffects.get(i);
if (!(videoEffect instanceof GlEffect)) {
// We cannot confirm whether Effect instances that are not GlEffect instances are
// no-ops.
return false;
}
} else if (videoEffect instanceof ScaleToFitTransformation) {
ScaleToFitTransformation scaleToFitTransformation =
(ScaleToFitTransformation) videoEffect;
if (scaleToFitTransformation.scaleX != 1f
|| scaleToFitTransformation.scaleY != 1f
|| scaleToFitTransformation.rotationDegrees != 0f) {
return true;
}
} else {
return true;
}
}
GlEffect glEffect = (GlEffect) videoEffect;
if (!glEffect.isNoOp(decodedWidth, decodedHeight)) {
return false;
}
}
return true;
}
}
}