From 099a542dacdbcc9d73aa473375b28f87b7c54866 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Fri, 27 May 2022 11:48:44 +0000 Subject: [PATCH] Transformer: Skip SSIM in tests when fallback is applied. SSIM calculation requires the input and output dimensions to be identical. For devices that can't encode the input dimensions, skip SSIM calculations and log the cause. Only apply this on tests where the encoder may not support the input file dimensions. PiperOrigin-RevId: 451364904 --- .../media3/transformer/AndroidTestUtil.java | 4 +- .../media3/transformer/SsimHelper.java | 2 +- .../TransformerAndroidTestRunner.java | 68 +++++++++++++------ .../transformer/mh/TranscodeQualityTest.java | 6 +- .../transformer/mh/TransformationTest.java | 8 +-- .../mh/analysis/BitrateAnalysisTest.java | 2 +- .../media3/transformer/Transformer.java | 4 +- 7 files changed, 62 insertions(+), 32 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 8fb3ae288c..51c20b6525 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -229,7 +229,9 @@ public final class AndroidTestUtil { /** * Checks whether the test should be skipped because the device is incapable of decoding and - * encoding the given formats. If the test should be skipped, logs the reason for skipping. + * encoding the given formats. + * + *

If the test should be skipped, logs the reason for skipping. * * @param context The {@link Context context}. * @param testId The test ID. diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java index 8d9bb05924..b504e28ac5 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java @@ -68,7 +68,7 @@ public final class SsimHelper { *

The method compares every {@link #DEFAULT_COMPARISON_INTERVAL n-th} frame from both videos. * * @param context The {@link Context}. - * @param referenceVideoPath The path to the reference video file, must be in {@link + * @param referenceVideoPath The path to the reference video file, which must be in {@linkplain * Context#getAssets() Assets}. * @param actualVideoPath The path to the actual video file. * @throws IOException When unable to open the provided video paths. diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java index 286c858e5b..1c162647ca 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.compatqual.NullableType; import org.json.JSONException; @@ -49,7 +50,7 @@ public class TransformerAndroidTestRunner { public static class Builder { private final Context context; private final Transformer transformer; - private boolean calculateSsim; + private boolean maybeCalculateSsim; private int timeoutSeconds; private boolean suppressAnalysisExceptions; @Nullable private Map inputValues; @@ -81,19 +82,21 @@ public class TransformerAndroidTestRunner { } /** - * Sets whether to calculate the SSIM of the transformation output. + * Sets whether to try to calculate the SSIM of the transformation output. + * + *

SSIM requires the input and output video dimensions to match. Therefore, if encoder + * resolution fallback occurs, this calculation is skipped. * *

The calculation involves decoding and comparing both the input and the output video. - * Consequently this calculation is not cost-free. Requires the input and output video to be the - * same size. + * Consequently this calculation is not cost-free. * *

The default value is {@code false}. * - * @param calculateSsim Whether to calculate SSIM. + * @param maybeCalculateSsim Whether to try to calculate SSIM. * @return This {@link Builder}. */ - public Builder setCalculateSsim(boolean calculateSsim) { - this.calculateSsim = calculateSsim; + public Builder setMaybeCalculateSsim(boolean maybeCalculateSsim) { + this.maybeCalculateSsim = maybeCalculateSsim; return this; } @@ -137,7 +140,7 @@ public class TransformerAndroidTestRunner { context, transformer, timeoutSeconds, - calculateSsim, + maybeCalculateSsim, suppressAnalysisExceptions, inputValues); } @@ -147,7 +150,7 @@ public class TransformerAndroidTestRunner { private final CodecNameForwardingCodecFactory transformerCodecFactory; private final Transformer transformer; private final int timeoutSeconds; - private final boolean calculateSsim; + private final boolean maybeCalculateSsim; private final boolean suppressAnalysisExceptions; @Nullable private final Map inputValues; @@ -155,7 +158,7 @@ public class TransformerAndroidTestRunner { Context context, Transformer transformer, int timeoutSeconds, - boolean calculateSsim, + boolean maybeCalculateSsim, boolean suppressAnalysisExceptions, @Nullable Map inputValues) { this.context = context; @@ -168,7 +171,7 @@ public class TransformerAndroidTestRunner { .setEncoderFactory(transformerCodecFactory) .build(); this.timeoutSeconds = timeoutSeconds; - this.calculateSsim = calculateSsim; + this.maybeCalculateSsim = maybeCalculateSsim; this.suppressAnalysisExceptions = suppressAnalysisExceptions; this.inputValues = inputValues; } @@ -220,7 +223,7 @@ public class TransformerAndroidTestRunner { private TransformationTestResult runInternal(String testId, MediaItem mediaItem) throws InterruptedException, IOException, TimeoutException, TransformationException { if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET) - && calculateSsim) { + && maybeCalculateSsim) { throw new UnsupportedOperationException( "SSIM calculation is not supported for clipped inputs."); } @@ -231,6 +234,7 @@ public class TransformerAndroidTestRunner { AtomicReference<@NullableType TransformationResult> transformationResultReference = new AtomicReference<>(); CountDownLatch countDownLatch = new CountDownLatch(1); + AtomicBoolean fallbackResolutionApplied = new AtomicBoolean(false); long startTimeMs = SystemClock.DEFAULT.elapsedRealtime(); Transformer testTransformer = @@ -251,6 +255,20 @@ public class TransformerAndroidTestRunner { transformationExceptionReference.set(exception); countDownLatch.countDown(); } + + @Override + public void onFallbackApplied( + MediaItem inputMediaItem, + TransformationRequest originalTransformationRequest, + TransformationRequest fallbackTransformationRequest) { + // Note: As TransformationRequest only reports the output height but not the + // output width, it's not possible to check whether the encoder has changed + // the output aspect ratio. + if (originalTransformationRequest.outputHeight + != fallbackTransformationRequest.outputHeight) { + fallbackResolutionApplied.set(true); + } + } }) .build(); @@ -299,15 +317,23 @@ public class TransformerAndroidTestRunner { .setFilePath(outputVideoFile.getPath()) .setElapsedTimeMs(elapsedTimeMs); + if (!maybeCalculateSsim) { + return resultBuilder.build(); + } + if (fallbackResolutionApplied.get()) { + Log.i( + TAG, + testId + + ": Skipping SSIM calculation because an encoder resolution fallback was applied."); + return resultBuilder.build(); + } try { - if (calculateSsim) { - double ssim = - SsimHelper.calculate( - context, - /* referenceVideoPath= */ checkNotNull(mediaItem.localConfiguration).uri.toString(), - outputVideoFile.getPath()); - resultBuilder.setSsim(ssim); - } + double ssim = + SsimHelper.calculate( + context, + /* referenceVideoPath= */ checkNotNull(mediaItem.localConfiguration).uri.toString(), + outputVideoFile.getPath()); + resultBuilder.setSsim(ssim); } catch (InterruptedException interruptedException) { // InterruptedException is a special unexpected case because it is not related to Ssim // calculation, so it should be thrown, rather than processed as part of the @@ -317,7 +343,7 @@ public class TransformerAndroidTestRunner { if (Util.SDK_INT == 21 && "Nexus 5".equals(Util.MODEL)) { // b/233584640 Log.i(TAG, testId + ": Skipping SSIM calculation due to known device-specific issue"); } else { - // Catch all (checked and unchecked) failures throw by the SsimHelper and process them as + // Catch all (checked and unchecked) failures thrown by the SsimHelper and process them as // part of the TransformationTestResult. Exception analysisException = analysisFailure instanceof Exception diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java index d0e62e3301..3947435c16 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java @@ -58,7 +58,7 @@ public final class TranscodeQualityTest { TransformationTestResult result = new TransformerAndroidTestRunner.Builder(context, transformer) - .setCalculateSsim(true) + .setMaybeCalculateSsim(true) .build() .run( testId, @@ -95,7 +95,7 @@ public final class TranscodeQualityTest { TransformationTestResult result = new TransformerAndroidTestRunner.Builder(context, transformer) - .setCalculateSsim(true) + .setMaybeCalculateSsim(true) .build() .run( testId, @@ -125,7 +125,7 @@ public final class TranscodeQualityTest { TransformationTestResult result = new TransformerAndroidTestRunner.Builder(context, transformer) - .setCalculateSsim(true) + .setMaybeCalculateSsim(true) .build() .run( testId, diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java index bbc9aa29a1..0452abb65c 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java @@ -54,7 +54,7 @@ public class TransformationTest { .setEncoderFactory(AndroidTestUtil.FORCE_ENCODE_ENCODER_FACTORY) .build(); new TransformerAndroidTestRunner.Builder(context, transformer) - .setCalculateSsim(true) + .setMaybeCalculateSsim(true) .build() .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @@ -84,7 +84,7 @@ public class TransformationTest { /* enableFallback= */ true)) .build(); new TransformerAndroidTestRunner.Builder(context, transformer) - .setCalculateSsim(true) + .setMaybeCalculateSsim(true) .build() .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @@ -105,7 +105,7 @@ public class TransformationTest { Transformer transformer = new Transformer.Builder(context).setEncoderFactory(FORCE_ENCODE_ENCODER_FACTORY).build(); new TransformerAndroidTestRunner.Builder(context, transformer) - .setCalculateSsim(true) + .setMaybeCalculateSsim(true) .setTimeoutSeconds(180) .build() .run(testId, MediaItem.fromUri(Uri.parse(MP4_REMOTE_4K60_PORTRAIT_URI_STRING))); @@ -121,7 +121,7 @@ public class TransformationTest { .setRemoveAudio(true) .build(); new TransformerAndroidTestRunner.Builder(context, transformer) - .setCalculateSsim(true) + .setMaybeCalculateSsim(true) .build() .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java index 3d31d43718..cd9ccbfecd 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java @@ -116,7 +116,7 @@ public class BitrateAnalysisTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setInputValues(inputValues) - .setCalculateSsim(true) + .setMaybeCalculateSsim(true) .build() .run(testId, MediaItem.fromUri(Uri.parse(fileUri))); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 41acc4b6a0..0fc6473e64 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -513,7 +513,9 @@ public final class Transformer { * @param inputMediaItem The {@link MediaItem} for which the transformation is requested. * @param originalTransformationRequest The unsupported {@link TransformationRequest} used when * building {@link Transformer}. - * @param fallbackTransformationRequest The alternative {@link TransformationRequest}. + * @param fallbackTransformationRequest The alternative {@link TransformationRequest}, with + * supported {@link TransformationRequest#outputHeight} and {@link + * TransformationRequest#videoMimeType} values set. */ default void onFallbackApplied( MediaItem inputMediaItem,