loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Transformer: Skip SSIM in tests when fallback is applied.

Browse Source 
SSIM calculation requires the input and output dimensions to be identical.

For devices that can't encode the input dimensions, skip SSIM calculations and
log the cause. Only apply this on tests where the encoder may not support the
input file dimensions.

PiperOrigin-RevId: 451364904
This commit is contained in:
huangdarwin 2022-05-27 11:48:44 +00:00 committed by Marc Baechinger
parent fe3831c5b4
commit 099a542dac
7 changed files with 62 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -229,7 +229,9 @@ public final class AndroidTestUtil {
/**
* Checks whether the test should be skipped because the device is incapable of decoding and
* encoding the given formats. If the test should be skipped, logs the reason for skipping.
* encoding the given formats.
*
* <p>If the test should be skipped, logs the reason for skipping.
*
* @param context The {@link Context context}.
* @param testId The test ID.

2
libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java
View File

@ -68,7 +68,7 @@ public final class SsimHelper {
* <p>The method compares every {@link #DEFAULT_COMPARISON_INTERVAL n-th} frame from both videos.
*
* @param context The {@link Context}.
* @param referenceVideoPath The path to the reference video file, must be in {@link
* @param referenceVideoPath The path to the reference video file, which must be in {@linkplain
* Context#getAssets() Assets}.
* @param actualVideoPath The path to the actual video file.
* @throws IOException When unable to open the provided video paths.

68
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java
View File

@ -33,6 +33,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.compatqual.NullableType;
import org.json.JSONException;
@ -49,7 +50,7 @@ public class TransformerAndroidTestRunner {
public static class Builder {
private final Context context;
private final Transformer transformer;
private boolean calculateSsim;
private boolean maybeCalculateSsim;
private int timeoutSeconds;
private boolean suppressAnalysisExceptions;
@Nullable private Map<String, Object> inputValues;
@ -81,19 +82,21 @@ public class TransformerAndroidTestRunner {
}
/**
* Sets whether to calculate the SSIM of the transformation output.
* Sets whether to try to calculate the SSIM of the transformation output.
*
* <p>SSIM requires the input and output video dimensions to match. Therefore, if encoder
* resolution fallback occurs, this calculation is skipped.
*
* <p>The calculation involves decoding and comparing both the input and the output video.
* Consequently this calculation is not cost-free. Requires the input and output video to be the
* same size.
* Consequently this calculation is not cost-free.
*
* <p>The default value is {@code false}.
*
* @param calculateSsim Whether to calculate SSIM.
* @param maybeCalculateSsim Whether to try to calculate SSIM.
* @return This {@link Builder}.
*/
public Builder setCalculateSsim(boolean calculateSsim) {
this.calculateSsim = calculateSsim;
public Builder setMaybeCalculateSsim(boolean maybeCalculateSsim) {
this.maybeCalculateSsim = maybeCalculateSsim;
return this;
}
@ -137,7 +140,7 @@ public class TransformerAndroidTestRunner {
context,
transformer,
timeoutSeconds,
calculateSsim,
maybeCalculateSsim,
suppressAnalysisExceptions,
inputValues);
}
@ -147,7 +150,7 @@ public class TransformerAndroidTestRunner {
private final CodecNameForwardingCodecFactory transformerCodecFactory;
private final Transformer transformer;
private final int timeoutSeconds;
private final boolean calculateSsim;
private final boolean maybeCalculateSsim;
private final boolean suppressAnalysisExceptions;
@Nullable private final Map<String, Object> inputValues;
@ -155,7 +158,7 @@ public class TransformerAndroidTestRunner {
Context context,
Transformer transformer,
int timeoutSeconds,
boolean calculateSsim,
boolean maybeCalculateSsim,
boolean suppressAnalysisExceptions,
@Nullable Map<String, Object> inputValues) {
this.context = context;
@ -168,7 +171,7 @@ public class TransformerAndroidTestRunner {
.setEncoderFactory(transformerCodecFactory)
.build();
this.timeoutSeconds = timeoutSeconds;
this.calculateSsim = calculateSsim;
this.maybeCalculateSsim = maybeCalculateSsim;
this.suppressAnalysisExceptions = suppressAnalysisExceptions;
this.inputValues = inputValues;
}
@ -220,7 +223,7 @@ public class TransformerAndroidTestRunner {
private TransformationTestResult runInternal(String testId, MediaItem mediaItem)
throws InterruptedException, IOException, TimeoutException, TransformationException {
if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET)
&& calculateSsim) {
&& maybeCalculateSsim) {
throw new UnsupportedOperationException(
"SSIM calculation is not supported for clipped inputs.");
}
@ -231,6 +234,7 @@ public class TransformerAndroidTestRunner {
AtomicReference<@NullableType TransformationResult> transformationResultReference =
new AtomicReference<>();
CountDownLatch countDownLatch = new CountDownLatch(1);
AtomicBoolean fallbackResolutionApplied = new AtomicBoolean(false);
long startTimeMs = SystemClock.DEFAULT.elapsedRealtime();
Transformer testTransformer =
@ -251,6 +255,20 @@ public class TransformerAndroidTestRunner {
transformationExceptionReference.set(exception);
countDownLatch.countDown();
}
@Override
public void onFallbackApplied(
MediaItem inputMediaItem,
TransformationRequest originalTransformationRequest,
TransformationRequest fallbackTransformationRequest) {
// Note: As TransformationRequest only reports the output height but not the
// output width, it's not possible to check whether the encoder has changed
// the output aspect ratio.
if (originalTransformationRequest.outputHeight
!= fallbackTransformationRequest.outputHeight) {
fallbackResolutionApplied.set(true);
}
}
})
.build();
@ -299,15 +317,23 @@ public class TransformerAndroidTestRunner {
.setFilePath(outputVideoFile.getPath())
.setElapsedTimeMs(elapsedTimeMs);
if (!maybeCalculateSsim) {
return resultBuilder.build();
}
if (fallbackResolutionApplied.get()) {
Log.i(
TAG,
testId
+ ": Skipping SSIM calculation because an encoder resolution fallback was applied.");
return resultBuilder.build();
}
try {
if (calculateSsim) {
double ssim =
SsimHelper.calculate(
context,
/* referenceVideoPath= */ checkNotNull(mediaItem.localConfiguration).uri.toString(),
outputVideoFile.getPath());
resultBuilder.setSsim(ssim);
}
double ssim =
SsimHelper.calculate(
context,
/* referenceVideoPath= */ checkNotNull(mediaItem.localConfiguration).uri.toString(),
outputVideoFile.getPath());
resultBuilder.setSsim(ssim);
} catch (InterruptedException interruptedException) {
// InterruptedException is a special unexpected case because it is not related to Ssim
// calculation, so it should be thrown, rather than processed as part of the
@ -317,7 +343,7 @@ public class TransformerAndroidTestRunner {
if (Util.SDK_INT == 21 && "Nexus 5".equals(Util.MODEL)) { // b/233584640
Log.i(TAG, testId + ": Skipping SSIM calculation due to known device-specific issue");
} else {
// Catch all (checked and unchecked) failures throw by the SsimHelper and process them as
// Catch all (checked and unchecked) failures thrown by the SsimHelper and process them as
// part of the TransformationTestResult.
Exception analysisException =
analysisFailure instanceof Exception

6
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java
View File

@ -58,7 +58,7 @@ public final class TranscodeQualityTest {
TransformationTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.setMaybeCalculateSsim(true)
.build()
.run(
testId,
@ -95,7 +95,7 @@ public final class TranscodeQualityTest {
TransformationTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.setMaybeCalculateSsim(true)
.build()
.run(
testId,
@ -125,7 +125,7 @@ public final class TranscodeQualityTest {
TransformationTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.setMaybeCalculateSsim(true)
.build()
.run(
testId,

8
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java
View File

@ -54,7 +54,7 @@ public class TransformationTest {
.setEncoderFactory(AndroidTestUtil.FORCE_ENCODE_ENCODER_FACTORY)
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.setMaybeCalculateSsim(true)
.build()
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
}
@ -84,7 +84,7 @@ public class TransformationTest {
/* enableFallback= */ true))
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.setMaybeCalculateSsim(true)
.build()
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
}
@ -105,7 +105,7 @@ public class TransformationTest {
Transformer transformer =
new Transformer.Builder(context).setEncoderFactory(FORCE_ENCODE_ENCODER_FACTORY).build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.setMaybeCalculateSsim(true)
.setTimeoutSeconds(180)
.build()
.run(testId, MediaItem.fromUri(Uri.parse(MP4_REMOTE_4K60_PORTRAIT_URI_STRING)));
@ -121,7 +121,7 @@ public class TransformationTest {
.setRemoveAudio(true)
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.setMaybeCalculateSsim(true)
.build()
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
}

2
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java
View File

@ -116,7 +116,7 @@ public class BitrateAnalysisTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.setInputValues(inputValues)
.setCalculateSsim(true)
.setMaybeCalculateSsim(true)
.build()
.run(testId, MediaItem.fromUri(Uri.parse(fileUri)));
}

4
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -513,7 +513,9 @@ public final class Transformer {
* @param inputMediaItem The {@link MediaItem} for which the transformation is requested.
* @param originalTransformationRequest The unsupported {@link TransformationRequest} used when
* building {@link Transformer}.
* @param fallbackTransformationRequest The alternative {@link TransformationRequest}.
* @param fallbackTransformationRequest The alternative {@link TransformationRequest}, with
* supported {@link TransformationRequest#outputHeight} and {@link
* TransformationRequest#videoMimeType} values set.
*/
default void onFallbackApplied(
MediaItem inputMediaItem,