diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 256de9a93c..1ef723df22 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -41,12 +41,16 @@ import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.Util; import androidx.media3.effect.DefaultGlObjectsProvider; +import androidx.media3.effect.GlEffect; +import androidx.media3.effect.GlShaderProgram; +import androidx.media3.effect.PassthroughShaderProgram; import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.test.utils.BitmapPixelTestUtil; @@ -58,6 +62,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.json.JSONException; import org.json.JSONObject; @@ -998,6 +1003,27 @@ public final class AndroidTestUtil { return bitmaps.build(); } + /** + * Creates a {@link GlEffect} that counts the number of frames processed in {@code frameCount}. + */ + public static GlEffect createFrameCountingEffect(AtomicInteger frameCount) { + return new GlEffect() { + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { + return new PassthroughShaderProgram() { + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, + GlTextureInfo inputTexture, + long presentationTimeUs) { + super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs); + frameCount.incrementAndGet(); + } + }; + } + }; + } + /** A customizable forwarding {@link Codec.EncoderFactory} that forces encoding. */ public static final class ForceEncodeEncoderFactory implements Codec.EncoderFactory { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index fd8a25b44c..26ae3087e2 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -29,6 +29,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_ import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_270; import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET; import static androidx.media3.transformer.AndroidTestUtil.assumeFormatsSupported; +import static androidx.media3.transformer.AndroidTestUtil.createFrameCountingEffect; import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects; import static androidx.media3.transformer.AndroidTestUtil.generateTextureFromBitmap; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; @@ -56,8 +57,6 @@ import android.util.Pair; import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.Format; -import androidx.media3.common.GlObjectsProvider; -import androidx.media3.common.GlTextureInfo; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.OnInputFrameProcessedListener; @@ -76,8 +75,6 @@ import androidx.media3.effect.DefaultGlObjectsProvider; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.FrameCache; import androidx.media3.effect.GlEffect; -import androidx.media3.effect.GlShaderProgram; -import androidx.media3.effect.PassthroughShaderProgram; import androidx.media3.effect.Presentation; import androidx.media3.effect.RgbFilter; import androidx.media3.effect.ScaleAndRotateTransformation; @@ -1909,24 +1906,6 @@ public class TransformerEndToEndTest { }); } - private static GlEffect createFrameCountingEffect(AtomicInteger frameCount) { - return new GlEffect() { - @Override - public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { - return new PassthroughShaderProgram() { - @Override - public void queueInputFrame( - GlObjectsProvider glObjectsProvider, - GlTextureInfo inputTexture, - long presentationTimeUs) { - super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs); - frameCount.incrementAndGet(); - } - }; - } - }; - } - private final class TestTextureAssetLoaderFactory implements AssetLoader.Factory { private final int width; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeSpeedTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeSpeedTest.java index 28f87f0237..45b71f5187 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeSpeedTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeSpeedTest.java @@ -16,21 +16,31 @@ package androidx.media3.transformer.mh; import static androidx.media3.common.MimeTypes.VIDEO_H264; +import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.transformer.AndroidTestUtil.JPG_ULTRA_HDR_ASSET; import static androidx.media3.transformer.AndroidTestUtil.MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS; import static androidx.media3.transformer.AndroidTestUtil.assumeFormatsSupported; +import static androidx.media3.transformer.AndroidTestUtil.createFrameCountingEffect; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; import android.content.Context; import android.net.Uri; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.Clock; import androidx.media3.common.util.Util; import androidx.media3.effect.Presentation; import androidx.media3.transformer.AndroidTestUtil; +import androidx.media3.transformer.AssetLoader; +import androidx.media3.transformer.Codec; +import androidx.media3.transformer.DefaultAssetLoaderFactory; +import androidx.media3.transformer.DefaultDecoderFactory; import androidx.media3.transformer.EditedMediaItem; import androidx.media3.transformer.Effects; +import androidx.media3.transformer.ExperimentalAnalyzerModeFactory; import androidx.media3.transformer.ExportTestResult; import androidx.media3.transformer.Transformer; import androidx.media3.transformer.TransformerAndroidTestRunner; @@ -38,6 +48,7 @@ import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -47,6 +58,7 @@ import org.junit.runner.RunWith; /** Checks transcoding speed. */ @RunWith(AndroidJUnit4.class) public class TranscodeSpeedTest { + private final Context context = ApplicationProvider.getApplicationContext(); @Rule public final TestName testName = new TestName(); private String testId; @@ -58,7 +70,6 @@ public class TranscodeSpeedTest { @Test public void export1920x1080_to1080p_completesWithAtLeast20Fps() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); assumeFormatsSupported( context, testId, @@ -73,7 +84,7 @@ public class TranscodeSpeedTest { MediaItem.fromUri(Uri.parse(MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS.uri)) .buildUpon() .setClippingConfiguration( - new MediaItem.ClippingConfiguration.Builder().setEndPositionMs(15_000).build()) + new MediaItem.ClippingConfiguration.Builder().setEndPositionMs(45_000).build()) .build(); EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).setRemoveAudio(true).build(); @@ -88,7 +99,6 @@ public class TranscodeSpeedTest { @Test public void exportImage_to720p_completesWithHighThroughput() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); Format outputFormat = new Format.Builder() .setSampleMimeType(VIDEO_H264) @@ -112,7 +122,7 @@ public class TranscodeSpeedTest { || Ascii.toLowerCase(Util.MODEL).contains("fold") || Ascii.toLowerCase(Util.MODEL).contains("tablet")); if (Util.SDK_INT == 33 && Ascii.toLowerCase(Util.MODEL).contains("pixel 6")) { - // Pixel 6 is usually quick, unless it's on API 33. + // Pixel 6 is usually quick, unless it's on API 33. See b/358519058. isHighPerformance = false; } // This test uses ULTRA_HDR_URI_STRING because it's high resolution. @@ -140,4 +150,87 @@ public class TranscodeSpeedTest { // Devices with a fast GPU and encoder will drop under 300 fps. assertThat(result.throughputFps).isAtLeast(isHighPerformance ? 400 : 20); } + + @Test + public void + analyzeVideo_onHighPerformanceDevice_withConfiguredOperatingRate_completesWithHighThroughput() + throws Exception { + assumeTrue( + Ascii.toLowerCase(Util.MODEL).contains("pixel") + && (Ascii.toLowerCase(Util.MODEL).contains("6") + || Ascii.toLowerCase(Util.MODEL).contains("7") + || Ascii.toLowerCase(Util.MODEL).contains("8") + || Ascii.toLowerCase(Util.MODEL).contains("fold") + || Ascii.toLowerCase(Util.MODEL).contains("tablet"))); + // Pixel 6 is usually quick, unless it's on API 33. See b/358519058. + assumeFalse(Util.SDK_INT == 33 && Ascii.toLowerCase(Util.MODEL).contains("pixel 6")); + AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0); + + ExportTestResult result = + analyzeVideoWithConfiguredOperatingRate( + testId, + Uri.parse(MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS.uri), + /* durationMs= */ 45_000, + videoFramesSeen); + int expectedFrameCount = 1350; + checkState(videoFramesSeen.get() == expectedFrameCount); + + float throughputFps = 1000f * videoFramesSeen.get() / result.elapsedTimeMs; + assertThat(throughputFps).isAtLeast(350); + } + + @Test + public void analyzeVideo_withConfiguredOperatingRate_completesWithCorrectNumberOfFrames() + throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS.videoFormat, + /* outputFormat= */ null); + AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0); + + analyzeVideoWithConfiguredOperatingRate( + testId, + Uri.parse(MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS.uri), + /* durationMs= */ 15_000, + videoFramesSeen); + int expectedFrameCount = 450; + + assertThat(videoFramesSeen.get()).isEqualTo(expectedFrameCount); + } + + private static ExportTestResult analyzeVideoWithConfiguredOperatingRate( + String testId, Uri mediaUri, long durationMs, AtomicInteger videoFramesSeen) + throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + videoFramesSeen.set(0); + Codec.DecoderFactory decoderFactory = + new DefaultDecoderFactory.Builder(context).setShouldConfigureOperatingRate(true).build(); + AssetLoader.Factory assetLoaderFactory = + new DefaultAssetLoaderFactory(context, decoderFactory, Clock.DEFAULT); + Transformer transformer = + ExperimentalAnalyzerModeFactory.buildAnalyzer(context) + .buildUpon() + .setAssetLoaderFactory(assetLoaderFactory) + .build(); + MediaItem mediaItem = + MediaItem.fromUri(mediaUri) + .buildUpon() + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder().setEndPositionMs(durationMs).build()) + .build(); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(mediaItem) + .setRemoveAudio(true) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(), + ImmutableList.of(createFrameCountingEffect(videoFramesSeen)))) + .build(); + + return new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java index c34ef22eaf..06a8c75ba3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java @@ -76,6 +76,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { private Listener listener; private boolean enableDecoderFallback; private @C.Priority int codecPriority; + private boolean shouldConfigureOperatingRate; private MediaCodecSelector mediaCodecSelector; /** Creates a new {@link Builder}. */ @@ -83,6 +84,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { this.context = context.getApplicationContext(); listener = (codecName, codecInitializationExceptions) -> {}; codecPriority = C.PRIORITY_PROCESSING_FOREGROUND; + shouldConfigureOperatingRate = false; mediaCodecSelector = MediaCodecSelector.DEFAULT; } @@ -131,6 +133,27 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { return this; } + /** + * Sets whether a device-specific decoder {@linkplain MediaFormat#KEY_OPERATING_RATE operating + * rate} should be requested. + * + *
This is a best-effort hint to the codec. Setting this to {@code true} might improve + * decoding performance. + * + *
The effect of this field will be most noticeable when no other {@link MediaCodec} + * instances are in use. + * + *
Defaults to {@code false}. + * + * @param shouldConfigureOperatingRate Whether to apply an {@link + * MediaFormat#KEY_OPERATING_RATE} configuration to the decoder. + */ + @CanIgnoreReturnValue + public Builder setShouldConfigureOperatingRate(boolean shouldConfigureOperatingRate) { + this.shouldConfigureOperatingRate = shouldConfigureOperatingRate; + return this; + } + /** * Sets the {@link MediaCodecSelector} used when selecting a decoder. * @@ -152,6 +175,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { private final boolean enableDecoderFallback; private final Listener listener; private final @C.Priority int codecPriority; + private final boolean shouldConfigureOperatingRate; private final MediaCodecSelector mediaCodecSelector; /** @@ -184,6 +208,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { this.enableDecoderFallback = builder.enableDecoderFallback; this.listener = builder.listener; this.codecPriority = builder.codecPriority; + this.shouldConfigureOperatingRate = builder.shouldConfigureOperatingRate; this.mediaCodecSelector = builder.mediaCodecSelector; } @@ -244,6 +269,10 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { mediaFormat.setInteger(MediaFormat.KEY_IMPORTANCE, max(0, -codecPriority)); } + if (shouldConfigureOperatingRate) { + configureOperatingRate(mediaFormat); + } + return createCodecForMediaFormat( mediaFormat, format, outputSurface, devicePrefersSoftwareDecoder(format)); } @@ -324,6 +353,27 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { throw codecInitExceptions.get(0); } + private static void configureOperatingRate(MediaFormat mediaFormat) { + if (Util.SDK_INT < 25) { + // Not setting priority and operating rate achieves better decoding performance. + return; + } + + if (deviceNeedsPriorityWorkaround()) { + // Setting KEY_PRIORITY to 1 leads to worse performance on many devices. + mediaFormat.setInteger(MediaFormat.KEY_PRIORITY, 1); + } + + // Setting KEY_OPERATING_RATE to Integer.MAX_VALUE leads to slower operation on some devices. + mediaFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, 10000); + } + + private static boolean deviceNeedsPriorityWorkaround() { + // On these chipsets, decoder configuration fails if KEY_OPERATING_RATE is set but not + // KEY_PRIORITY. See b/358519863. + return Util.SDK_INT >= 31 && Build.SOC_MODEL.equals("s5e8835"); + } + private static boolean deviceNeedsDisable8kWorkaround(Format format) { // Fixed on API 31+. See http://b/278234847#comment40 for more information. return SDK_INT < 31