loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a DefaultDecoderFactory option to configure operating rate

Browse Source 
This has the largest impact during operations with no encoder, such as
frame extraction. Add a matching performance test.

PiperOrigin-RevId: 661220044
This commit is contained in:
dancho 2024-08-09 04:59:32 -07:00 committed by Copybara-Service
parent f37f9690f4
commit 931b0e25f1
4 changed files with 174 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -41,12 +41,16 @@ import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.Util;
import androidx.media3.effect.DefaultGlObjectsProvider;
import androidx.media3.effect.GlEffect;
import androidx.media3.effect.GlShaderProgram;
import androidx.media3.effect.PassthroughShaderProgram;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.test.utils.BitmapPixelTestUtil;
@ -58,6 +62,7 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.json.JSONException;
import org.json.JSONObject;
@ -998,6 +1003,27 @@ public final class AndroidTestUtil {
return bitmaps.build();
}
/**
* Creates a {@link GlEffect} that counts the number of frames processed in {@code frameCount}.
*/
public static GlEffect createFrameCountingEffect(AtomicInteger frameCount) {
return new GlEffect() {
@Override
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
return new PassthroughShaderProgram() {
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider,
GlTextureInfo inputTexture,
long presentationTimeUs) {
super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs);
frameCount.incrementAndGet();
}
};
}
};
}
/** A customizable forwarding {@link Codec.EncoderFactory} that forces encoding. */
public static final class ForceEncodeEncoderFactory implements Codec.EncoderFactory {

23
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -29,6 +29,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_
import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_270;
import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.assumeFormatsSupported;
import static androidx.media3.transformer.AndroidTestUtil.createFrameCountingEffect;
import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects;
import static androidx.media3.transformer.AndroidTestUtil.generateTextureFromBitmap;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
@ -56,8 +57,6 @@ import android.util.Pair;
import androidx.media3.common.C;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.OnInputFrameProcessedListener;
@ -76,8 +75,6 @@ import androidx.media3.effect.DefaultGlObjectsProvider;
import androidx.media3.effect.DefaultVideoFrameProcessor;
import androidx.media3.effect.FrameCache;
import androidx.media3.effect.GlEffect;
import androidx.media3.effect.GlShaderProgram;
import androidx.media3.effect.PassthroughShaderProgram;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.RgbFilter;
import androidx.media3.effect.ScaleAndRotateTransformation;
@ -1909,24 +1906,6 @@ public class TransformerEndToEndTest {
});
}
private static GlEffect createFrameCountingEffect(AtomicInteger frameCount) {
return new GlEffect() {
@Override
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
return new PassthroughShaderProgram() {
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider,
GlTextureInfo inputTexture,
long presentationTimeUs) {
super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs);
frameCount.incrementAndGet();
}
};
}
};
}
private final class TestTextureAssetLoaderFactory implements AssetLoader.Factory {
private final int width;

101
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeSpeedTest.java
View File

@ -16,21 +16,31 @@
package androidx.media3.transformer.mh;
import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.transformer.AndroidTestUtil.JPG_ULTRA_HDR_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS;
import static androidx.media3.transformer.AndroidTestUtil.assumeFormatsSupported;
import static androidx.media3.transformer.AndroidTestUtil.createFrameCountingEffect;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Util;
import androidx.media3.effect.Presentation;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.AssetLoader;
import androidx.media3.transformer.Codec;
import androidx.media3.transformer.DefaultAssetLoaderFactory;
import androidx.media3.transformer.DefaultDecoderFactory;
import androidx.media3.transformer.EditedMediaItem;
import androidx.media3.transformer.Effects;
import androidx.media3.transformer.ExperimentalAnalyzerModeFactory;
import androidx.media3.transformer.ExportTestResult;
import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner;
@ -38,6 +48,7 @@ import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@ -47,6 +58,7 @@ import org.junit.runner.RunWith;
/** Checks transcoding speed. */
@RunWith(AndroidJUnit4.class)
public class TranscodeSpeedTest {
private final Context context = ApplicationProvider.getApplicationContext();
@Rule public final TestName testName = new TestName();
private String testId;
@ -58,7 +70,6 @@ public class TranscodeSpeedTest {
@Test
public void export1920x1080_to1080p_completesWithAtLeast20Fps() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
assumeFormatsSupported(
context,
testId,
@ -73,7 +84,7 @@ public class TranscodeSpeedTest {
MediaItem.fromUri(Uri.parse(MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS.uri))
.buildUpon()
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder().setEndPositionMs(15_000).build())
new MediaItem.ClippingConfiguration.Builder().setEndPositionMs(45_000).build())
.build();
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setRemoveAudio(true).build();
@ -88,7 +99,6 @@ public class TranscodeSpeedTest {
@Test
public void exportImage_to720p_completesWithHighThroughput() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
Format outputFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
@ -112,7 +122,7 @@ public class TranscodeSpeedTest {
|| Ascii.toLowerCase(Util.MODEL).contains("fold")
|| Ascii.toLowerCase(Util.MODEL).contains("tablet"));
if (Util.SDK_INT == 33 && Ascii.toLowerCase(Util.MODEL).contains("pixel 6")) {
// Pixel 6 is usually quick, unless it's on API 33.
// Pixel 6 is usually quick, unless it's on API 33. See b/358519058.
isHighPerformance = false;
}
// This test uses ULTRA_HDR_URI_STRING because it's high resolution.
@ -140,4 +150,87 @@ public class TranscodeSpeedTest {
// Devices with a fast GPU and encoder will drop under 300 fps.
assertThat(result.throughputFps).isAtLeast(isHighPerformance ? 400 : 20);
}
@Test
public void
analyzeVideo_onHighPerformanceDevice_withConfiguredOperatingRate_completesWithHighThroughput()
throws Exception {
assumeTrue(
Ascii.toLowerCase(Util.MODEL).contains("pixel")
&& (Ascii.toLowerCase(Util.MODEL).contains("6")
|| Ascii.toLowerCase(Util.MODEL).contains("7")
|| Ascii.toLowerCase(Util.MODEL).contains("8")
|| Ascii.toLowerCase(Util.MODEL).contains("fold")
|| Ascii.toLowerCase(Util.MODEL).contains("tablet")));
// Pixel 6 is usually quick, unless it's on API 33. See b/358519058.
assumeFalse(Util.SDK_INT == 33 && Ascii.toLowerCase(Util.MODEL).contains("pixel 6"));
AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0);
ExportTestResult result =
analyzeVideoWithConfiguredOperatingRate(
testId,
Uri.parse(MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS.uri),
/* durationMs= */ 45_000,
videoFramesSeen);
int expectedFrameCount = 1350;
checkState(videoFramesSeen.get() == expectedFrameCount);
float throughputFps = 1000f * videoFramesSeen.get() / result.elapsedTimeMs;
assertThat(throughputFps).isAtLeast(350);
}
@Test
public void analyzeVideo_withConfiguredOperatingRate_completesWithCorrectNumberOfFrames()
throws Exception {
Context context = ApplicationProvider.getApplicationContext();
assumeFormatsSupported(
context,
testId,
/* inputFormat= */ MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS.videoFormat,
/* outputFormat= */ null);
AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0);
analyzeVideoWithConfiguredOperatingRate(
testId,
Uri.parse(MP4_LONG_ASSET_WITH_INCREASING_TIMESTAMPS.uri),
/* durationMs= */ 15_000,
videoFramesSeen);
int expectedFrameCount = 450;
assertThat(videoFramesSeen.get()).isEqualTo(expectedFrameCount);
}
private static ExportTestResult analyzeVideoWithConfiguredOperatingRate(
String testId, Uri mediaUri, long durationMs, AtomicInteger videoFramesSeen)
throws Exception {
Context context = ApplicationProvider.getApplicationContext();
videoFramesSeen.set(0);
Codec.DecoderFactory decoderFactory =
new DefaultDecoderFactory.Builder(context).setShouldConfigureOperatingRate(true).build();
AssetLoader.Factory assetLoaderFactory =
new DefaultAssetLoaderFactory(context, decoderFactory, Clock.DEFAULT);
Transformer transformer =
ExperimentalAnalyzerModeFactory.buildAnalyzer(context)
.buildUpon()
.setAssetLoaderFactory(assetLoaderFactory)
.build();
MediaItem mediaItem =
MediaItem.fromUri(mediaUri)
.buildUpon()
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder().setEndPositionMs(durationMs).build())
.build();
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem)
.setRemoveAudio(true)
.setEffects(
new Effects(
/* audioProcessors= */ ImmutableList.of(),
ImmutableList.of(createFrameCountingEffect(videoFramesSeen))))
.build();
return new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
}
}

50
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java
View File

@ -76,6 +76,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
private Listener listener;
private boolean enableDecoderFallback;
private @C.Priority int codecPriority;
private boolean shouldConfigureOperatingRate;
private MediaCodecSelector mediaCodecSelector;
/** Creates a new {@link Builder}. */
@ -83,6 +84,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
this.context = context.getApplicationContext();
listener = (codecName, codecInitializationExceptions) -> {};
codecPriority = C.PRIORITY_PROCESSING_FOREGROUND;
shouldConfigureOperatingRate = false;
mediaCodecSelector = MediaCodecSelector.DEFAULT;
}
@ -131,6 +133,27 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
return this;
}
/**
* Sets whether a device-specific decoder {@linkplain MediaFormat#KEY_OPERATING_RATE operating
* rate} should be requested.
*
* <p>This is a best-effort hint to the codec. Setting this to {@code true} might improve
* decoding performance.
*
* <p>The effect of this field will be most noticeable when no other {@link MediaCodec}
* instances are in use.
*
* <p>Defaults to {@code false}.
*
* @param shouldConfigureOperatingRate Whether to apply an {@link
* MediaFormat#KEY_OPERATING_RATE} configuration to the decoder.
*/
@CanIgnoreReturnValue
public Builder setShouldConfigureOperatingRate(boolean shouldConfigureOperatingRate) {
this.shouldConfigureOperatingRate = shouldConfigureOperatingRate;
return this;
}
/**
* Sets the {@link MediaCodecSelector} used when selecting a decoder.
*
@ -152,6 +175,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
private final boolean enableDecoderFallback;
private final Listener listener;
private final @C.Priority int codecPriority;
private final boolean shouldConfigureOperatingRate;
private final MediaCodecSelector mediaCodecSelector;
/**
@ -184,6 +208,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
this.enableDecoderFallback = builder.enableDecoderFallback;
this.listener = builder.listener;
this.codecPriority = builder.codecPriority;
this.shouldConfigureOperatingRate = builder.shouldConfigureOperatingRate;
this.mediaCodecSelector = builder.mediaCodecSelector;
}
@ -244,6 +269,10 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
mediaFormat.setInteger(MediaFormat.KEY_IMPORTANCE, max(0, -codecPriority));
}
if (shouldConfigureOperatingRate) {
configureOperatingRate(mediaFormat);
}
return createCodecForMediaFormat(
mediaFormat, format, outputSurface, devicePrefersSoftwareDecoder(format));
}
@ -324,6 +353,27 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
throw codecInitExceptions.get(0);
}
private static void configureOperatingRate(MediaFormat mediaFormat) {
if (Util.SDK_INT < 25) {
// Not setting priority and operating rate achieves better decoding performance.
return;
}
if (deviceNeedsPriorityWorkaround()) {
// Setting KEY_PRIORITY to 1 leads to worse performance on many devices.
mediaFormat.setInteger(MediaFormat.KEY_PRIORITY, 1);
}
// Setting KEY_OPERATING_RATE to Integer.MAX_VALUE leads to slower operation on some devices.
mediaFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, 10000);
}
private static boolean deviceNeedsPriorityWorkaround() {
// On these chipsets, decoder configuration fails if KEY_OPERATING_RATE is set but not
// KEY_PRIORITY. See b/358519863.
return Util.SDK_INT >= 31 && Build.SOC_MODEL.equals("s5e8835");
}
private static boolean deviceNeedsDisable8kWorkaround(Format format) {
// Fixed on API 31+. See http://b/278234847#comment40 for more information.
return SDK_INT < 31