Add an experimental flag for renderer dynamic scheduling
Use ExoPlayer dynamic scheduling to reduce the render() interval for older API devices where `DefaultCodec.getMaxPendingFrameCount()` is set to 1 in order to prevent frame drops. Controlled via API on DefaultDecoderFactory. Add TransformerForegroundSpeedTest that mimics transcoding while the app is in foreground. PiperOrigin-RevId: 662925764
This commit is contained in:
dancho
Copybara-Service
parent
dbc9f5e0d1
commit
879771ded2
Binary file not shown.
@ -377,6 +377,18 @@ public final class AndroidTestUtil {
|
||||
.build())
|
||||
.build();
|
||||
|
||||
public static final AssetInfo MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS =
|
||||
new AssetInfo.Builder("asset:///media/mp4/long_1080p_lowbitrate.mp4")
|
||||
.setVideoFormat(
|
||||
new Format.Builder()
|
||||
.setSampleMimeType(VIDEO_H264)
|
||||
.setWidth(1920)
|
||||
.setHeight(1080)
|
||||
.setFrameRate(30.00f)
|
||||
.setCodecs("avc1.42C028")
|
||||
.build())
|
||||
.build();
|
||||
|
||||
/** Baseline profile level 3.0 H.264 stream, which should be supported on all devices. */
|
||||
public static final AssetInfo MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S =
|
||||
new AssetInfo.Builder("asset:///media/mp4/sample_with_increasing_timestamps_320w_240h.mp4")
|
||||
|
||||
@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Copyright 2024 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.transformer.mh;
|
||||
|
||||
import static androidx.media3.transformer.AndroidTestUtil.MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS;
|
||||
import static androidx.media3.transformer.AndroidTestUtil.assumeFormatsSupported;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static org.junit.Assume.assumeTrue;
|
||||
|
||||
import android.content.Context;
|
||||
import android.net.Uri;
|
||||
import androidx.media3.common.MediaItem;
|
||||
import androidx.media3.common.MimeTypes;
|
||||
import androidx.media3.common.audio.ChannelMixingAudioProcessor;
|
||||
import androidx.media3.common.audio.ChannelMixingMatrix;
|
||||
import androidx.media3.common.audio.SonicAudioProcessor;
|
||||
import androidx.media3.common.util.Clock;
|
||||
import androidx.media3.common.util.Util;
|
||||
import androidx.media3.effect.Presentation;
|
||||
import androidx.media3.transformer.AndroidTestUtil;
|
||||
import androidx.media3.transformer.AssetLoader;
|
||||
import androidx.media3.transformer.Codec;
|
||||
import androidx.media3.transformer.DefaultAssetLoaderFactory;
|
||||
import androidx.media3.transformer.DefaultDecoderFactory;
|
||||
import androidx.media3.transformer.EditedMediaItem;
|
||||
import androidx.media3.transformer.Effects;
|
||||
import androidx.media3.transformer.ExportTestResult;
|
||||
import androidx.media3.transformer.SurfaceTestActivity;
|
||||
import androidx.media3.transformer.Transformer;
|
||||
import androidx.media3.transformer.TransformerAndroidTestRunner;
|
||||
import androidx.test.core.app.ApplicationProvider;
|
||||
import androidx.test.ext.junit.rules.ActivityScenarioRule;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestName;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/** Checks transcoding speed when running in foreground. */
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public class TranscodeForegroundSpeedTest {
|
||||
private final Context context = ApplicationProvider.getApplicationContext();
|
||||
@Rule public final TestName testName = new TestName();
|
||||
|
||||
// Creating a SurfaceTestActivity rule turns the screen on and puts the test app in foreground.
|
||||
// This affects transcoding performance as foreground apps are more likely to schedule on the
|
||||
// faster CPU cores.
|
||||
@Rule
|
||||
public ActivityScenarioRule<SurfaceTestActivity> rule =
|
||||
new ActivityScenarioRule<>(SurfaceTestActivity.class);
|
||||
|
||||
private String testId;
|
||||
|
||||
@Before
|
||||
public void setUpTestId() {
|
||||
testId = testName.getMethodName();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void
|
||||
export1080pWithAudioTo720p_onMediumPerformanceDeviceWithDynamicScheduling_completesWithAtLeast140Fps()
|
||||
throws Exception {
|
||||
assumeTrue(
|
||||
Ascii.toLowerCase(Util.MODEL).contains("pixel 2")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("dn2103")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("sm-g960f")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("g8441"));
|
||||
assumeFormatsSupported(
|
||||
context,
|
||||
testId,
|
||||
/* inputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat,
|
||||
/* outputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat);
|
||||
|
||||
ExportTestResult exportTestResult =
|
||||
exportVideoAndAudioTo720pWithDynamicScheduling(
|
||||
testId,
|
||||
Uri.parse(MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.uri),
|
||||
/* durationMs= */ 30_000);
|
||||
|
||||
// Running this without dynamic scheduling runs at under 80 fps.
|
||||
assertThat(exportTestResult.throughputFps).isAtLeast(140);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void
|
||||
export1080pWithAudioTo720p_onLowerPerformanceDevicesWithDynamicScheduling_completesWithAtLeast60Fps()
|
||||
throws Exception {
|
||||
assumeTrue(
|
||||
(Ascii.toLowerCase(Util.MODEL).contains("f-01l")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("asus_x00td")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("redmi note 5")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("mha-l29")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("oneplus a6013")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("cph1803")
|
||||
|| Ascii.toLowerCase(Util.MODEL).contains("mi a2 lite")));
|
||||
assumeFormatsSupported(
|
||||
context,
|
||||
testId,
|
||||
/* inputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat,
|
||||
/* outputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat);
|
||||
|
||||
ExportTestResult exportTestResult =
|
||||
exportVideoAndAudioTo720pWithDynamicScheduling(
|
||||
testId,
|
||||
Uri.parse(MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.uri),
|
||||
/* durationMs= */ 15_000);
|
||||
|
||||
// Running this without dynamic scheduling runs at under 40 fps.
|
||||
assertThat(exportTestResult.throughputFps).isAtLeast(60);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void export1080pWithAudioTo720p_withDynamicScheduling_completesWithCorrectNumberOfFrames()
|
||||
throws Exception {
|
||||
assumeFormatsSupported(
|
||||
context,
|
||||
testId,
|
||||
/* inputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat,
|
||||
/* outputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat);
|
||||
|
||||
ExportTestResult exportTestResult =
|
||||
exportVideoAndAudioTo720pWithDynamicScheduling(
|
||||
testId,
|
||||
Uri.parse(MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.uri),
|
||||
/* durationMs= */ 5_000);
|
||||
|
||||
assertThat(exportTestResult.exportResult.videoFrameCount).isEqualTo(150);
|
||||
}
|
||||
|
||||
private static ExportTestResult exportVideoAndAudioTo720pWithDynamicScheduling(
|
||||
String testId, Uri mediaUri, long durationMs) throws Exception {
|
||||
Context context = ApplicationProvider.getApplicationContext();
|
||||
Codec.DecoderFactory decoderFactory =
|
||||
new DefaultDecoderFactory.Builder(context)
|
||||
.experimentalSetDynamicSchedulingEnabled(true)
|
||||
.setShouldConfigureOperatingRate(true)
|
||||
.build();
|
||||
AssetLoader.Factory assetLoaderFactory =
|
||||
new DefaultAssetLoaderFactory(context, decoderFactory, Clock.DEFAULT);
|
||||
Transformer transformer =
|
||||
new Transformer.Builder(context)
|
||||
.setVideoMimeType(MimeTypes.VIDEO_H264)
|
||||
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context))
|
||||
.setAssetLoaderFactory(assetLoaderFactory)
|
||||
.build();
|
||||
MediaItem mediaItem =
|
||||
MediaItem.fromUri(mediaUri)
|
||||
.buildUpon()
|
||||
.setClippingConfiguration(
|
||||
new MediaItem.ClippingConfiguration.Builder().setEndPositionMs(durationMs).build())
|
||||
.build();
|
||||
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
|
||||
sonicAudioProcessor.setOutputSampleRateHz(44_100);
|
||||
ChannelMixingAudioProcessor mixingAudioProcessor = new ChannelMixingAudioProcessor();
|
||||
mixingAudioProcessor.putChannelMixingMatrix(
|
||||
ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1));
|
||||
EditedMediaItem editedMediaItem =
|
||||
new EditedMediaItem.Builder(mediaItem)
|
||||
.setEffects(
|
||||
new Effects(
|
||||
ImmutableList.of(sonicAudioProcessor, mixingAudioProcessor),
|
||||
ImmutableList.of(
|
||||
Presentation.createForWidthAndHeight(
|
||||
1280, 720, Presentation.LAYOUT_SCALE_TO_FIT))))
|
||||
.build();
|
||||
|
||||
return new TransformerAndroidTestRunner.Builder(context, transformer)
|
||||
.build()
|
||||
.run(testId, editedMediaItem);
|
||||
}
|
||||
}
|
||||
@ -78,6 +78,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
|
||||
private @C.Priority int codecPriority;
|
||||
private boolean shouldConfigureOperatingRate;
|
||||
private MediaCodecSelector mediaCodecSelector;
|
||||
private boolean dynamicSchedulingEnabled;
|
||||
|
||||
/** Creates a new {@link Builder}. */
|
||||
public Builder(Context context) {
|
||||
@ -165,6 +166,28 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets whether decoder dynamic scheduling is enabled.
|
||||
*
|
||||
* <p>If enabled, the {@link ExoPlayerAssetLoader} can change how often the rendering loop for
|
||||
* {@linkplain DefaultCodec decoders} created by this factory is run.
|
||||
*
|
||||
* <p>On some devices, setting this to {@code true} will {@linkplain
|
||||
* DefaultCodec#queueInputBuffer feed} and {@linkplain DefaultCodec#releaseOutputBuffer drain}
|
||||
* decoders more frequently, and will lead to improved performance.
|
||||
*
|
||||
* <p>The default value is {@code false}.
|
||||
*
|
||||
* <p>This method is experimental, and will be renamed or removed in a future release.
|
||||
*
|
||||
* @param dynamicSchedulingEnabled Whether to enable dynamic scheduling.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public Builder experimentalSetDynamicSchedulingEnabled(boolean dynamicSchedulingEnabled) {
|
||||
this.dynamicSchedulingEnabled = dynamicSchedulingEnabled;
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Creates an instance of {@link DefaultDecoderFactory}, using defaults if values are unset. */
|
||||
public DefaultDecoderFactory build() {
|
||||
return new DefaultDecoderFactory(this);
|
||||
@ -177,6 +200,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
|
||||
private final @C.Priority int codecPriority;
|
||||
private final boolean shouldConfigureOperatingRate;
|
||||
private final MediaCodecSelector mediaCodecSelector;
|
||||
private final boolean dynamicSchedulingEnabled;
|
||||
|
||||
/**
|
||||
* @deprecated Use {@link Builder} instead.
|
||||
@ -210,6 +234,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
|
||||
this.codecPriority = builder.codecPriority;
|
||||
this.shouldConfigureOperatingRate = builder.shouldConfigureOperatingRate;
|
||||
this.mediaCodecSelector = builder.mediaCodecSelector;
|
||||
this.dynamicSchedulingEnabled = builder.dynamicSchedulingEnabled;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -327,6 +352,15 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
|
||||
return codec;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether decoder dynamic scheduling is enabled.
|
||||
*
|
||||
* <p>See {@link Builder#experimentalSetDynamicSchedulingEnabled}.
|
||||
*/
|
||||
public boolean isDynamicSchedulingEnabled() {
|
||||
return dynamicSchedulingEnabled;
|
||||
}
|
||||
|
||||
private static DefaultCodec createCodecFromDecoderInfos(
|
||||
Context context,
|
||||
List<MediaCodecInfo> decoderInfos,
|
||||
@ -354,7 +388,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
|
||||
}
|
||||
|
||||
private static void configureOperatingRate(MediaFormat mediaFormat) {
|
||||
if (Util.SDK_INT < 25) {
|
||||
if (SDK_INT < 25) {
|
||||
// Not setting priority and operating rate achieves better decoding performance.
|
||||
return;
|
||||
}
|
||||
@ -371,7 +405,8 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory {
|
||||
private static boolean deviceNeedsPriorityWorkaround() {
|
||||
// On these chipsets, decoder configuration fails if KEY_OPERATING_RATE is set but not
|
||||
// KEY_PRIORITY. See b/358519863.
|
||||
return Util.SDK_INT >= 31 && Build.SOC_MODEL.equals("s5e8835");
|
||||
return SDK_INT >= 31
|
||||
&& (Build.SOC_MODEL.equals("s5e8835") || Build.SOC_MODEL.equals("SA8155P"));
|
||||
}
|
||||
|
||||
private static boolean deviceNeedsDisable8kWorkaround(Format format) {
|
||||
|
||||
@ -54,6 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
this.decoderFactory = decoderFactory;
|
||||
this.hdrMode = hdrMode;
|
||||
decodeOnlyPresentationTimestamps = new ArrayList<>();
|
||||
maxDecoderPendingFrameCount = C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -61,6 +62,22 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
return TAG;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*
|
||||
* <p>The duration is calculated based on the number of {@linkplain #maxDecoderPendingFrameCount
|
||||
* allowed pending frames}.
|
||||
*/
|
||||
@Override
|
||||
public long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) {
|
||||
if (maxDecoderPendingFrameCount == C.INDEX_UNSET) {
|
||||
return DEFAULT_DURATION_TO_PROGRESS_US;
|
||||
}
|
||||
// TODO: b/258809496 - Consider using async API and dynamic scheduling when decoder input
|
||||
// slots are available.
|
||||
return maxDecoderPendingFrameCount * 2_000L;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Format overrideInputFormat(Format format) {
|
||||
if (hdrMode == Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR
|
||||
|
||||
@ -189,6 +189,10 @@ public final class ExoPlayerAssetLoader implements AssetLoader {
|
||||
.setLooper(looper)
|
||||
.setUsePlatformDiagnostics(false)
|
||||
.setReleaseTimeoutMs(getReleaseTimeoutMs());
|
||||
if (decoderFactory instanceof DefaultDecoderFactory) {
|
||||
playerBuilder.experimentalSetDynamicSchedulingEnabled(
|
||||
((DefaultDecoderFactory) decoderFactory).isDynamicSchedulingEnabled());
|
||||
}
|
||||
if (clock != Clock.DEFAULT) {
|
||||
// Transformer.Builder#setClock is also @VisibleForTesting, so if we're using a non-default
|
||||
// clock we must be in a test context.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user