From fe640da5e889e72aff5cf0b096acdd2dd7db5f4f Mon Sep 17 00:00:00 2001 From: tofunmi Date: Fri, 15 Mar 2024 09:59:48 -0700 Subject: [PATCH] Transformer: add speedChange API PiperOrigin-RevId: 616163061 --- .../transformer/TransformerEndToEndTest.java | 37 +++++++++++++++++++ .../androidx/media3/transformer/Effects.java | 29 +++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 92f7f0b2da..37a3a5df5c 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -52,6 +52,7 @@ import android.os.Handler; import android.os.HandlerThread; import android.os.Looper; import android.os.Message; +import android.util.Pair; import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.Format; @@ -64,6 +65,7 @@ import androidx.media3.common.audio.AudioProcessor.AudioFormat; import androidx.media3.common.audio.ChannelMixingAudioProcessor; import androidx.media3.common.audio.ChannelMixingMatrix; import androidx.media3.common.audio.SonicAudioProcessor; +import androidx.media3.common.audio.SpeedProvider; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Util; import androidx.media3.datasource.DataSourceBitmapLoader; @@ -81,6 +83,7 @@ import androidx.media3.extractor.mp4.Mp4Extractor; import androidx.media3.extractor.text.DefaultSubtitleParserFactory; import androidx.media3.test.utils.FakeExtractorOutput; import androidx.media3.test.utils.FakeTrackOutput; +import androidx.media3.test.utils.TestSpeedProvider; import androidx.media3.test.utils.TestUtil; import androidx.media3.transformer.AssetLoader.CompositionSettings; import androidx.test.core.app.ApplicationProvider; @@ -868,6 +871,40 @@ public class TransformerEndToEndTest { assertThat(result.exportResult.audioConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED); } + @Test + public void speedAdjustedMedia_completesWithCorrectDuration() throws Exception { + Transformer transformer = new Transformer.Builder(context).build(); + SpeedProvider speedProvider = + TestSpeedProvider.createWithStartTimes( + new long[] { + 0L, + 3 * C.MICROS_PER_SECOND, + 6 * C.MICROS_PER_SECOND, + 9 * C.MICROS_PER_SECOND, + 12 * C.MICROS_PER_SECOND + }, + new float[] {0.5f, 0.75f, 1f, 1.5f, 2f}); + Pair speedEffect = + Effects.createExperimentalSpeedChangingEffect(speedProvider); + Effects effects = + new Effects( + /* audioProcessors= */ ImmutableList.of(speedEffect.first), + /* videoEffects= */ ImmutableList.of(speedEffect.second)); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + MediaItem.fromUri(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)) + .setEffects(effects) + .build(); + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + // The input video is 15.537 seconds. + // 3 / 0.5 + 3 / 0.75 + 3 + 3 / 1.5 + 3.537 / 2 rounds up to 16_770 + assertThat(result.exportResult.durationMs).isAtMost(16_770); + } + @Test public void videoEncoderFormatUnsupported_completesWithError() throws Exception { if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java index ec597ed31b..a5fdfa8f68 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Effects.java @@ -15,10 +15,15 @@ */ package androidx.media3.transformer; +import android.util.Pair; import androidx.media3.common.Effect; import androidx.media3.common.MediaItem; import androidx.media3.common.audio.AudioProcessor; +import androidx.media3.common.audio.SpeedChangingAudioProcessor; +import androidx.media3.common.audio.SpeedProvider; import androidx.media3.common.util.UnstableApi; +import androidx.media3.effect.SpeedChangeEffect; +import androidx.media3.effect.TimestampAdjustment; import com.google.common.collect.ImmutableList; import java.util.List; @@ -54,4 +59,28 @@ public final class Effects { this.audioProcessors = ImmutableList.copyOf(audioProcessors); this.videoEffects = ImmutableList.copyOf(videoEffects); } + + /** + * Creates an interlinked {@linkplain AudioProcessor audio processor} and {@linkplain Effect video + * effect} that changes the speed to media samples in segments of the input file specified by the + * given {@link SpeedProvider}. + * + *

The {@linkplain AudioProcessor audio processor} and {@linkplain Effect video effect} are + * interlinked to help maintain A/V sync. When using Transformer, if the input file doesn't have + * audio, or audio is being removed, you may have to {@linkplain + * Composition.Builder#experimentalSetForceAudioTrack force an audio track} for the interlinked + * effects to function correctly. Alternatively, you can use {@link SpeedChangeEffect} when input + * has no audio. + * + * @param speedProvider The {@link SpeedProvider} determining the speed for the media at specific + * timestamps. + */ + public static Pair createExperimentalSpeedChangingEffect( + SpeedProvider speedProvider) { + SpeedChangingAudioProcessor speedChangingAudioProcessor = + new SpeedChangingAudioProcessor(speedProvider); + Effect audioDrivenvideoEffect = + new TimestampAdjustment(speedChangingAudioProcessor::getSpeedAdjustedTimeAsync); + return Pair.create(speedChangingAudioProcessor, audioDrivenvideoEffect); + } }