diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSpeedAdjustmentsTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSpeedAdjustmentsTest.java new file mode 100644 index 0000000000..6b63f93a62 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSpeedAdjustmentsTest.java @@ -0,0 +1,136 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET; +import static com.google.common.truth.Truth.assertThat; + +import android.app.Instrumentation; +import android.content.Context; +import android.util.Pair; +import android.view.SurfaceView; +import androidx.media3.common.Effect; +import androidx.media3.common.MediaItem; +import androidx.media3.common.audio.AudioProcessor; +import androidx.media3.effect.GlEffect; +import androidx.media3.test.utils.TestSpeedProvider; +import androidx.test.ext.junit.rules.ActivityScenarioRule; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Instrumentation tests for {@link CompositionPlayer} with Speed Adjustments. */ +@RunWith(AndroidJUnit4.class) +public class CompositionPlayerSpeedAdjustmentsTest { + private static final long TEST_TIMEOUT_MS = 10_000; + + @Rule + public ActivityScenarioRule rule = + new ActivityScenarioRule<>(SurfaceTestActivity.class); + + private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + private final Context applicationContext = instrumentation.getContext().getApplicationContext(); + + private CompositionPlayer compositionPlayer; + private SurfaceView surfaceView; + + @Before + public void setupSurfaces() { + rule.getScenario().onActivity(activity -> surfaceView = activity.getSurfaceView()); + } + + @After + public void closeActivity() { + rule.getScenario().close(); + } + + @Test + public void videoPreview_withSpeedAdjustment_timestampsAreCorrect() throws Exception { + Pair effects = + Effects.createExperimentalSpeedChangingEffect( + TestSpeedProvider.createWithStartTimes( + new long[] {0, 300_000L, 600_000L}, new float[] {2f, 1f, 0.5f})); + EditedMediaItem video = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET.uri)) + .setDurationUs(1_000_000) + .setEffects( + new Effects(ImmutableList.of(effects.first), ImmutableList.of(effects.second))) + .build(); + ImmutableList expectedTimestamps = + ImmutableList.of( + 0L, 16683L, 33366L, 50050L, 66733L, 83416L, 100100L, 116783L, 133466L, 150300L, 183666L, + 217033L, 250400L, 283766L, 317133L, 350500L, 383866L, 417233L, 451200L, 517932L, + 584666L, 651400L, 718132L, 784866L, 851600L, 918332L, 985066L, 1051800L, 1118532L, + 1185266L); + + ImmutableList timestampsFromCompositionPlayer = getTimestampsFromCompositionPlayer(video); + + assertThat(timestampsFromCompositionPlayer).isEqualTo(expectedTimestamps); + } + + private ImmutableList getTimestampsFromCompositionPlayer(EditedMediaItem item) + throws Exception { + PlayerTestListener compositionPlayerListener = new PlayerTestListener(TEST_TIMEOUT_MS); + InputTimestampRecordingShaderProgram timestampRecordingShaderProgram = + new InputTimestampRecordingShaderProgram(); + ImmutableList timestampRecordingEditedMediaItems = + appendVideoEffects( + item, + /* effects= */ ImmutableList.of( + (GlEffect) (context, useHdr) -> timestampRecordingShaderProgram)); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(compositionPlayerListener); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence(timestampRecordingEditedMediaItems)) + .experimentalSetForceAudioTrack(true) + .build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + compositionPlayerListener.waitUntilPlayerEnded(); + instrumentation.runOnMainSync(() -> compositionPlayer.release()); + + return timestampRecordingShaderProgram.getInputTimestampsUs(); + } + + private static ImmutableList appendVideoEffects( + EditedMediaItem item, List effects) { + return ImmutableList.of( + item.buildUpon() + .setEffects( + new Effects( + item.effects.audioProcessors, + new ImmutableList.Builder() + .addAll(item.effects.videoEffects) + .addAll(effects) + .build())) + .build()); + } +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java index bcfbd6f395..c17664771f 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertThrows; import android.app.Instrumentation; import android.content.Context; import android.graphics.BitmapFactory; +import android.util.Pair; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.TextureView; @@ -42,6 +43,7 @@ import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; +import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.util.SystemClock; import androidx.media3.common.util.Util; import androidx.media3.datasource.AssetDataSource; @@ -53,10 +55,12 @@ import androidx.media3.exoplayer.image.BitmapFactoryImageDecoder; import androidx.media3.exoplayer.image.ImageDecoder; import androidx.media3.exoplayer.image.ImageDecoderException; import androidx.media3.exoplayer.source.ExternalLoader; +import androidx.media3.test.utils.TestSpeedProvider; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.rules.ActivityScenarioRule; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.platform.app.InstrumentationRegistry; +import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.List; import java.util.concurrent.Executor; @@ -379,6 +383,64 @@ public class CompositionPlayerTest { listener.waitUntilPlayerEnded(); } + @Test + public void videoPreview_withSpeedUp_playerEnds() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + Pair effects = + Effects.createExperimentalSpeedChangingEffect( + TestSpeedProvider.createWithStartTimes(new long[] {0}, new float[] {2f})); + EditedMediaItem video = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET.uri)) + .setDurationUs(1_000_000) + .setEffects( + new Effects(ImmutableList.of(effects.first), ImmutableList.of(effects.second))) + .build(); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(video)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + listener.waitUntilPlayerEnded(); + } + + @Test + public void videoPreview_withSlowDown_playerEnds() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + Pair effects = + Effects.createExperimentalSpeedChangingEffect( + TestSpeedProvider.createWithStartTimes(new long[] {0}, new float[] {0.5f})); + EditedMediaItem video = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET.uri)) + .setDurationUs(1_000_000) + .setEffects( + new Effects(ImmutableList.of(effects.first), ImmutableList.of(effects.second))) + .build(); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(video)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + listener.waitUntilPlayerEnded(); + } + @Test public void playback_videoSinkProviderFails_playerRaisesError() { PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java index 6b8df86dca..b01a5e2c2a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java @@ -35,14 +35,17 @@ import androidx.annotation.Nullable; import androidx.annotation.RestrictTo; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; +import androidx.media3.common.Effect; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.PlaybackException; import androidx.media3.common.Player; import androidx.media3.common.PreviewingVideoGraph; import androidx.media3.common.SimpleBasePlayer; +import androidx.media3.common.Timeline; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoSize; +import androidx.media3.common.audio.SpeedProvider; import androidx.media3.common.util.Clock; import androidx.media3.common.util.HandlerWrapper; import androidx.media3.common.util.Log; @@ -50,6 +53,7 @@ import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.effect.PreviewingSingleInputVideoGraph; +import androidx.media3.effect.TimestampAdjustment; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.RendererCapabilities; @@ -60,12 +64,16 @@ import androidx.media3.exoplayer.source.ClippingMediaSource; import androidx.media3.exoplayer.source.ConcatenatingMediaSource2; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.exoplayer.source.ExternalLoader; +import androidx.media3.exoplayer.source.ForwardingTimeline; +import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.MergingMediaSource; import androidx.media3.exoplayer.source.SilenceMediaSource; import androidx.media3.exoplayer.source.TrackGroupArray; +import androidx.media3.exoplayer.source.WrappingMediaSource; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; +import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.util.EventLogger; import androidx.media3.exoplayer.video.CompositingVideoSinkProvider; import androidx.media3.exoplayer.video.VideoFrameReleaseControl; @@ -331,6 +339,7 @@ public final class CompositionPlayer extends SimpleBasePlayer !composition.sequences.isEmpty() && composition.sequences.size() <= MAX_SUPPORTED_SEQUENCES); checkState(this.composition == null); + composition = deactivateSpeedAdjustingVideoEffects(composition); setCompositionInternal(composition); if (videoOutput != null) { @@ -577,6 +586,32 @@ public final class CompositionPlayer extends SimpleBasePlayer // Internal methods + private static Composition deactivateSpeedAdjustingVideoEffects(Composition composition) { + List newSequences = new ArrayList<>(); + for (EditedMediaItemSequence sequence : composition.sequences) { + List newEditedMediaItems = new ArrayList<>(); + for (EditedMediaItem editedMediaItem : sequence.editedMediaItems) { + ImmutableList videoEffects = editedMediaItem.effects.videoEffects; + List newVideoEffects = new ArrayList<>(); + for (Effect videoEffect : videoEffects) { + if (videoEffect instanceof TimestampAdjustment) { + newVideoEffects.add( + new InactiveTimestampAdjustment(((TimestampAdjustment) videoEffect).speedProvider)); + } else { + newVideoEffects.add(videoEffect); + } + } + newEditedMediaItems.add( + editedMediaItem + .buildUpon() + .setEffects(new Effects(editedMediaItem.effects.audioProcessors, newVideoEffects)) + .build()); + } + newSequences.add(new EditedMediaItemSequence(newEditedMediaItems, sequence.isLooping)); + } + return composition.buildUpon().setSequences(newSequences).build(); + } + private void updatePlaybackState() { if (players.isEmpty() || playbackException != null) { playbackState = STATE_IDLE; @@ -712,15 +747,15 @@ public final class CompositionPlayer extends SimpleBasePlayer new SilenceMediaSource(editedMediaItem.durationUs), editedMediaItem.mediaItem.clippingConfiguration.startPositionUs, editedMediaItem.mediaItem.clippingConfiguration.endPositionUs); - mediaSourceBuilder.add( + MediaSource mergingMediaSource = new MergingMediaSource( defaultMediaSourceFactory.createMediaSource(editedMediaItem.mediaItem), - // Generate silence as long as the MediaItem without clipping, because the actual - // media track starts at the clipped position. For example, if a video is 1000ms - // long and clipped 900ms from the start, its MediaSource will be enabled at 900ms - // during track selection, rather than at 0ms. - silenceMediaSource), - /* initialPlaceholderDurationMs= */ usToMs(durationUs)); + silenceMediaSource); + MediaSource itemMediaSource = + wrapWithVideoEffectsBasedMediaSources( + mergingMediaSource, editedMediaItem.effects.videoEffects, durationUs); + mediaSourceBuilder.add( + itemMediaSource, /* initialPlaceholderDurationMs= */ usToMs(durationUs)); } else { mediaSourceBuilder.add( editedMediaItem.mediaItem, /* initialPlaceholderDurationMs= */ usToMs(durationUs)); @@ -729,6 +764,62 @@ public final class CompositionPlayer extends SimpleBasePlayer player.setMediaSource(mediaSourceBuilder.build()); } + private MediaSource wrapWithVideoEffectsBasedMediaSources( + MediaSource mediaSource, ImmutableList videoEffects, long durationUs) { + MediaSource newMediaSource = mediaSource; + for (Effect videoEffect : videoEffects) { + if (videoEffect instanceof InactiveTimestampAdjustment) { + newMediaSource = + wrapWithSpeedChangingMediaSource( + newMediaSource, + ((InactiveTimestampAdjustment) videoEffect).speedProvider, + durationUs); + } + } + return newMediaSource; + } + + private MediaSource wrapWithSpeedChangingMediaSource( + MediaSource mediaSource, SpeedProvider speedProvider, long durationUs) { + return new WrappingMediaSource(mediaSource) { + + @Override + public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long startPositionUs) { + return new SpeedProviderMediaPeriod( + super.createPeriod(id, allocator, startPositionUs), speedProvider); + } + + @Override + public void releasePeriod(MediaPeriod mediaPeriod) { + MediaPeriod wrappedPeriod = ((SpeedProviderMediaPeriod) mediaPeriod).mediaPeriod; + super.releasePeriod(wrappedPeriod); + } + + @Override + protected void onChildSourceInfoRefreshed(Timeline newTimeline) { + Timeline timeline = + new ForwardingTimeline(newTimeline) { + @Override + public Window getWindow( + int windowIndex, Window window, long defaultPositionProjectionUs) { + Window wrappedWindow = + newTimeline.getWindow(windowIndex, window, defaultPositionProjectionUs); + wrappedWindow.durationUs = durationUs; + return wrappedWindow; + } + + @Override + public Period getPeriod(int periodIndex, Period period, boolean setIds) { + Timeline.Period wrappedPeriod = newTimeline.getPeriod(periodIndex, period, setIds); + wrappedPeriod.durationUs = durationUs; + return wrappedPeriod; + } + }; + super.onChildSourceInfoRefreshed(timeline); + } + }; + } + private long getContentPositionMs() { return players.isEmpty() ? C.TIME_UNSET : players.get(0).getContentPosition(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/InactiveTimestampAdjustment.java b/libraries/transformer/src/main/java/androidx/media3/transformer/InactiveTimestampAdjustment.java new file mode 100644 index 0000000000..490b90efd5 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/InactiveTimestampAdjustment.java @@ -0,0 +1,46 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import android.content.Context; +import androidx.media3.common.audio.SpeedProvider; +import androidx.media3.common.util.SpeedProviderUtil; +import androidx.media3.effect.GlEffect; +import androidx.media3.effect.GlShaderProgram; +import androidx.media3.effect.PassthroughShaderProgram; + +/* package */ class InactiveTimestampAdjustment implements GlEffect { + public final SpeedProvider speedProvider; + + public InactiveTimestampAdjustment(SpeedProvider speedProvider) { + this.speedProvider = speedProvider; + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { + return new PassthroughShaderProgram(); + } + + @Override + public boolean isNoOp(int inputWidth, int inputHeight) { + return true; + } + + @Override + public long getDurationAfterEffectApplied(long durationUs) { + return SpeedProviderUtil.getDurationAfterSpeedProviderApplied(speedProvider, durationUs); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedProviderMediaPeriod.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedProviderMediaPeriod.java new file mode 100644 index 0000000000..165c291f0c --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedProviderMediaPeriod.java @@ -0,0 +1,287 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.StreamKey; +import androidx.media3.common.audio.SpeedProvider; +import androidx.media3.common.util.Assertions; +import androidx.media3.common.util.LongArray; +import androidx.media3.common.util.NullableType; +import androidx.media3.common.util.Util; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.exoplayer.FormatHolder; +import androidx.media3.exoplayer.LoadingInfo; +import androidx.media3.exoplayer.SeekParameters; +import androidx.media3.exoplayer.source.MediaPeriod; +import androidx.media3.exoplayer.source.SampleStream; +import androidx.media3.exoplayer.source.TrackGroupArray; +import androidx.media3.exoplayer.trackselection.ExoTrackSelection; +import com.google.common.primitives.Floats; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** A {@link MediaPeriod} that adjusts the timestamps as specified by the speed provider. */ +/* package */ final class SpeedProviderMediaPeriod implements MediaPeriod, MediaPeriod.Callback { + + public final MediaPeriod mediaPeriod; + private final SpeedProviderMapper speedProviderMapper; + private @MonotonicNonNull Callback callback; + + /** + * Create an instance. + * + * @param mediaPeriod The wrapped {@link MediaPeriod}. + * @param speedProvider The offset to apply to all timestamps coming from the wrapped period. + */ + public SpeedProviderMediaPeriod(MediaPeriod mediaPeriod, SpeedProvider speedProvider) { + this.mediaPeriod = mediaPeriod; + this.speedProviderMapper = new SpeedProviderMapper(speedProvider); + } + + /** Returns the wrapped {@link MediaPeriod}. */ + public MediaPeriod getWrappedMediaPeriod() { + return mediaPeriod; + } + + @Override + public void prepare(Callback callback, long positionUs) { + this.callback = callback; + mediaPeriod.prepare(/* callback= */ this, speedProviderMapper.getOriginalTimeUs(positionUs)); + } + + @Override + public void maybeThrowPrepareError() throws IOException { + mediaPeriod.maybeThrowPrepareError(); + } + + @Override + public TrackGroupArray getTrackGroups() { + return mediaPeriod.getTrackGroups(); + } + + @Override + public List getStreamKeys(List trackSelections) { + return mediaPeriod.getStreamKeys(trackSelections); + } + + @Override + public long selectTracks( + @NullableType ExoTrackSelection[] selections, + boolean[] mayRetainStreamFlags, + @NullableType SampleStream[] streams, + boolean[] streamResetFlags, + long positionUs) { + @NullableType SampleStream[] childStreams = new SampleStream[streams.length]; + for (int i = 0; i < streams.length; i++) { + SpeedProviderMapperSampleStream sampleStream = (SpeedProviderMapperSampleStream) streams[i]; + childStreams[i] = sampleStream != null ? sampleStream.getChildStream() : null; + } + long startPositionUs = + mediaPeriod.selectTracks( + selections, + mayRetainStreamFlags, + childStreams, + streamResetFlags, + speedProviderMapper.getOriginalTimeUs(positionUs)); + for (int i = 0; i < streams.length; i++) { + @Nullable SampleStream childStream = childStreams[i]; + if (childStream == null) { + streams[i] = null; + } else if (streams[i] == null + || ((SpeedProviderMapperSampleStream) streams[i]).getChildStream() != childStream) { + streams[i] = new SpeedProviderMapperSampleStream(childStream, speedProviderMapper); + } + } + return speedProviderMapper.getAdjustedTimeUs(startPositionUs); + } + + @Override + public void discardBuffer(long positionUs, boolean toKeyframe) { + mediaPeriod.discardBuffer(speedProviderMapper.getOriginalTimeUs(positionUs), toKeyframe); + } + + @Override + public long readDiscontinuity() { + long discontinuityPositionUs = mediaPeriod.readDiscontinuity(); + return discontinuityPositionUs == C.TIME_UNSET + ? C.TIME_UNSET + : speedProviderMapper.getAdjustedTimeUs(discontinuityPositionUs); + } + + @Override + public long seekToUs(long positionUs) { + return speedProviderMapper.getAdjustedTimeUs( + mediaPeriod.seekToUs(speedProviderMapper.getOriginalTimeUs(positionUs))); + } + + @Override + public long getAdjustedSeekPositionUs(long positionUs, SeekParameters seekParameters) { + return speedProviderMapper.getAdjustedTimeUs( + mediaPeriod.getAdjustedSeekPositionUs( + speedProviderMapper.getOriginalTimeUs(positionUs), seekParameters)); + } + + @Override + public long getBufferedPositionUs() { + long bufferedPositionUs = mediaPeriod.getBufferedPositionUs(); + return bufferedPositionUs == C.TIME_END_OF_SOURCE + ? C.TIME_END_OF_SOURCE + : speedProviderMapper.getAdjustedTimeUs(bufferedPositionUs); + } + + @Override + public long getNextLoadPositionUs() { + long nextLoadPositionUs = mediaPeriod.getNextLoadPositionUs(); + return nextLoadPositionUs == C.TIME_END_OF_SOURCE + ? C.TIME_END_OF_SOURCE + : speedProviderMapper.getAdjustedTimeUs(nextLoadPositionUs); + } + + @Override + public boolean continueLoading(LoadingInfo loadingInfo) { + return mediaPeriod.continueLoading( + loadingInfo + .buildUpon() + .setPlaybackPositionUs( + speedProviderMapper.getOriginalTimeUs(loadingInfo.playbackPositionUs)) + .build()); + } + + @Override + public boolean isLoading() { + return mediaPeriod.isLoading(); + } + + @Override + public void reevaluateBuffer(long positionUs) { + mediaPeriod.reevaluateBuffer(speedProviderMapper.getOriginalTimeUs(positionUs)); + } + + @Override + public void onPrepared(MediaPeriod mediaPeriod) { + Assertions.checkNotNull(callback).onPrepared(/* mediaPeriod= */ this); + } + + @Override + public void onContinueLoadingRequested(MediaPeriod source) { + Assertions.checkNotNull(callback).onContinueLoadingRequested(/* source= */ this); + } + + private static final class SpeedProviderMapper { + + private final long[] outputSegmentStartTimesUs; + private final long[] inputSegmentStartTimesUs; + private final float[] speeds; + + public SpeedProviderMapper(SpeedProvider speedProvider) { + LongArray outputSegmentStartTimesUs = new LongArray(); + LongArray inputSegmentStartTimesUs = new LongArray(); + List speeds = new ArrayList<>(); + + long lastOutputSegmentStartTimeUs = 0; + long lastInputSegmentStartTimeUs = 0; + float lastSpeed = speedProvider.getSpeed(lastInputSegmentStartTimeUs); + outputSegmentStartTimesUs.add(lastOutputSegmentStartTimeUs); + inputSegmentStartTimesUs.add(lastInputSegmentStartTimeUs); + speeds.add(lastSpeed); + long nextSpeedChangeTimeUs = + speedProvider.getNextSpeedChangeTimeUs(lastInputSegmentStartTimeUs); + + while (nextSpeedChangeTimeUs != C.TIME_UNSET) { + lastOutputSegmentStartTimeUs += + (long) ((nextSpeedChangeTimeUs - lastInputSegmentStartTimeUs) / lastSpeed); + lastInputSegmentStartTimeUs = nextSpeedChangeTimeUs; + lastSpeed = speedProvider.getSpeed(lastInputSegmentStartTimeUs); + outputSegmentStartTimesUs.add(lastOutputSegmentStartTimeUs); + inputSegmentStartTimesUs.add(lastInputSegmentStartTimeUs); + speeds.add(lastSpeed); + nextSpeedChangeTimeUs = speedProvider.getNextSpeedChangeTimeUs(lastInputSegmentStartTimeUs); + } + this.outputSegmentStartTimesUs = outputSegmentStartTimesUs.toArray(); + this.inputSegmentStartTimesUs = inputSegmentStartTimesUs.toArray(); + this.speeds = Floats.toArray(speeds); + } + + public long getAdjustedTimeUs(long originalTimeUs) { + int index = + Util.binarySearchFloor( + inputSegmentStartTimesUs, + originalTimeUs, + /* inclusive= */ true, + /* stayInBounds= */ true); + return (long) + (outputSegmentStartTimesUs[index] + + (originalTimeUs - inputSegmentStartTimesUs[index]) / speeds[index]); + } + + public long getOriginalTimeUs(long adjustedTimeUs) { + int index = + Util.binarySearchFloor( + outputSegmentStartTimesUs, + adjustedTimeUs, + /* inclusive= */ true, + /* stayInBounds= */ true); + return (long) + (inputSegmentStartTimesUs[index] + + (adjustedTimeUs - outputSegmentStartTimesUs[index]) * speeds[index]); + } + } + + private static final class SpeedProviderMapperSampleStream implements SampleStream { + + private final SampleStream sampleStream; + private final SpeedProviderMapper speedProviderMapper; + + public SpeedProviderMapperSampleStream( + SampleStream sampleStream, SpeedProviderMapper speedProviderMapper) { + this.sampleStream = sampleStream; + this.speedProviderMapper = speedProviderMapper; + } + + public SampleStream getChildStream() { + return sampleStream; + } + + @Override + public boolean isReady() { + return sampleStream.isReady(); + } + + @Override + public void maybeThrowError() throws IOException { + sampleStream.maybeThrowError(); + } + + @Override + public int readData( + FormatHolder formatHolder, DecoderInputBuffer buffer, @ReadFlags int readFlags) { + int readResult = sampleStream.readData(formatHolder, buffer, readFlags); + if (readResult == C.RESULT_BUFFER_READ) { + buffer.timeUs = speedProviderMapper.getAdjustedTimeUs(buffer.timeUs); + } + return readResult; + } + + @Override + public int skipData(long positionUs) { + return sampleStream.skipData(speedProviderMapper.getOriginalTimeUs(positionUs)); + } + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/SpeedProviderMediaPeriodTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/SpeedProviderMediaPeriodTest.java new file mode 100644 index 0000000000..668d2c0da8 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/SpeedProviderMediaPeriodTest.java @@ -0,0 +1,307 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.TrackGroup; +import androidx.media3.common.audio.SpeedProvider; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.exoplayer.FormatHolder; +import androidx.media3.exoplayer.LoadingInfo; +import androidx.media3.exoplayer.SeekParameters; +import androidx.media3.exoplayer.drm.DrmSessionEventListener; +import androidx.media3.exoplayer.drm.DrmSessionManager; +import androidx.media3.exoplayer.source.MediaPeriod; +import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.MediaSourceEventListener.EventDispatcher; +import androidx.media3.exoplayer.source.SampleStream; +import androidx.media3.exoplayer.source.TrackGroupArray; +import androidx.media3.exoplayer.trackselection.ExoTrackSelection; +import androidx.media3.exoplayer.trackselection.FixedTrackSelection; +import androidx.media3.exoplayer.upstream.DefaultAllocator; +import androidx.media3.test.utils.FakeMediaPeriod; +import androidx.media3.test.utils.FakeSampleStream; +import androidx.media3.test.utils.TestSpeedProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.util.concurrent.CountDownLatch; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link SpeedProviderMediaPeriod}. */ +@RunWith(AndroidJUnit4.class) +public final class SpeedProviderMediaPeriodTest { + + SpeedProvider speedProvider = + TestSpeedProvider.createWithStartTimes( + new long[] {0, 1_000_000, 2_000_000}, new float[] {0.5f, 1f, 2f}); + + @Test + public void selectTracks_createsSampleStreamAdjustingTimes() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 500_000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(fakeMediaPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 0); + FormatHolder formatHolder = new FormatHolder(); + DecoderInputBuffer inputBuffer = + new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); + ImmutableList.Builder readResults = ImmutableList.builder(); + + SampleStream sampleStream = + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + readResults.add(sampleStream.readData(formatHolder, inputBuffer, FLAG_REQUIRE_FORMAT)); + readResults.add(sampleStream.readData(formatHolder, inputBuffer, /* readFlags= */ 0)); + long readBufferTimeUs = inputBuffer.timeUs; + readResults.add(sampleStream.readData(formatHolder, inputBuffer, /* readFlags= */ 0)); + boolean readEndOfStreamBuffer = inputBuffer.isEndOfStream(); + + assertThat(readResults.build()) + .containsExactly(C.RESULT_FORMAT_READ, C.RESULT_BUFFER_READ, C.RESULT_BUFFER_READ); + assertThat(readBufferTimeUs).isEqualTo(1_000_000); + assertThat(readEndOfStreamBuffer).isTrue(); + } + + @Test + public void getBufferedPositionUs_returnsAdjustedPosition() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 500_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_500_000, C.BUFFER_FLAG_KEY_FRAME))); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(fakeMediaPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 0); + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + + assertThat(speedProviderMediaPeriod.getBufferedPositionUs()).isEqualTo(2_500_000); + } + + @Test + public void getNextLoadPositionUs_returnsAdjustedPosition() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 2_500_000, C.BUFFER_FLAG_KEY_FRAME))); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(fakeMediaPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 0); + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + + assertThat(speedProviderMediaPeriod.getNextLoadPositionUs()).isEqualTo(3_250_000); + } + + @Test + public void prepare_isForwardedWithAdjustedTime() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(spyPeriod, speedProvider); + + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 250_000); + + verify(spyPeriod).prepare(any(), eq(125_000L)); + } + + @Test + public void discardBuffer_isForwardedWithAdjustedTime() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 8000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(spyPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 3_250_000); + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + + speedProviderMediaPeriod.discardBuffer(/* positionUs= */ 3_250_000, /* toKeyframe= */ true); + + verify(spyPeriod).discardBuffer(2_500_000, /* toKeyframe= */ true); + } + + @Test + public void discardBuffer_positionIsZero_isForwardedWithAdjustedTime() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 8000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(spyPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 0); + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + + speedProviderMediaPeriod.discardBuffer(/* positionUs= */ 0, /* toKeyframe= */ true); + + verify(spyPeriod).discardBuffer(0, /* toKeyframe= */ true); + } + + @Test + public void readDiscontinuity_isForwardedWithAdjustedTime() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 8000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + fakeMediaPeriod.setDiscontinuityPositionUs(1_000_000); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(spyPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 500_000); + + assertThat(speedProviderMediaPeriod.readDiscontinuity()).isEqualTo(2_000_000); + verify(spyPeriod).readDiscontinuity(); + } + + @Test + public void seekTo_isForwardedWithAdjustedTime() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 8000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(spyPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 2_000_000); + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + + long seekResultTimeUs = speedProviderMediaPeriod.seekToUs(/* positionUs= */ 3_000_000); + + verify(spyPeriod).seekToUs(2_000_000); + assertThat(seekResultTimeUs).isEqualTo(3_000_000); + } + + @Test + public void getAdjustedSeekPosition_isForwardedWithAdjustedTime() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 8000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + fakeMediaPeriod.setSeekToUsOffset(2000); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(spyPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 2_000_000); + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + + long adjustedSeekPositionUs = + speedProviderMediaPeriod.getAdjustedSeekPositionUs( + /* positionUs= */ 2_000_000, SeekParameters.DEFAULT); + + verify(spyPeriod).getAdjustedSeekPositionUs(1_000_000, SeekParameters.DEFAULT); + assertThat(adjustedSeekPositionUs).isEqualTo(2_002_000); + } + + @Test + public void continueLoading_isForwardedWithOriginalTime() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 8000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(spyPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 3_250_000); + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + + speedProviderMediaPeriod.continueLoading( + new LoadingInfo.Builder().setPlaybackPositionUs(3_250_000).build()); + + verify(spyPeriod) + .continueLoading(new LoadingInfo.Builder().setPlaybackPositionUs(2_500_000).build()); + } + + @Test + public void reevaluateBuffer_isForwardedWithOriginalTime() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 8000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + SpeedProviderMediaPeriod speedProviderMediaPeriod = + new SpeedProviderMediaPeriod(spyPeriod, speedProvider); + prepareMediaPeriodSync(speedProviderMediaPeriod, /* positionUs= */ 3_250_000); + selectTracksOnMediaPeriodAndTriggerLoading(speedProviderMediaPeriod); + + speedProviderMediaPeriod.reevaluateBuffer(/* positionUs= */ 3_250_000); + + verify(spyPeriod).reevaluateBuffer(2_500_000); + } + + private static FakeMediaPeriod createFakeMediaPeriod( + ImmutableList sampleStreamItems) { + EventDispatcher eventDispatcher = + new EventDispatcher() + .withParameters(/* windowIndex= */ 0, new MediaPeriodId(/* periodUid= */ new Object())); + return new FakeMediaPeriod( + new TrackGroupArray(new TrackGroup(new Format.Builder().build())), + new DefaultAllocator(/* trimOnReset= */ false, /* individualAllocationSize= */ 1024), + (unusedFormat, unusedMediaPeriodId) -> sampleStreamItems, + eventDispatcher, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* deferOnPrepared= */ false); + } + + private static void prepareMediaPeriodSync(MediaPeriod mediaPeriod, long positionUs) + throws Exception { + CountDownLatch prepareCountDown = new CountDownLatch(1); + mediaPeriod.prepare( + new MediaPeriod.Callback() { + @Override + public void onPrepared(MediaPeriod mediaPeriod) { + prepareCountDown.countDown(); + } + + @Override + public void onContinueLoadingRequested(MediaPeriod source) { + mediaPeriod.continueLoading(new LoadingInfo.Builder().setPlaybackPositionUs(0).build()); + } + }, + positionUs); + prepareCountDown.await(); + } + + private static SampleStream selectTracksOnMediaPeriodAndTriggerLoading(MediaPeriod mediaPeriod) { + ExoTrackSelection selection = + new FixedTrackSelection(mediaPeriod.getTrackGroups().get(0), /* track= */ 0); + SampleStream[] streams = new SampleStream[1]; + mediaPeriod.selectTracks( + /* selections= */ new ExoTrackSelection[] {selection}, + /* mayRetainStreamFlags= */ new boolean[] {false}, + streams, + /* streamResetFlags= */ new boolean[] {false}, + /* positionUs= */ 0); + mediaPeriod.continueLoading(new LoadingInfo.Builder().setPlaybackPositionUs(0).build()); + return streams[0]; + } +}