From 2b031484fe84562994915d5d867226672d7238a5 Mon Sep 17 00:00:00 2001 From: claincly Date: Fri, 23 Aug 2024 07:50:46 -0700 Subject: [PATCH] Allow looping sequences in CompositionPlayer PiperOrigin-RevId: 666793658 --- ...LoopingSequence_outputsCorrectSamples.dump | 24 +++++++ ...LoopingSequence_outputsCorrectSamples.dump | 54 ++++++++++++++++ .../media3/transformer/CompositionPlayer.java | 62 +++++++++++++++++++ .../CompositionPlayerAudioPlaybackTest.java | 60 ++++++++++++++++++ .../transformer/CompositionPlayerTest.java | 18 ------ 5 files changed, 200 insertions(+), 18 deletions(-) create mode 100644 libraries/test_data/src/test/assets/audiosinkdumps/wav/compositionPlayback_withLongLoopingSequence_outputsCorrectSamples.dump create mode 100644 libraries/test_data/src/test/assets/audiosinkdumps/wav/compositionPlayback_withShortLoopingSequence_outputsCorrectSamples.dump diff --git a/libraries/test_data/src/test/assets/audiosinkdumps/wav/compositionPlayback_withLongLoopingSequence_outputsCorrectSamples.dump b/libraries/test_data/src/test/assets/audiosinkdumps/wav/compositionPlayback_withLongLoopingSequence_outputsCorrectSamples.dump new file mode 100644 index 0000000000..0e744f2ca5 --- /dev/null +++ b/libraries/test_data/src/test/assets/audiosinkdumps/wav/compositionPlayback_withLongLoopingSequence_outputsCorrectSamples.dump @@ -0,0 +1,24 @@ +AudioSink: + buffer count = 6 + config: + pcmEncoding = 2 + channelCount = 1 + sampleRate = 44100 + buffer #0: + time = 0 + data = -419876658 + buffer #1: + time = 100000 + data = -1236081112 + buffer #2: + time = 200000 + data = -1630460924 + buffer #3: + time = 300000 + data = 1478130841 + buffer #4: + time = 348616 + data = -2449 + buffer #5: + time = 348639 + data = -68458611 diff --git a/libraries/test_data/src/test/assets/audiosinkdumps/wav/compositionPlayback_withShortLoopingSequence_outputsCorrectSamples.dump b/libraries/test_data/src/test/assets/audiosinkdumps/wav/compositionPlayback_withShortLoopingSequence_outputsCorrectSamples.dump new file mode 100644 index 0000000000..763cd8c111 --- /dev/null +++ b/libraries/test_data/src/test/assets/audiosinkdumps/wav/compositionPlayback_withShortLoopingSequence_outputsCorrectSamples.dump @@ -0,0 +1,54 @@ +AudioSink: + buffer count = 16 + config: + pcmEncoding = 2 + channelCount = 1 + sampleRate = 44100 + buffer #0: + time = 0 + data = -419876658 + buffer #1: + time = 100000 + data = -1236081112 + buffer #2: + time = 200000 + data = -1630460924 + buffer #3: + time = 300000 + data = 1478130841 + buffer #4: + time = 348616 + data = -2449 + buffer #5: + time = 348639 + data = 590036013 + buffer #6: + time = 448639 + data = -61907402 + buffer #7: + time = 500000 + data = -404977619 + buffer #8: + time = 648639 + data = -1276039913 + buffer #9: + time = 697256 + data = -1085 + buffer #10: + time = 697278 + data = -317156192 + buffer #11: + time = 797278 + data = -1765342951 + buffer #12: + time = 897278 + data = 1454848200 + buffer #13: + time = 997278 + data = -111836408 + buffer #14: + time = 1000000 + data = -1471531958 + buffer #15: + time = 1045895 + data = -2263 diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java index fc85bd57e6..29d46e1d02 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java @@ -342,6 +342,7 @@ public final class CompositionPlayer extends SimpleBasePlayer && composition.sequences.size() <= MAX_SUPPORTED_SEQUENCES); checkState(this.composition == null); composition = deactivateSpeedAdjustingVideoEffects(composition); + composition = modifySequencesToSameDuration(composition); setCompositionInternal(composition); if (videoOutput != null) { @@ -980,6 +981,67 @@ public final class CompositionPlayer extends SimpleBasePlayer return false; } + /** + * Repeats or clips the non-zero indexed sequence to match the duration of the zero indexed + * sequence (the primary sequence). + */ + private static Composition modifySequencesToSameDuration(Composition composition) { + EditedMediaItemSequence primarySequence = composition.sequences.get(0); + checkArgument( + !primarySequence.isLooping, + "CompositionPlayer doesn't support looping the first sequence."); + long primarySequenceDurationUs = getSequenceDurationUs(primarySequence); + ArrayList rebuiltSequences = + new ArrayList<>(composition.sequences.size()); + rebuiltSequences.add(primarySequence); + + // TODO: b/331392198 - Repeat only looping sequences, after sequences can be of arbitrary + // length. + for (int i = 1; i < composition.sequences.size(); i++) { + EditedMediaItemSequence sequence = composition.sequences.get(i); + long sequenceDurationUs = getSequenceDurationUs(sequence); + if (sequenceDurationUs == primarySequenceDurationUs) { + rebuiltSequences.add(sequence); + continue; + } + + ArrayList repeatedEditedMediaItems = new ArrayList<>(); + long repeatSequenceTimes = primarySequenceDurationUs / sequenceDurationUs; + for (int j = 0; j < repeatSequenceTimes; j++) { + repeatedEditedMediaItems.addAll(sequence.editedMediaItems); + } + + long remainingDurationUs = + primarySequenceDurationUs - repeatSequenceTimes * sequenceDurationUs; + for (int j = 0; j < sequence.editedMediaItems.size(); j++) { + EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(j); + if (editedMediaItem.getPresentationDurationUs() <= remainingDurationUs) { + remainingDurationUs -= editedMediaItem.getPresentationDurationUs(); + repeatedEditedMediaItems.add(editedMediaItem); + } else { + // TODO: b/289989542 - Handle already clipped, or speed adjusted media. + checkState(editedMediaItem.getPresentationDurationUs() == editedMediaItem.durationUs); + repeatedEditedMediaItems.add( + editedMediaItem + .buildUpon() + .setMediaItem( + editedMediaItem + .mediaItem + .buildUpon() + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setEndPositionUs(remainingDurationUs) + .build()) + .build()) + .build()); + break; + } + } + rebuiltSequences.add(new EditedMediaItemSequence(repeatedEditedMediaItems)); + } + return composition.buildUpon().setSequences(rebuiltSequences).build(); + } + /** * A {@link VideoFrameReleaseControl.FrameTimingEvaluator} for composition frames. * diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerAudioPlaybackTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerAudioPlaybackTest.java index 95087c5c14..59b8d43685 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerAudioPlaybackTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerAudioPlaybackTest.java @@ -34,6 +34,7 @@ import androidx.media3.test.utils.FakeClock; import androidx.media3.test.utils.robolectric.TestPlayerRunHelper; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -264,6 +265,65 @@ public final class CompositionPlayerAudioPlaybackTest { context, capturingAudioSink, "audiosinkdumps/wav/sample.wav_repeated.dump"); } + @Test + public void compositionPlayback_withShortLoopingSequence_outputsCorrectSamples() + throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItemSequence primarySequence = + new EditedMediaItemSequence( + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .build()); + EditedMediaItemSequence loopingSequence = + new EditedMediaItemSequence( + ImmutableList.of( + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ)) + .setDurationUs(348_000L) + .build()), + /* isLooping= */ true); + Composition composition = new Composition.Builder(primarySequence, loopingSequence).build(); + player.setComposition(composition); + player.prepare(); + player.play(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, + capturingAudioSink, + "audiosinkdumps/wav/compositionPlayback_withShortLoopingSequence_outputsCorrectSamples.dump"); + } + + @Test + public void compositionPlayback_withLongLoopingSequence_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItemSequence primarySequence = + new EditedMediaItemSequence( + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ)) + .setDurationUs(348_000L) + .build()); + EditedMediaItemSequence loopingSequence = + new EditedMediaItemSequence( + ImmutableList.of( + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .build()), + /* isLooping= */ true); + Composition composition = new Composition.Builder(primarySequence, loopingSequence).build(); + player.setComposition(composition); + player.prepare(); + player.play(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, + capturingAudioSink, + "audiosinkdumps/wav/compositionPlayback_withLongLoopingSequence_outputsCorrectSamples.dump"); + } + @Test public void sequencePlayback_withOneRepeat_outputsCorrectSamples() throws Exception { CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerTest.java index 88e9799611..216591d504 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerTest.java @@ -281,24 +281,6 @@ public class CompositionPlayerTest { player.release(); } - @Test - public void setComposition_unmatchingDurations_throws() { - CompositionPlayer player = buildCompositionPlayer(); - - Composition composition = - new Composition.Builder( - ImmutableList.of( - new EditedMediaItemSequence( - new EditedMediaItem.Builder(MediaItem.EMPTY).setDurationUs(1).build()), - new EditedMediaItemSequence( - new EditedMediaItem.Builder(MediaItem.EMPTY).setDurationUs(2).build()))) - .build(); - - assertThrows(IllegalArgumentException.class, () -> player.setComposition(composition)); - - player.release(); - } - @Test public void prepare_withoutCompositionSet_throws() { CompositionPlayer player = buildCompositionPlayer();