From a879bc2154a01a11eb6ad0384a8c77ba93a5cb0e Mon Sep 17 00:00:00 2001 From: claincly Date: Mon, 16 Sep 2024 05:52:19 -0700 Subject: [PATCH] Rewrite sequence duration matching on MediaSource level This simplifies the later handling of speed adjusted media. PiperOrigin-RevId: 675114587 --- .../media3/transformer/CompositionPlayer.java | 182 +++++++----------- .../transformer/SequenceRenderersFactory.java | 25 ++- 2 files changed, 94 insertions(+), 113 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java index a8b1e26337..27009c1450 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java @@ -342,7 +342,6 @@ public final class CompositionPlayer extends SimpleBasePlayer && composition.sequences.size() <= MAX_SUPPORTED_SEQUENCES); checkState(this.composition == null); composition = deactivateSpeedAdjustingVideoEffects(composition); - composition = modifySequencesToSameDuration(composition); setCompositionInternal(composition); if (videoOutput != null) { @@ -678,6 +677,8 @@ public final class CompositionPlayer extends SimpleBasePlayer .build(); playbackVideoGraphWrapper.addListener(this); + long primarySequenceDurationUs = + getSequenceDurationUs(checkNotNull(composition.sequences.get(0))); // Video playback is disabled when one EditedMediaItem removes video. boolean disableVideoPlayback = shouldDisableVideoPlayback(composition); for (int i = 0; i < composition.sequences.size(); i++) { @@ -709,7 +710,13 @@ public final class CompositionPlayer extends SimpleBasePlayer player.addListener(new PlayerListener(i)); player.addAnalyticsListener(new EventLogger()); player.setPauseAtEndOfMediaItems(true); - setPlayerSequence(player, editedMediaItemSequence, /* shouldGenerateSilence= */ i == 0); + + if (i == 0) { + setPrimaryPlayerSequence(player, editedMediaItemSequence); + } else { + setSecondaryPlayerSequence(player, editedMediaItemSequence, primarySequenceDurationUs); + } + players.add(player); if (i == 0) { // Invalidate the player state before initializing the playlist to force SimpleBasePlayer @@ -732,9 +739,7 @@ public final class CompositionPlayer extends SimpleBasePlayer compositionInternalListenerHandler); } - /** Sets the {@linkplain EditedMediaItemSequence sequence} to be played by the player. */ - private void setPlayerSequence( - ExoPlayer player, EditedMediaItemSequence sequence, boolean shouldGenerateSilence) { + private void setPrimaryPlayerSequence(ExoPlayer player, EditedMediaItemSequence sequence) { ConcatenatingMediaSource2.Builder mediaSourceBuilder = new ConcatenatingMediaSource2.Builder().useDefaultMediaSourceFactory(context); @@ -742,39 +747,67 @@ public final class CompositionPlayer extends SimpleBasePlayer EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(i); checkArgument(editedMediaItem.durationUs != C.TIME_UNSET); long durationUs = editedMediaItem.getPresentationDurationUs(); - - if (shouldGenerateSilence) { - DefaultMediaSourceFactory defaultMediaSourceFactory = - new DefaultMediaSourceFactory(context); - if (externalImageLoader != null) { - defaultMediaSourceFactory.setExternalImageLoader(externalImageLoader); - } - MediaSource silenceMediaSource = - new ClippingMediaSource( - new SilenceMediaSource(editedMediaItem.durationUs), - editedMediaItem.mediaItem.clippingConfiguration.startPositionUs, - editedMediaItem.mediaItem.clippingConfiguration.endPositionUs); - - // The MediaSource that loads the MediaItem - MediaSource mainMediaSource = - defaultMediaSourceFactory.createMediaSource(editedMediaItem.mediaItem); - if (editedMediaItem.removeAudio) { - mainMediaSource = - new FilteringMediaSource( - mainMediaSource, ImmutableSet.of(C.TRACK_TYPE_VIDEO, C.TRACK_TYPE_IMAGE)); - } - - MediaSource mergingMediaSource = - new MergingMediaSource(mainMediaSource, silenceMediaSource); - MediaSource itemMediaSource = - wrapWithVideoEffectsBasedMediaSources( - mergingMediaSource, editedMediaItem.effects.videoEffects, durationUs); - mediaSourceBuilder.add( - itemMediaSource, /* initialPlaceholderDurationMs= */ usToMs(durationUs)); - } else { - mediaSourceBuilder.add( - editedMediaItem.mediaItem, /* initialPlaceholderDurationMs= */ usToMs(durationUs)); + // Generate silence for primary sequence. + DefaultMediaSourceFactory defaultMediaSourceFactory = new DefaultMediaSourceFactory(context); + if (externalImageLoader != null) { + defaultMediaSourceFactory.setExternalImageLoader(externalImageLoader); } + MediaSource silenceMediaSource = + new ClippingMediaSource( + new SilenceMediaSource(editedMediaItem.durationUs), + editedMediaItem.mediaItem.clippingConfiguration.startPositionUs, + editedMediaItem.mediaItem.clippingConfiguration.endPositionUs); + + // The MediaSource that loads the MediaItem + MediaSource mainMediaSource = + defaultMediaSourceFactory.createMediaSource(editedMediaItem.mediaItem); + if (editedMediaItem.removeAudio) { + mainMediaSource = + new FilteringMediaSource( + mainMediaSource, ImmutableSet.of(C.TRACK_TYPE_VIDEO, C.TRACK_TYPE_IMAGE)); + } + + MediaSource mergingMediaSource = new MergingMediaSource(mainMediaSource, silenceMediaSource); + MediaSource itemMediaSource = + wrapWithVideoEffectsBasedMediaSources( + mergingMediaSource, editedMediaItem.effects.videoEffects, durationUs); + mediaSourceBuilder.add( + itemMediaSource, /* initialPlaceholderDurationMs= */ usToMs(durationUs)); + } + player.setMediaSource(mediaSourceBuilder.build()); + } + + private void setSecondaryPlayerSequence( + ExoPlayer player, EditedMediaItemSequence sequence, long primarySequenceDurationUs) { + + // TODO: b/331392198 - Repeat only looping sequences, after sequences can be of arbitrary + // length. + ConcatenatingMediaSource2.Builder mediaSourceBuilder = + new ConcatenatingMediaSource2.Builder().useDefaultMediaSourceFactory(context); + + long accumulatedDurationUs = 0; + int i = 0; + while (accumulatedDurationUs < primarySequenceDurationUs) { + EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(i); + long itemPresentationDurationUs = editedMediaItem.getPresentationDurationUs(); + if (accumulatedDurationUs + itemPresentationDurationUs <= primarySequenceDurationUs) { + mediaSourceBuilder.add( + editedMediaItem.mediaItem, + /* initialPlaceholderDurationMs= */ usToMs(itemPresentationDurationUs)); + accumulatedDurationUs += itemPresentationDurationUs; + } else { + MediaItem mediaItem = editedMediaItem.mediaItem; + long remainingDurationUs = primarySequenceDurationUs - accumulatedDurationUs; + // TODO: b/289989542 - Handle already clipped, or speed adjusted media. + mediaSourceBuilder.add( + new ClippingMediaSource( + new DefaultMediaSourceFactory(context).createMediaSource(mediaItem), + mediaItem.clippingConfiguration.startPositionUs, + mediaItem.clippingConfiguration.startPositionUs + remainingDurationUs), + /* initialPlaceholderDurationMs= */ usToMs(remainingDurationUs)); + break; + } + i = (i + 1) % sequence.editedMediaItems.size(); } player.setMediaSource(mediaSourceBuilder.build()); } @@ -945,19 +978,7 @@ public final class CompositionPlayer extends SimpleBasePlayer private static long getCompositionDurationUs(Composition composition) { checkState(!composition.sequences.isEmpty()); - - long compositionDurationUs = getSequenceDurationUs(composition.sequences.get(0)); - for (int i = 0; i < composition.sequences.size(); i++) { - long sequenceDurationUs = getSequenceDurationUs(composition.sequences.get(i)); - checkArgument( - compositionDurationUs == sequenceDurationUs, - Util.formatInvariant( - "Non-matching sequence durations. First sequence duration: %d us, sequence [%d]" - + " duration: %d us", - compositionDurationUs, i, sequenceDurationUs)); - } - - return compositionDurationUs; + return getSequenceDurationUs(composition.sequences.get(0)); } private static long getSequenceDurationUs(EditedMediaItemSequence sequence) { @@ -982,67 +1003,6 @@ public final class CompositionPlayer extends SimpleBasePlayer return false; } - /** - * Repeats or clips the non-zero indexed sequence to match the duration of the zero indexed - * sequence (the primary sequence). - */ - private static Composition modifySequencesToSameDuration(Composition composition) { - EditedMediaItemSequence primarySequence = composition.sequences.get(0); - checkArgument( - !primarySequence.isLooping, - "CompositionPlayer doesn't support looping the first sequence."); - long primarySequenceDurationUs = getSequenceDurationUs(primarySequence); - ArrayList rebuiltSequences = - new ArrayList<>(composition.sequences.size()); - rebuiltSequences.add(primarySequence); - - // TODO: b/331392198 - Repeat only looping sequences, after sequences can be of arbitrary - // length. - for (int i = 1; i < composition.sequences.size(); i++) { - EditedMediaItemSequence sequence = composition.sequences.get(i); - long sequenceDurationUs = getSequenceDurationUs(sequence); - if (sequenceDurationUs == primarySequenceDurationUs) { - rebuiltSequences.add(sequence); - continue; - } - - ArrayList repeatedEditedMediaItems = new ArrayList<>(); - long repeatSequenceTimes = primarySequenceDurationUs / sequenceDurationUs; - for (int j = 0; j < repeatSequenceTimes; j++) { - repeatedEditedMediaItems.addAll(sequence.editedMediaItems); - } - - long remainingDurationUs = - primarySequenceDurationUs - repeatSequenceTimes * sequenceDurationUs; - for (int j = 0; j < sequence.editedMediaItems.size(); j++) { - EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(j); - if (editedMediaItem.getPresentationDurationUs() <= remainingDurationUs) { - remainingDurationUs -= editedMediaItem.getPresentationDurationUs(); - repeatedEditedMediaItems.add(editedMediaItem); - } else { - // TODO: b/289989542 - Handle already clipped, or speed adjusted media. - checkState(editedMediaItem.getPresentationDurationUs() == editedMediaItem.durationUs); - repeatedEditedMediaItems.add( - editedMediaItem - .buildUpon() - .setMediaItem( - editedMediaItem - .mediaItem - .buildUpon() - .setClippingConfiguration( - new MediaItem.ClippingConfiguration.Builder() - .setEndPositionUs(remainingDurationUs) - .build()) - .build()) - .build()); - break; - } - } - rebuiltSequences.add(new EditedMediaItemSequence(repeatedEditedMediaItems)); - } - return composition.buildUpon().setSequences(rebuiltSequences).build(); - } - /** * A {@link VideoFrameReleaseControl.FrameTimingEvaluator} for composition frames. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java index c635fd6da1..f028bd0eee 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java @@ -156,11 +156,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; sequence.editedMediaItems.get(0).mediaItem.clippingConfiguration.startPositionUs; } for (int i = 0; i < mediaItemIndex; i++) { - offsetToCompositionTimeUs += sequence.editedMediaItems.get(i).getPresentationDurationUs(); + offsetToCompositionTimeUs += + getRepeatedEditedMediaItem(sequence, i).getPresentationDurationUs(); } return offsetToCompositionTimeUs; } + /** + * Gets the {@link EditedMediaItem} of a given {@code index}. + * + *

The index could be greater than {@link EditedMediaItemSequence#editedMediaItems} because the + * sequence might be {@linkplain EditedMediaItemSequence#isLooping looping}. + */ + private static EditedMediaItem getRepeatedEditedMediaItem( + EditedMediaItemSequence sequence, int index) { + if (sequence.isLooping) { + index %= sequence.editedMediaItems.size(); + } + return sequence.editedMediaItems.get(index); + } + private static final class SequenceAudioRenderer extends MediaCodecAudioRenderer { private final EditedMediaItemSequence sequence; private final AudioGraphInputAudioSink audioSink; @@ -208,10 +223,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; MediaSource.MediaPeriodId mediaPeriodId) throws ExoPlaybackException { checkState(getTimeline().getWindowCount() == 1); + + // TODO: b/331392198 - Repeat only looping sequences, after sequences can be of arbitrary + // length. + // The media item might have been repeated in the sequence. int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid); // We must first update the pending media item state before calling super.onStreamChanged() // because the super method will call onProcessedStreamChange() - pendingEditedMediaItem = sequence.editedMediaItems.get(mediaItemIndex); + pendingEditedMediaItem = getRepeatedEditedMediaItem(sequence, mediaItemIndex); pendingOffsetToCompositionTimeUs = getOffsetToCompositionTimeUs(sequence, mediaItemIndex, offsetUs); super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); @@ -279,6 +298,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throws ExoPlaybackException { checkState(getTimeline().getWindowCount() == 1); super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); + // The media item might have been repeated in the sequence. int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid); offsetToCompositionTimeUs = getOffsetToCompositionTimeUs(sequence, mediaItemIndex, offsetUs); pendingEffect = sequence.editedMediaItems.get(mediaItemIndex).effects.videoEffects; @@ -407,6 +427,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkState(getTimeline().getWindowCount() == 1); super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); streamStartPositionUs = startPositionUs; + // The media item might have been repeated in the sequence. int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid); editedMediaItem = sequence.editedMediaItems.get(mediaItemIndex); offsetToCompositionTimeUs = getOffsetToCompositionTimeUs(sequence, mediaItemIndex, offsetUs);