loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Rewrite sequence duration matching on MediaSource level

Browse Source 
This simplifies the later handling of speed adjusted media.

PiperOrigin-RevId: 675114587
This commit is contained in:
claincly 2024-09-16 05:52:19 -07:00 committed by Copybara-Service
parent e938d27846
commit a879bc2154
2 changed files with 94 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java
View File

@ -342,7 +342,6 @@ public final class CompositionPlayer extends SimpleBasePlayer
&& composition.sequences.size() <= MAX_SUPPORTED_SEQUENCES);
checkState(this.composition == null);
composition = deactivateSpeedAdjustingVideoEffects(composition);
composition = modifySequencesToSameDuration(composition);
setCompositionInternal(composition);
if (videoOutput != null) {
@ -678,6 +677,8 @@ public final class CompositionPlayer extends SimpleBasePlayer
.build();
playbackVideoGraphWrapper.addListener(this);
long primarySequenceDurationUs =
getSequenceDurationUs(checkNotNull(composition.sequences.get(0)));
// Video playback is disabled when one EditedMediaItem removes video.
boolean disableVideoPlayback = shouldDisableVideoPlayback(composition);
for (int i = 0; i < composition.sequences.size(); i++) {
@ -709,7 +710,13 @@ public final class CompositionPlayer extends SimpleBasePlayer
player.addListener(new PlayerListener(i));
player.addAnalyticsListener(new EventLogger());
player.setPauseAtEndOfMediaItems(true);
setPlayerSequence(player, editedMediaItemSequence, /* shouldGenerateSilence= */ i == 0);
if (i == 0) {
setPrimaryPlayerSequence(player, editedMediaItemSequence);
} else {
setSecondaryPlayerSequence(player, editedMediaItemSequence, primarySequenceDurationUs);
}
players.add(player);
if (i == 0) {
// Invalidate the player state before initializing the playlist to force SimpleBasePlayer
@ -732,9 +739,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
compositionInternalListenerHandler);
}
/** Sets the {@linkplain EditedMediaItemSequence sequence} to be played by the player. */
private void setPlayerSequence(
ExoPlayer player, EditedMediaItemSequence sequence, boolean shouldGenerateSilence) {
private void setPrimaryPlayerSequence(ExoPlayer player, EditedMediaItemSequence sequence) {
ConcatenatingMediaSource2.Builder mediaSourceBuilder =
new ConcatenatingMediaSource2.Builder().useDefaultMediaSourceFactory(context);
@ -742,10 +747,8 @@ public final class CompositionPlayer extends SimpleBasePlayer
EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(i);
checkArgument(editedMediaItem.durationUs != C.TIME_UNSET);
long durationUs = editedMediaItem.getPresentationDurationUs();
if (shouldGenerateSilence) {
DefaultMediaSourceFactory defaultMediaSourceFactory =
new DefaultMediaSourceFactory(context);
// Generate silence for primary sequence.
DefaultMediaSourceFactory defaultMediaSourceFactory = new DefaultMediaSourceFactory(context);
if (externalImageLoader != null) {
defaultMediaSourceFactory.setExternalImageLoader(externalImageLoader);
}
@ -764,17 +767,47 @@ public final class CompositionPlayer extends SimpleBasePlayer
mainMediaSource, ImmutableSet.of(C.TRACK_TYPE_VIDEO, C.TRACK_TYPE_IMAGE));
}
MediaSource mergingMediaSource =
new MergingMediaSource(mainMediaSource, silenceMediaSource);
MediaSource mergingMediaSource = new MergingMediaSource(mainMediaSource, silenceMediaSource);
MediaSource itemMediaSource =
wrapWithVideoEffectsBasedMediaSources(
mergingMediaSource, editedMediaItem.effects.videoEffects, durationUs);
mediaSourceBuilder.add(
itemMediaSource, /* initialPlaceholderDurationMs= */ usToMs(durationUs));
} else {
mediaSourceBuilder.add(
editedMediaItem.mediaItem, /* initialPlaceholderDurationMs= */ usToMs(durationUs));
}
player.setMediaSource(mediaSourceBuilder.build());
}
private void setSecondaryPlayerSequence(
ExoPlayer player, EditedMediaItemSequence sequence, long primarySequenceDurationUs) {
// TODO: b/331392198 - Repeat only looping sequences, after sequences can be of arbitrary
// length.
ConcatenatingMediaSource2.Builder mediaSourceBuilder =
new ConcatenatingMediaSource2.Builder().useDefaultMediaSourceFactory(context);
long accumulatedDurationUs = 0;
int i = 0;
while (accumulatedDurationUs < primarySequenceDurationUs) {
EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(i);
long itemPresentationDurationUs = editedMediaItem.getPresentationDurationUs();
if (accumulatedDurationUs + itemPresentationDurationUs <= primarySequenceDurationUs) {
mediaSourceBuilder.add(
editedMediaItem.mediaItem,
/* initialPlaceholderDurationMs= */ usToMs(itemPresentationDurationUs));
accumulatedDurationUs += itemPresentationDurationUs;
} else {
MediaItem mediaItem = editedMediaItem.mediaItem;
long remainingDurationUs = primarySequenceDurationUs - accumulatedDurationUs;
// TODO: b/289989542 - Handle already clipped, or speed adjusted media.
mediaSourceBuilder.add(
new ClippingMediaSource(
new DefaultMediaSourceFactory(context).createMediaSource(mediaItem),
mediaItem.clippingConfiguration.startPositionUs,
mediaItem.clippingConfiguration.startPositionUs + remainingDurationUs),
/* initialPlaceholderDurationMs= */ usToMs(remainingDurationUs));
break;
}
i = (i + 1) % sequence.editedMediaItems.size();
}
player.setMediaSource(mediaSourceBuilder.build());
}
@ -945,19 +978,7 @@ public final class CompositionPlayer extends SimpleBasePlayer
private static long getCompositionDurationUs(Composition composition) {
checkState(!composition.sequences.isEmpty());
long compositionDurationUs = getSequenceDurationUs(composition.sequences.get(0));
for (int i = 0; i < composition.sequences.size(); i++) {
long sequenceDurationUs = getSequenceDurationUs(composition.sequences.get(i));
checkArgument(
compositionDurationUs == sequenceDurationUs,
Util.formatInvariant(
"Non-matching sequence durations. First sequence duration: %d us, sequence [%d]"
+ " duration: %d us",
compositionDurationUs, i, sequenceDurationUs));
}
return compositionDurationUs;
return getSequenceDurationUs(composition.sequences.get(0));
}
private static long getSequenceDurationUs(EditedMediaItemSequence sequence) {
@ -982,67 +1003,6 @@ public final class CompositionPlayer extends SimpleBasePlayer
return false;
}
/**
* Repeats or clips the non-zero indexed sequence to match the duration of the zero indexed
* sequence (the primary sequence).
*/
private static Composition modifySequencesToSameDuration(Composition composition) {
EditedMediaItemSequence primarySequence = composition.sequences.get(0);
checkArgument(
!primarySequence.isLooping,
"CompositionPlayer doesn't support looping the first sequence.");
long primarySequenceDurationUs = getSequenceDurationUs(primarySequence);
ArrayList<EditedMediaItemSequence> rebuiltSequences =
new ArrayList<>(composition.sequences.size());
rebuiltSequences.add(primarySequence);
// TODO: b/331392198 - Repeat only looping sequences, after sequences can be of arbitrary
// length.
for (int i = 1; i < composition.sequences.size(); i++) {
EditedMediaItemSequence sequence = composition.sequences.get(i);
long sequenceDurationUs = getSequenceDurationUs(sequence);
if (sequenceDurationUs == primarySequenceDurationUs) {
rebuiltSequences.add(sequence);
continue;
}
ArrayList<EditedMediaItem> repeatedEditedMediaItems = new ArrayList<>();
long repeatSequenceTimes = primarySequenceDurationUs / sequenceDurationUs;
for (int j = 0; j < repeatSequenceTimes; j++) {
repeatedEditedMediaItems.addAll(sequence.editedMediaItems);
}
long remainingDurationUs =
primarySequenceDurationUs - repeatSequenceTimes * sequenceDurationUs;
for (int j = 0; j < sequence.editedMediaItems.size(); j++) {
EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(j);
if (editedMediaItem.getPresentationDurationUs() <= remainingDurationUs) {
remainingDurationUs -= editedMediaItem.getPresentationDurationUs();
repeatedEditedMediaItems.add(editedMediaItem);
} else {
// TODO: b/289989542 - Handle already clipped, or speed adjusted media.
checkState(editedMediaItem.getPresentationDurationUs() == editedMediaItem.durationUs);
repeatedEditedMediaItems.add(
editedMediaItem
.buildUpon()
.setMediaItem(
editedMediaItem
.mediaItem
.buildUpon()
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setEndPositionUs(remainingDurationUs)
.build())
.build())
.build());
break;
}
}
rebuiltSequences.add(new EditedMediaItemSequence(repeatedEditedMediaItems));
}
return composition.buildUpon().setSequences(rebuiltSequences).build();
}
/**
* A {@link VideoFrameReleaseControl.FrameTimingEvaluator} for composition frames.
*

25
libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java
View File

@ -156,11 +156,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
sequence.editedMediaItems.get(0).mediaItem.clippingConfiguration.startPositionUs;
}
for (int i = 0; i < mediaItemIndex; i++) {
offsetToCompositionTimeUs += sequence.editedMediaItems.get(i).getPresentationDurationUs();
offsetToCompositionTimeUs +=
getRepeatedEditedMediaItem(sequence, i).getPresentationDurationUs();
}
return offsetToCompositionTimeUs;
}
/**
* Gets the {@link EditedMediaItem} of a given {@code index}.
*
* <p>The index could be greater than {@link EditedMediaItemSequence#editedMediaItems} because the
* sequence might be {@linkplain EditedMediaItemSequence#isLooping looping}.
*/
private static EditedMediaItem getRepeatedEditedMediaItem(
EditedMediaItemSequence sequence, int index) {
if (sequence.isLooping) {
index %= sequence.editedMediaItems.size();
}
return sequence.editedMediaItems.get(index);
}
private static final class SequenceAudioRenderer extends MediaCodecAudioRenderer {
private final EditedMediaItemSequence sequence;
private final AudioGraphInputAudioSink audioSink;
@ -208,10 +223,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
MediaSource.MediaPeriodId mediaPeriodId)
throws ExoPlaybackException {
checkState(getTimeline().getWindowCount() == 1);
// TODO: b/331392198 - Repeat only looping sequences, after sequences can be of arbitrary
// length.
// The media item might have been repeated in the sequence.
int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid);
// We must first update the pending media item state before calling super.onStreamChanged()
// because the super method will call onProcessedStreamChange()
pendingEditedMediaItem = sequence.editedMediaItems.get(mediaItemIndex);
pendingEditedMediaItem = getRepeatedEditedMediaItem(sequence, mediaItemIndex);
pendingOffsetToCompositionTimeUs =
getOffsetToCompositionTimeUs(sequence, mediaItemIndex, offsetUs);
super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId);
@ -279,6 +298,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
throws ExoPlaybackException {
checkState(getTimeline().getWindowCount() == 1);
super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId);
// The media item might have been repeated in the sequence.
int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid);
offsetToCompositionTimeUs = getOffsetToCompositionTimeUs(sequence, mediaItemIndex, offsetUs);
pendingEffect = sequence.editedMediaItems.get(mediaItemIndex).effects.videoEffects;
@ -407,6 +427,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
checkState(getTimeline().getWindowCount() == 1);
super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId);
streamStartPositionUs = startPositionUs;
// The media item might have been repeated in the sequence.
int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid);
editedMediaItem = sequence.editedMediaItems.get(mediaItemIndex);
offsetToCompositionTimeUs = getOffsetToCompositionTimeUs(sequence, mediaItemIndex, offsetUs);