loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

CompositionPlayer: skip decode-only frames upstream of the VideoGraph

Browse Source 
This is necessary for prewarming. With prewarming, in a sequence of 2
videos, the second renderer is enabled before the first one is disabled,
and decode-only frames should be skipped before the second renderer is
started. The problem is that the second renderer will forward frames to
a BufferingVideoSink before it is started, which  will delay the frame
handling and therefore not skip the frame before the renderer is
started.

PiperOrigin-RevId: 721032049
This commit is contained in:
kimvde 2025-01-29 10:42:39 -08:00 committed by Copybara-Service
parent 3c0e2ee198
commit 9af43c7381
6 changed files with 28 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java
View File

@ -173,8 +173,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void setStreamTimestampInfo(
long streamStartPositionUs, long bufferTimestampAdjustmentUs, long lastResetPositionUs) {
public void setStreamTimestampInfo(long streamStartPositionUs, long bufferTimestampAdjustmentUs) {
if (streamStartPositionUs != this.streamStartPositionUs) {
videoFrameRenderControl.onStreamStartPositionChanged(streamStartPositionUs);
this.streamStartPositionUs = streamStartPositionUs;

13
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -814,9 +814,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
videoSink.flush(/* resetPosition= */ true);
}
videoSink.setStreamTimestampInfo(
getOutputStreamStartPositionUs(),
getBufferTimestampAdjustmentUs(),
getLastResetPositionUs());
getOutputStreamStartPositionUs(), getBufferTimestampAdjustmentUs());
pendingVideoSinkInputStreamChange = true;
}
super.onPositionReset(positionUs, joining);
@ -1480,6 +1478,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
if (videoSink != null) {
// Skip decode-only buffers, e.g. after seeking, immediately.
if (isDecodeOnlyBuffer && !isLastBuffer) {
skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
return true;
}
long framePresentationTimeUs = bufferPresentationTimeUs + getBufferTimestampAdjustmentUs();
return videoSink.handleInputFrame(
framePresentationTimeUs,
@ -1631,9 +1634,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
// Signaling end of the previous stream.
videoSink.signalEndOfCurrentInputStream();
videoSink.setStreamTimestampInfo(
getOutputStreamStartPositionUs(),
getBufferTimestampAdjustmentUs(),
getLastResetPositionUs());
getOutputStreamStartPositionUs(), getBufferTimestampAdjustmentUs());
} else {
videoFrameReleaseControl.onProcessedStreamChange();
}

41
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java
View File

@ -441,7 +441,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
if (newOutputStreamStartPositionUs != null
&& newOutputStreamStartPositionUs != outputStreamStartPositionUs) {
defaultVideoSink.setStreamTimestampInfo(
newOutputStreamStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET);
newOutputStreamStartPositionUs, bufferTimestampAdjustmentUs);
outputStreamStartPositionUs = newOutputStreamStartPositionUs;
}
boolean isLastFrame =
@ -588,8 +588,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
if (streamStartPositionsUs.size() == 1) {
long lastStartPositionUs = checkNotNull(streamStartPositionsUs.pollFirst());
// defaultVideoSink should use the latest startPositionUs if none is passed after flushing.
defaultVideoSink.setStreamTimestampInfo(
lastStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET);
defaultVideoSink.setStreamTimestampInfo(lastStartPositionUs, bufferTimestampAdjustmentUs);
}
lastOutputBufferPresentationTimeUs = C.TIME_UNSET;
finalBufferPresentationTimeUs = C.TIME_UNSET;
@ -611,7 +610,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
private void setBufferTimestampAdjustment(long bufferTimestampAdjustmentUs) {
this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs;
defaultVideoSink.setStreamTimestampInfo(
outputStreamStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET);
outputStreamStartPositionUs, bufferTimestampAdjustmentUs);
}
private boolean shouldRenderToInputVideoSink() {
@ -638,7 +637,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
@Nullable private Format inputFormat;
private @InputType int inputType;
private long inputBufferTimestampAdjustmentUs;
private long lastResetPositionUs;
/**
* The buffer presentation timestamp, in microseconds, of the most recently registered frame.
@ -796,7 +794,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
@Override
public void setStreamTimestampInfo(
long streamStartPositionUs, long bufferTimestampAdjustmentUs, long lastResetPositionUs) {
long streamStartPositionUs, long bufferTimestampAdjustmentUs) {
// Input timestamps should always be positive because they are offset by ExoPlayer. Adding a
// position to the queue with timestamp 0 should therefore always apply it as long as it is
// the only position in the queue.
@ -809,7 +807,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
// directly at the output of the VideoGraph because no frame has been input yet following the
// flush.
PlaybackVideoGraphWrapper.this.setBufferTimestampAdjustment(inputBufferTimestampAdjustmentUs);
this.lastResetPositionUs = lastResetPositionUs;
}
@Override
@ -837,26 +834,10 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
public boolean handleInputFrame(
long framePresentationTimeUs, boolean isLastFrame, VideoFrameHandler videoFrameHandler) {
checkState(isInitialized());
if (!shouldRenderToInputVideoSink()) {
return false;
}
// The sink takes in frames with monotonically increasing, non-offset frame
// timestamps. That is, with two ten-second long videos, the first frame of the second video
// should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the
// timestamp of the said frame would be 0s, but the streamOffset is incremented by 10s to
// include the duration of the first video. Thus this correction is needed to account for the
// different handling of presentation timestamps in ExoPlayer and VideoFrameProcessor.
//
// inputBufferTimestampAdjustmentUs adjusts the frame presentation time (which is relative to
// the start of a composition) to the buffer timestamp (that corresponds to the player
// position).
long bufferPresentationTimeUs = framePresentationTimeUs - inputBufferTimestampAdjustmentUs;
if (bufferPresentationTimeUs < lastResetPositionUs && !isLastFrame) {
videoFrameHandler.skip();
return true;
}
if (checkStateNotNull(videoFrameProcessor).getPendingInputFrameCount()
>= videoFrameProcessorMaxPendingFrameCount) {
return false;
@ -865,7 +846,17 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
return false;
}
lastBufferPresentationTimeUs = bufferPresentationTimeUs;
// The sink takes in frames with monotonically increasing, non-offset frame
// timestamps. That is, with two 10s long videos, the first frame of the second video should
// bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the timestamp of
// the said frame would be 0s, but the streamOffset is incremented by 10s to include the
// duration of the first video. Thus this correction is needed to account for the different
// handling of presentation timestamps in ExoPlayer and VideoFrameProcessor.
//
// inputBufferTimestampAdjustmentUs adjusts the frame presentation time (which is relative to
// the start of a composition) to the buffer timestamp (that corresponds to the player
// position).
lastBufferPresentationTimeUs = framePresentationTimeUs - inputBufferTimestampAdjustmentUs;
// Use the frame presentation time as render time so that the SurfaceTexture is accompanied
// by this timestamp. Setting a realtime based release time is only relevant when rendering to
// a SurfaceView, but we render to a surface in this case.

4
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java
View File

@ -217,10 +217,8 @@ public interface VideoSink {
* current stream, in microseconds.
* @param bufferTimestampAdjustmentUs The timestamp adjustment to add to the buffer presentation
* timestamps to convert them to frame presentation timestamps, in microseconds.
* @param lastResetPositionUs The renderer last reset position, in microseconds.
*/
void setStreamTimestampInfo(
long streamStartPositionUs, long bufferTimestampAdjustmentUs, long lastResetPositionUs);
void setStreamTimestampInfo(long streamStartPositionUs, long bufferTimestampAdjustmentUs);
/** Sets the output surface info. */
void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution);

6
libraries/transformer/src/main/java/androidx/media3/transformer/BufferingVideoSink.java
View File

@ -193,12 +193,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void setStreamTimestampInfo(
long streamStartPositionUs, long bufferTimestampAdjustmentUs, long lastResetPositionUs) {
public void setStreamTimestampInfo(long streamStartPositionUs, long bufferTimestampAdjustmentUs) {
executeOrDelay(
videoSink ->
videoSink.setStreamTimestampInfo(
streamStartPositionUs, bufferTimestampAdjustmentUs, lastResetPositionUs));
videoSink.setStreamTimestampInfo(streamStartPositionUs, bufferTimestampAdjustmentUs));
}
@Override

4
libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java
View File

@ -505,9 +505,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (inputStreamPending) {
checkState(streamStartPositionUs != C.TIME_UNSET);
videoSink.setStreamTimestampInfo(
streamStartPositionUs,
/* bufferTimestampAdjustmentUs= */ offsetToCompositionTimeUs,
getLastResetPositionUs());
streamStartPositionUs, /* bufferTimestampAdjustmentUs= */ offsetToCompositionTimeUs);
videoSink.onInputStreamChanged(
VideoSink.INPUT_TYPE_BITMAP,
new Format.Builder()