mirror of
https://github.com/androidx/media.git
synced 2025-04-30 06:46:50 +08:00
Various refactorings in VideoFrameRenderControl
- Rename a few methods/variable to improve readability. - Refactor how video size changes are tracked. This will simplify the upcoming logic to refactor the timestamp logic in VideoFrameRenderControl because we will use the same logic for outputVideoSize and for outputStreamStartPositionUs. PiperOrigin-RevId: 696026515
This commit is contained in:
kimvde
Copybara-Service
parent
838c621d00
commit
64e92cb8e1
@ -151,7 +151,7 @@ import java.util.concurrent.Executor;
|
||||
@Override
|
||||
public void onInputStreamChanged(@InputType int inputType, Format format) {
|
||||
if (format.width != inputFormat.width || format.height != inputFormat.height) {
|
||||
videoFrameRenderControl.onOutputSizeChanged(format.width, format.height);
|
||||
videoFrameRenderControl.onVideoSizeChanged(format.width, format.height);
|
||||
}
|
||||
if (format.frameRate != inputFormat.frameRate) {
|
||||
videoFrameReleaseControl.setFrameRate(format.frameRate);
|
||||
|
||||
@ -355,7 +355,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
}
|
||||
// The frame presentation time is relative to the start of the Composition and without the
|
||||
// renderer offset
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(
|
||||
framePresentationTimeUs - bufferTimestampAdjustmentUs);
|
||||
}
|
||||
|
||||
@ -471,7 +471,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
private void onStreamTimestampInfoChange(
|
||||
long bufferTimestampAdjustmentUs, long bufferPresentationTimeUs, long streamStartPositionUs) {
|
||||
this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs;
|
||||
videoFrameRenderControl.onStreamStartPositionChange(
|
||||
videoFrameRenderControl.onStreamStartPositionChanged(
|
||||
bufferPresentationTimeUs, streamStartPositionUs);
|
||||
}
|
||||
|
||||
|
||||
@ -40,23 +40,21 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
void onVideoSizeChanged(VideoSize videoSize);
|
||||
|
||||
/**
|
||||
* Called to release the {@linkplain
|
||||
* VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)} oldest frame that is
|
||||
* available for rendering}.
|
||||
* Called to release the {@linkplain VideoFrameRenderControl#onFrameAvailableForRendering(long)
|
||||
* oldest frame that is available for rendering}.
|
||||
*
|
||||
* @param renderTimeNs The specific time, in nano seconds, that this frame should be rendered or
|
||||
* {@link VideoFrameProcessor#RENDER_OUTPUT_FRAME_IMMEDIATELY} if the frame needs to be
|
||||
* rendered immediately.
|
||||
* @param presentationTimeUs The frame's presentation time, in microseconds, which was announced
|
||||
* with {@link VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)}.
|
||||
* with {@link VideoFrameRenderControl#onFrameAvailableForRendering(long)}.
|
||||
* @param isFirstFrame Whether this is the first frame of the stream.
|
||||
*/
|
||||
void renderFrame(long renderTimeNs, long presentationTimeUs, boolean isFirstFrame);
|
||||
|
||||
/**
|
||||
* Called to drop the {@linkplain
|
||||
* VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)} oldest frame that is
|
||||
* available for rendering}.
|
||||
* Called to drop the {@linkplain VideoFrameRenderControl#onFrameAvailableForRendering(long)
|
||||
* oldest frame that is available for rendering}.
|
||||
*/
|
||||
void dropFrame();
|
||||
}
|
||||
@ -64,20 +62,20 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
private final FrameRenderer frameRenderer;
|
||||
private final VideoFrameReleaseControl videoFrameReleaseControl;
|
||||
private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo;
|
||||
private final TimedValueQueue<VideoSize> videoSizeChanges;
|
||||
|
||||
/**
|
||||
* A queue of unprocessed input frame sizes. Each size is associated with the timestamp from which
|
||||
* it should be applied.
|
||||
*/
|
||||
private final TimedValueQueue<VideoSize> videoSizes;
|
||||
|
||||
private final TimedValueQueue<Long> streamStartPositionsUs;
|
||||
private final LongArrayQueue presentationTimestampsUs;
|
||||
|
||||
/**
|
||||
* Stores a video size that is announced with {@link #onOutputSizeChanged(int, int)} until an
|
||||
* output frame is made available. Once the next frame arrives, we associate the frame's timestamp
|
||||
* with the video size change in {@link #videoSizeChanges} and clear this field.
|
||||
*/
|
||||
@Nullable private VideoSize pendingOutputVideoSize;
|
||||
|
||||
private VideoSize reportedVideoSize;
|
||||
private long lastInputPresentationTimeUs;
|
||||
private long lastOutputPresentationTimeUs;
|
||||
private VideoSize outputVideoSize;
|
||||
private long outputStreamStartPositionUs;
|
||||
private long lastPresentationTimeUs;
|
||||
|
||||
/** Creates an instance. */
|
||||
public VideoFrameRenderControl(
|
||||
@ -85,36 +83,35 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
this.frameRenderer = frameRenderer;
|
||||
this.videoFrameReleaseControl = videoFrameReleaseControl;
|
||||
videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
|
||||
videoSizeChanges = new TimedValueQueue<>();
|
||||
videoSizes = new TimedValueQueue<>();
|
||||
streamStartPositionsUs = new TimedValueQueue<>();
|
||||
presentationTimestampsUs = new LongArrayQueue();
|
||||
reportedVideoSize = VideoSize.UNKNOWN;
|
||||
lastPresentationTimeUs = C.TIME_UNSET;
|
||||
lastInputPresentationTimeUs = C.TIME_UNSET;
|
||||
outputVideoSize = VideoSize.UNKNOWN;
|
||||
lastOutputPresentationTimeUs = C.TIME_UNSET;
|
||||
}
|
||||
|
||||
/** Flushes the renderer. */
|
||||
public void flush() {
|
||||
presentationTimestampsUs.clear();
|
||||
lastPresentationTimeUs = C.TIME_UNSET;
|
||||
lastInputPresentationTimeUs = C.TIME_UNSET;
|
||||
lastOutputPresentationTimeUs = C.TIME_UNSET;
|
||||
if (streamStartPositionsUs.size() > 0) {
|
||||
// There is a pending streaming start position change. If seeking within the same stream, keep
|
||||
// the pending start position with timestamp zero ensures the start position is applied on the
|
||||
// frames after flushing. Otherwise if seeking to another stream, a new start position will be
|
||||
// set before a new frame arrives so we'll be able to apply the new start position.
|
||||
// the pending start position with min timestamp to ensure the start position is applied on
|
||||
// the frames after flushing. Otherwise if seeking to another stream, a new start position
|
||||
// will be set before a new frame arrives so we'll be able to apply the new start position.
|
||||
long lastStartPositionUs = getLastAndClear(streamStartPositionsUs);
|
||||
// Input timestamps should always be positive because they are offset by ExoPlayer. Adding a
|
||||
// position to the queue with timestamp 0 should therefore always apply it as long as it is
|
||||
// the only position in the queue.
|
||||
streamStartPositionsUs.add(/* timestamp= */ 0, lastStartPositionUs);
|
||||
}
|
||||
if (pendingOutputVideoSize == null) {
|
||||
if (videoSizeChanges.size() > 0) {
|
||||
// Do not clear the last pending video size, we still want to report the size change after a
|
||||
// flush. If after the flush, a new video size is announced, it will overwrite
|
||||
// pendingOutputVideoSize. When the next frame is available for rendering, we will announce
|
||||
// pendingOutputVideoSize.
|
||||
pendingOutputVideoSize = getLastAndClear(videoSizeChanges);
|
||||
}
|
||||
} else {
|
||||
// we keep the latest value of pendingOutputVideoSize
|
||||
videoSizeChanges.clear();
|
||||
if (videoSizes.size() > 0) {
|
||||
// Do not clear the last pending video size, we still want to report the size change after a
|
||||
// flush. If after the flush, a new video size is announced, it will be used instead.
|
||||
VideoSize lastVideoSize = getLastAndClear(videoSizes);
|
||||
videoSizes.add(/* timestamp= */ 0, lastVideoSize);
|
||||
}
|
||||
}
|
||||
|
||||
@ -126,7 +123,8 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
* {@code presentationTimeUs}.
|
||||
*/
|
||||
public boolean hasReleasedFrame(long presentationTimeUs) {
|
||||
return lastPresentationTimeUs != C.TIME_UNSET && lastPresentationTimeUs >= presentationTimeUs;
|
||||
return lastOutputPresentationTimeUs != C.TIME_UNSET
|
||||
&& lastOutputPresentationTimeUs >= presentationTimeUs;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -160,12 +158,12 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
case VideoFrameReleaseControl.FRAME_RELEASE_IGNORE:
|
||||
// TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
|
||||
// VideoGraph input frames in this case.
|
||||
lastPresentationTimeUs = presentationTimeUs;
|
||||
lastOutputPresentationTimeUs = presentationTimeUs;
|
||||
dropFrame();
|
||||
break;
|
||||
case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY:
|
||||
case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED:
|
||||
lastPresentationTimeUs = presentationTimeUs;
|
||||
lastOutputPresentationTimeUs = presentationTimeUs;
|
||||
renderFrame(
|
||||
/* shouldRenderImmediately= */ frameReleaseAction
|
||||
== VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
|
||||
@ -177,8 +175,10 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
}
|
||||
|
||||
/** Called when the size of the available frames has changed. */
|
||||
public void onOutputSizeChanged(int width, int height) {
|
||||
pendingOutputVideoSize = new VideoSize(width, height);
|
||||
public void onVideoSizeChanged(int width, int height) {
|
||||
videoSizes.add(
|
||||
lastInputPresentationTimeUs == C.TIME_UNSET ? 0 : lastInputPresentationTimeUs + 1,
|
||||
new VideoSize(width, height));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -186,16 +186,13 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
*
|
||||
* @param presentationTimeUs The frame's presentation timestamp, in microseconds.
|
||||
*/
|
||||
public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
|
||||
if (pendingOutputVideoSize != null) {
|
||||
videoSizeChanges.add(presentationTimeUs, pendingOutputVideoSize);
|
||||
pendingOutputVideoSize = null;
|
||||
}
|
||||
public void onFrameAvailableForRendering(long presentationTimeUs) {
|
||||
presentationTimestampsUs.add(presentationTimeUs);
|
||||
lastInputPresentationTimeUs = presentationTimeUs;
|
||||
// TODO b/257464707 - Support extensively modified media.
|
||||
}
|
||||
|
||||
public void onStreamStartPositionChange(long presentationTimeUs, long streamStartPositionUs) {
|
||||
public void onStreamStartPositionChanged(long presentationTimeUs, long streamStartPositionUs) {
|
||||
streamStartPositionsUs.add(presentationTimeUs, streamStartPositionUs);
|
||||
}
|
||||
|
||||
@ -207,9 +204,9 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
private void renderFrame(boolean shouldRenderImmediately) {
|
||||
long presentationTimeUs = checkStateNotNull(presentationTimestampsUs.remove());
|
||||
|
||||
boolean videoSizeUpdated = maybeUpdateVideoSize(presentationTimeUs);
|
||||
boolean videoSizeUpdated = maybeUpdateOutputVideoSize(presentationTimeUs);
|
||||
if (videoSizeUpdated) {
|
||||
frameRenderer.onVideoSizeChanged(reportedVideoSize);
|
||||
frameRenderer.onVideoSizeChanged(outputVideoSize);
|
||||
}
|
||||
long renderTimeNs =
|
||||
shouldRenderImmediately
|
||||
@ -230,13 +227,12 @@ import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean maybeUpdateVideoSize(long presentationTimeUs) {
|
||||
@Nullable VideoSize videoSize = videoSizeChanges.pollFloor(presentationTimeUs);
|
||||
if (videoSize == null) {
|
||||
return false;
|
||||
}
|
||||
if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) {
|
||||
reportedVideoSize = videoSize;
|
||||
private boolean maybeUpdateOutputVideoSize(long presentationTimeUs) {
|
||||
@Nullable VideoSize newOutputVideoSize = videoSizes.pollFloor(presentationTimeUs);
|
||||
if (newOutputVideoSize != null
|
||||
&& !newOutputVideoSize.equals(VideoSize.UNKNOWN)
|
||||
&& !newOutputVideoSize.equals(outputVideoSize)) {
|
||||
outputVideoSize = newOutputVideoSize;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
||||
@ -45,9 +45,9 @@ public class VideoFrameRenderControlTest {
|
||||
new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
|
||||
|
||||
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
|
||||
videoFrameRenderControl.onOutputSizeChanged(
|
||||
videoFrameRenderControl.onVideoSizeChanged(
|
||||
/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
|
||||
|
||||
InOrder inOrder = Mockito.inOrder(frameRenderer);
|
||||
@ -74,10 +74,10 @@ public class VideoFrameRenderControlTest {
|
||||
|
||||
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
|
||||
videoFrameReleaseControl.onStarted();
|
||||
videoFrameRenderControl.onOutputSizeChanged(
|
||||
videoFrameRenderControl.onVideoSizeChanged(
|
||||
/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
|
||||
|
||||
videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
|
||||
InOrder inOrder = Mockito.inOrder(frameRenderer);
|
||||
@ -119,11 +119,11 @@ public class VideoFrameRenderControlTest {
|
||||
|
||||
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
|
||||
videoFrameReleaseControl.onStarted();
|
||||
videoFrameRenderControl.onOutputSizeChanged(
|
||||
videoFrameRenderControl.onVideoSizeChanged(
|
||||
/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
|
||||
videoFrameRenderControl.onStreamStartPositionChange(
|
||||
videoFrameRenderControl.onStreamStartPositionChanged(
|
||||
/* presentationTimeUs= */ 0, /* streamStartPositionUs= */ 10_000);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
|
||||
|
||||
InOrder inOrder = Mockito.inOrder(frameRenderer);
|
||||
@ -136,9 +136,9 @@ public class VideoFrameRenderControlTest {
|
||||
|
||||
// 10 milliseconds pass
|
||||
clock.advanceTime(/* timeDiffMs= */ 10);
|
||||
videoFrameRenderControl.onStreamStartPositionChange(
|
||||
videoFrameRenderControl.onStreamStartPositionChanged(
|
||||
/* presentationTimeUs= */ 10_000, /* streamStartPositionUs= */ 20_000);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
|
||||
videoFrameRenderControl.render(/* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 0);
|
||||
|
||||
// Second frame has the second stream start position and it is also a first frame.
|
||||
@ -168,10 +168,10 @@ public class VideoFrameRenderControlTest {
|
||||
|
||||
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
|
||||
videoFrameReleaseControl.onStarted();
|
||||
videoFrameRenderControl.onOutputSizeChanged(
|
||||
videoFrameRenderControl.onVideoSizeChanged(
|
||||
/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
|
||||
videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
|
||||
|
||||
InOrder inOrder = Mockito.inOrder(frameRenderer);
|
||||
@ -206,16 +206,16 @@ public class VideoFrameRenderControlTest {
|
||||
|
||||
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
|
||||
videoFrameReleaseControl.onStarted();
|
||||
videoFrameRenderControl.onOutputSizeChanged(
|
||||
videoFrameRenderControl.onVideoSizeChanged(
|
||||
/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.flush();
|
||||
videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
|
||||
|
||||
InOrder inOrder = Mockito.inOrder(frameRenderer);
|
||||
inOrder.verifyNoMoreInteractions();
|
||||
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
|
||||
videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
|
||||
|
||||
// First frame was rendered with pending video size change.
|
||||
@ -250,9 +250,9 @@ public class VideoFrameRenderControlTest {
|
||||
new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
|
||||
|
||||
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
|
||||
videoFrameRenderControl.onOutputSizeChanged(
|
||||
videoFrameRenderControl.onVideoSizeChanged(
|
||||
/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
|
||||
|
||||
InOrder inOrder = Mockito.inOrder(frameRenderer);
|
||||
@ -277,9 +277,9 @@ public class VideoFrameRenderControlTest {
|
||||
new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
|
||||
|
||||
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
|
||||
videoFrameRenderControl.onOutputSizeChanged(
|
||||
videoFrameRenderControl.onVideoSizeChanged(
|
||||
/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
|
||||
videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0);
|
||||
videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
|
||||
|
||||
InOrder inOrder = Mockito.inOrder(frameRenderer);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user