loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move video frame release logic to VideoFrameReleaseControl

Browse Source 
This change moves the release timestamp adjustment logic out of
MediaCodecVideoRenderer and into a standalone component, the
VideoFrameReleaseControl. The plan for VideoFrameReleaseControl is to use
it:
- from MediaCodecVideoRenderer, when ExoPlayer plays video in standalone
  mode.
- from the CompositionPlayer's DefaultVideoSink, when CompositionPlayer
  supports multiple sequences.
- (temporarily) from the CompositionPlayer's custom ImageRenderer while
  we are implementing single-sequence preview, which is an intermediate
  milestone for composition preview.
PiperOrigin-RevId: 574420427
This commit is contained in:
christosts 2023-10-18 03:14:21 -07:00 committed by Copybara-Service
parent ff330bd8e9
commit eafe2e35f0
7 changed files with 1247 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java
View File

@ -92,6 +92,7 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities {
this.index = index;
this.playerId = playerId;
this.clock = clock;
onInit();
}
@Override
@ -263,6 +264,11 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities {
// Methods to be overridden by subclasses.
/** Called when the renderer is initialized. */
protected void onInit() {
// Do nothing
}
/**
* Called when the renderer is enabled.
*

151
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
View File

@ -40,6 +40,7 @@ import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
import androidx.media3.common.VideoSize;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.LongArrayQueue;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.TimedValueQueue;
@ -62,7 +63,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Context context;
private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
private final VideoSink.RenderControl renderControl;
private final VideoFrameReleaseControl videoFrameReleaseControl;
@Nullable private VideoSinkImpl videoSinkImpl;
@Nullable private List<Effect> videoEffects;
@ -73,7 +74,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
public CompositingVideoSinkProvider(
Context context,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
VideoSink.RenderControl renderControl) {
VideoFrameReleaseControl renderControl) {
this(
context,
new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory),
@ -84,10 +85,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* package */ CompositingVideoSinkProvider(
Context context,
PreviewingVideoGraph.Factory previewingVideoGraphFactory,
VideoSink.RenderControl renderControl) {
VideoFrameReleaseControl releaseControl) {
this.context = context;
this.previewingVideoGraphFactory = previewingVideoGraphFactory;
this.renderControl = renderControl;
this.videoFrameReleaseControl = releaseControl;
}
@Override
@ -97,7 +98,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
try {
videoSinkImpl =
new VideoSinkImpl(context, previewingVideoGraphFactory, renderControl, sourceFormat);
new VideoSinkImpl(
context, previewingVideoGraphFactory, videoFrameReleaseControl, sourceFormat);
} catch (VideoFrameProcessingException e) {
throw new VideoSink.VideoSinkException(e, sourceFormat);
}
@ -173,7 +175,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listener {
private final Context context;
private final VideoSink.RenderControl renderControl;
private final VideoFrameReleaseControl videoFrameReleaseControl;
private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo;
private final VideoFrameProcessor videoFrameProcessor;
private final LongArrayQueue processedFramesBufferTimestampsUs;
private final TimedValueQueue<Long> streamOffsets;
@ -207,11 +210,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private VideoSize processedFrameSize;
private VideoSize reportedVideoSize;
private boolean pendingVideoSizeChange;
private boolean renderedFirstFrame;
private long inputStreamOffsetUs;
private boolean pendingInputStreamOffsetChange;
private long outputStreamOffsetUs;
private float playbackSpeed;
// TODO b/292111083 - Remove the field and trigger the callback on every video size change.
private boolean onVideoSizeChangedCalled;
@ -220,11 +221,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
public VideoSinkImpl(
Context context,
PreviewingVideoGraph.Factory previewingVideoGraphFactory,
RenderControl renderControl,
VideoFrameReleaseControl videoFrameReleaseControl,
Format sourceFormat)
throws VideoFrameProcessingException {
this.context = context;
this.renderControl = renderControl;
this.videoFrameReleaseControl = videoFrameReleaseControl;
videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
processedFramesBufferTimestampsUs = new LongArrayQueue();
streamOffsets = new TimedValueQueue<>();
videoSizeChanges = new TimedValueQueue<>();
@ -237,7 +239,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
lastCodecBufferPresentationTimestampUs = C.TIME_UNSET;
processedFrameSize = VideoSize.UNKNOWN;
reportedVideoSize = VideoSize.UNKNOWN;
playbackSpeed = 1f;
// Playback thread handler.
handler = Util.createHandlerForCurrentLooper();
@ -293,7 +294,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
processedFramesBufferTimestampsUs.clear();
streamOffsets.clear();
handler.removeCallbacksAndMessages(/* token= */ null);
renderedFirstFrame = false;
videoFrameReleaseControl.reset();
if (registeredLastFrame) {
registeredLastFrame = false;
processedLastFrame = false;
@ -303,7 +304,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public boolean isReady() {
return renderedFirstFrame;
return videoFrameReleaseControl.isReady(/* rendererReady= */ true);
}
@Override
@ -385,47 +386,50 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long bufferPresentationTimeUs = processedFramesBufferTimestampsUs.element();
// check whether this buffer comes with a new stream offset.
if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) {
renderedFirstFrame = false;
videoFrameReleaseControl.onProcessedStreamChange();
}
long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1;
long frameRenderTimeNs =
renderControl.getFrameRenderTimeNs(
bufferPresentationTimeUs, positionUs, elapsedRealtimeUs, playbackSpeed);
if (frameRenderTimeNs == RenderControl.RENDER_TIME_TRY_AGAIN_LATER) {
return;
} else if (framePresentationTimeUs == RenderControl.RENDER_TIME_DROP) {
// TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
// VideoFrameProcessor input frames in this case.
releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame);
continue;
@VideoFrameReleaseControl.FrameReleaseAction
int frameReleaseAction =
videoFrameReleaseControl.getFrameReleaseAction(
bufferPresentationTimeUs,
positionUs,
elapsedRealtimeUs,
outputStreamOffsetUs,
isLastFrame,
videoFrameReleaseInfo);
switch (frameReleaseAction) {
case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER:
return;
case VideoFrameReleaseControl.FRAME_RELEASE_SKIP:
case VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME:
// Skipped frames are dropped.
case VideoFrameReleaseControl.FRAME_RELEASE_DROP:
case VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME:
// TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
// VideoFrameProcessor input frames in this case.
dropFrame(isLastFrame);
break;
case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY:
case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED:
renderFrame(
framePresentationTimeUs, bufferPresentationTimeUs, frameReleaseAction, isLastFrame);
break;
default:
throw new IllegalStateException(String.valueOf(frameReleaseAction));
}
renderControl.onNextFrame(bufferPresentationTimeUs);
if (videoFrameMetadataListener != null) {
videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
framePresentationTimeUs,
frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY
? System.nanoTime()
: frameRenderTimeNs,
checkNotNull(inputFormat),
/* mediaFormat= */ null);
}
releaseProcessedFrameInternal(
frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY
? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
: frameRenderTimeNs,
isLastFrame);
maybeNotifyVideoSizeChanged(bufferPresentationTimeUs);
}
}
@Override
public void setPlaybackSpeed(float speed) {
checkArgument(speed >= 0.0);
this.playbackSpeed = speed;
videoFrameReleaseControl.setPlaybackSpeed(speed);
}
// VideoGraph.Listener methods
@Override
public void onOutputSizeChanged(int width, int height) {
VideoSize newVideoSize = new VideoSize(width, height);
@ -477,12 +481,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
throw new IllegalStateException();
}
// Other methods
public void release() {
videoFrameProcessor.release();
handler.removeCallbacksAndMessages(/* token= */ null);
streamOffsets.clear();
processedFramesBufferTimestampsUs.clear();
renderedFirstFrame = false;
}
/** Sets the {@linkplain Effect video effects} to apply immediately. */
@ -541,18 +546,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
&& currentSurfaceAndSize.second.equals(outputResolution)) {
return;
}
renderedFirstFrame =
currentSurfaceAndSize == null || currentSurfaceAndSize.first.equals(outputSurface);
videoFrameReleaseControl.setOutputSurface(outputSurface);
currentSurfaceAndSize = Pair.create(outputSurface, outputResolution);
videoFrameProcessor.setOutputSurfaceInfo(
new SurfaceInfo(
outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
}
/** Clears the output surface info. */
public void clearOutputSurfaceInfo() {
videoFrameProcessor.setOutputSurfaceInfo(null);
currentSurfaceAndSize = null;
renderedFirstFrame = false;
}
private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) {
@ -565,21 +569,52 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return updatedOffset;
}
private void dropFrame(boolean isLastFrame) {
if (listenerExecutor != null) {
listenerExecutor.execute(
() -> {
if (listener != null) {
listener.onFrameDropped(this);
}
});
}
releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame);
}
private void renderFrame(
long framePresentationTimeUs,
long bufferPresentationTimeUs,
@VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction,
boolean isLastFrame) {
if (videoFrameMetadataListener != null) {
videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
framePresentationTimeUs,
frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY
? Clock.DEFAULT.nanoTime()
: videoFrameReleaseInfo.getReleaseTimeNs(),
checkNotNull(inputFormat),
/* mediaFormat= */ null);
}
if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && listenerExecutor != null) {
listenerExecutor.execute(
() -> {
if (listener != null) {
listener.onFirstFrameRendered(/* videoSink= */ this);
}
});
}
releaseProcessedFrameInternal(
frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY
? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
: videoFrameReleaseInfo.getReleaseTimeNs(),
isLastFrame);
maybeNotifyVideoSizeChanged(bufferPresentationTimeUs);
}
private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) {
videoFrameProcessor.renderOutputFrame(releaseTimeNs);
processedFramesBufferTimestampsUs.remove();
if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME) {
renderControl.onFrameDropped();
} else {
renderControl.onFrameRendered();
if (!renderedFirstFrame) {
if (listener != null) {
checkNotNull(listenerExecutor)
.execute(() -> checkNotNull(listener).onFirstFrameRendered(this));
}
renderedFirstFrame = true;
}
}
if (isLastFrame) {
releasedLastFrame = true;
}

459
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -18,7 +18,7 @@ package androidx.media3.exoplayer.video;
import static android.view.Display.DEFAULT_DISPLAY;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Util.msToUs;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED;
import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_VIDEO_MAX_RESOLUTION_EXCEEDED;
import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO;
@ -38,7 +38,6 @@ import android.media.MediaFormat;
import android.os.Bundle;
import android.os.Handler;
import android.os.Message;
import android.os.SystemClock;
import android.util.Pair;
import android.view.Display;
import android.view.Surface;
@ -57,7 +56,6 @@ import androidx.media3.common.PlaybackException;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoSize;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.Size;
@ -88,8 +86,8 @@ import com.google.common.util.concurrent.MoreExecutors;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.Executor;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Decodes and renders video using {@link MediaCodec}.
@ -115,7 +113,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* </ul>
*/
@UnstableApi
public class MediaCodecVideoRenderer extends MediaCodecRenderer implements VideoSink.RenderControl {
public class MediaCodecVideoRenderer extends MediaCodecRenderer
implements VideoFrameReleaseControl.FrameTimingEvaluator {
private static final String TAG = "MediaCodecVideoRenderer";
private static final String KEY_CROP_LEFT = "crop-left";
@ -139,19 +138,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
/** The minimum input buffer size for HEVC. */
private static final int HEVC_MAX_INPUT_SIZE_THRESHOLD = 2 * 1024 * 1024;
/** The maximum earliest time, in microseconds, to release a frame on the surface. */
private static final long MAX_EARLY_US_THRESHOLD = 50_000;
private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround;
private static boolean deviceNeedsSetOutputSurfaceWorkaround;
private final Context context;
private final VideoFrameReleaseHelper frameReleaseHelper;
private final VideoSinkProvider videoSinkProvider;
private final EventDispatcher eventDispatcher;
private final long allowedJoiningTimeMs;
private final int maxDroppedFramesToNotify;
private final boolean deviceNeedsNoPostProcessWorkaround;
private final VideoFrameReleaseControl videoFrameReleaseControl;
private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo;
private @MonotonicNonNull CodecMaxValues codecMaxValues;
private boolean codecNeedsSetOutputSurfaceWorkaround;
@ -160,15 +156,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
@Nullable private PlaceholderSurface placeholderSurface;
private boolean haveReportedFirstFrameRenderedForCurrentSurface;
private @C.VideoScalingMode int scalingMode;
private @C.FirstFrameState int firstFrameState;
private long initialPositionUs;
private long joiningDeadlineMs;
private long droppedFrameAccumulationStartTimeMs;
private int droppedFrames;
private int consecutiveDroppedFrameCount;
private int buffersInCodecCount;
private long lastBufferPresentationTimeUs;
private long lastRenderRealtimeUs;
private long totalVideoFrameProcessingOffsetUs;
private int videoFrameProcessingOffsetCount;
private long lastFrameReleaseTimeNs;
@ -393,23 +384,42 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
mediaCodecSelector,
enableDecoderFallback,
assumedMinimumCodecOperatingRate);
this.allowedJoiningTimeMs = allowedJoiningTimeMs;
this.maxDroppedFramesToNotify = maxDroppedFramesToNotify;
this.context = context.getApplicationContext();
frameReleaseHelper = new VideoFrameReleaseHelper(this.context);
videoFrameReleaseControl = new VideoFrameReleaseControl(this.context, allowedJoiningTimeMs);
videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
eventDispatcher = new EventDispatcher(eventHandler, eventListener);
@SuppressWarnings("nullness:assignment")
VideoSink.@Initialized RenderControl renderControl = this;
videoSinkProvider =
new CompositingVideoSinkProvider(context, videoFrameProcessorFactory, renderControl);
new CompositingVideoSinkProvider(
context, videoFrameProcessorFactory, videoFrameReleaseControl);
deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround();
joiningDeadlineMs = C.TIME_UNSET;
scalingMode = C.VIDEO_SCALING_MODE_DEFAULT;
decodedVideoSize = VideoSize.UNKNOWN;
tunnelingAudioSessionId = C.AUDIO_SESSION_ID_UNSET;
firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED;
reportedVideoSize = null;
}
// FrameTimingEvaluator methods
@Override
public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) {
return shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastReleaseUs);
}
@Override
public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
return shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastFrame);
}
@Override
public boolean shouldDropFramesToKeyframe(
long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
return shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastFrame);
}
// Renderer methods
@Override
public String getName() {
return TAG;
@ -523,53 +533,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
format);
}
// RenderControl implementation
@Override
public long getFrameRenderTimeNs(
long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) {
long earlyUs =
calculateEarlyTimeUs(
positionUs,
elapsedRealtimeUs,
presentationTimeUs,
getState() == STATE_STARTED,
playbackSpeed,
getClock());
if (isBufferLate(earlyUs)) {
return VideoSink.RenderControl.RENDER_TIME_DROP;
}
if (shouldForceRender(positionUs, earlyUs)) {
return VideoSink.RenderControl.RENDER_TIME_IMMEDIATELY;
}
if (getState() != STATE_STARTED
|| positionUs == initialPositionUs
|| earlyUs > MAX_EARLY_US_THRESHOLD) {
return VideoSink.RenderControl.RENDER_TIME_TRY_AGAIN_LATER;
}
// Compute the buffer's desired release time in nanoseconds.
long unadjustedFrameReleaseTimeNs = getClock().nanoTime() + (earlyUs * 1000);
// Apply a timestamp adjustment, if there is one.
return frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs);
}
@Override
public void onNextFrame(long presentationTimeUs) {
frameReleaseHelper.onNextFrame(presentationTimeUs);
}
@Override
public void onFrameRendered() {
lastRenderRealtimeUs = Util.msToUs(getClock().elapsedRealtime());
}
@Override
public void onFrameDropped() {
updateDroppedBufferCounters(
/* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1);
}
// Other methods
/**
@ -634,6 +597,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
}
}
@Override
protected void onInit() {
super.onInit();
videoFrameReleaseControl.setFrameTimingEvaluator(/* frameTimingEvaluator= */ this);
videoFrameReleaseControl.setClock(getClock());
}
@Override
protected void onEnabled(boolean joining, boolean mayRenderStartOfStream)
throws ExoPlaybackException {
@ -645,17 +615,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
releaseCodec();
}
eventDispatcher.enabled(decoderCounters);
firstFrameState =
mayRenderStartOfStream
? C.FIRST_FRAME_NOT_RENDERED
: C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED;
videoFrameReleaseControl.onEnabled(mayRenderStartOfStream);
}
@Override
public void enableMayRenderStartOfStream() {
if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) {
firstFrameState = C.FIRST_FRAME_NOT_RENDERED;
}
videoFrameReleaseControl.allowReleaseFirstFrameBeforeStarted();
}
@Override
@ -666,21 +631,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
videoSink.flush();
}
super.onPositionReset(positionUs, joining);
if (videoSinkProvider.isInitialized()) {
videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs());
}
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
frameReleaseHelper.onPositionReset();
lastBufferPresentationTimeUs = C.TIME_UNSET;
initialPositionUs = C.TIME_UNSET;
consecutiveDroppedFrameCount = 0;
videoFrameReleaseControl.reset();
if (joining) {
setJoiningDeadlineMs();
} else {
joiningDeadlineMs = C.TIME_UNSET;
videoFrameReleaseControl.join();
}
maybeUpdateOnFrameRenderedListener();
consecutiveDroppedFrameCount = 0;
}
@Override
@ -690,26 +649,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
@Override
public boolean isReady() {
if (super.isReady()
&& (videoSink == null || videoSink.isReady())
&& (firstFrameState == C.FIRST_FRAME_RENDERED
|| (placeholderSurface != null && displaySurface == placeholderSurface)
boolean readyToReleaseFrames = super.isReady() && (videoSink == null || videoSink.isReady());
if (readyToReleaseFrames
&& ((placeholderSurface != null && displaySurface == placeholderSurface)
|| getCodec() == null
|| tunneling)) {
// Ready. If we were joining then we've now joined, so clear the joining deadline.
joiningDeadlineMs = C.TIME_UNSET;
// Not releasing frames.
return true;
} else if (joiningDeadlineMs == C.TIME_UNSET) {
// Not joining.
return false;
} else if (getClock().elapsedRealtime() < joiningDeadlineMs) {
// Joining and still within the joining deadline.
return true;
} else {
// The joining deadline has been exceeded. Give up and clear the deadline.
joiningDeadlineMs = C.TIME_UNSET;
return false;
}
return videoFrameReleaseControl.isReady(readyToReleaseFrames);
}
@Override
@ -718,25 +666,24 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
droppedFrames = 0;
long elapsedRealtimeMs = getClock().elapsedRealtime();
droppedFrameAccumulationStartTimeMs = elapsedRealtimeMs;
lastRenderRealtimeUs = msToUs(elapsedRealtimeMs);
totalVideoFrameProcessingOffsetUs = 0;
videoFrameProcessingOffsetCount = 0;
frameReleaseHelper.onStarted();
videoFrameReleaseControl.onStarted();
}
@Override
protected void onStopped() {
joiningDeadlineMs = C.TIME_UNSET;
maybeNotifyDroppedFrames();
maybeNotifyVideoFrameProcessingOffset();
frameReleaseHelper.onStopped();
videoFrameReleaseControl.onStopped();
super.onStopped();
}
@Override
protected void onDisabled() {
reportedVideoSize = null;
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED);
videoFrameReleaseControl.onDisabled();
maybeUpdateOnFrameRenderedListener();
haveReportedFirstFrameRenderedForCurrentSurface = false;
tunnelingOnFrameRenderedListener = null;
try {
@ -783,7 +730,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
}
break;
case MSG_SET_CHANGE_FRAME_RATE_STRATEGY:
frameReleaseHelper.setChangeFrameRateStrategy((int) checkNotNull(message));
videoFrameReleaseControl.setChangeFrameRateStrategy((int) checkNotNull(message));
break;
case MSG_SET_VIDEO_FRAME_METADATA_LISTENER:
frameMetadataListener = (VideoFrameMetadataListener) checkNotNull(message);
@ -843,7 +790,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
// We only need to update the codec if the display surface has changed.
if (this.displaySurface != displaySurface) {
this.displaySurface = displaySurface;
frameReleaseHelper.onSurfaceChanged(displaySurface);
videoFrameReleaseControl.setOutputSurface(displaySurface);
haveReportedFirstFrameRenderedForCurrentSurface = false;
@State int state = getState();
@ -862,11 +809,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
if (displaySurface != null && displaySurface != placeholderSurface) {
// If we know the video size, report it again immediately.
maybeRenotifyVideoSizeChanged();
// We haven't rendered to the new display surface yet.
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
if (state == STATE_STARTED) {
// Set joining deadline to report MediaCodecVideoRenderer is ready.
setJoiningDeadlineMs();
videoFrameReleaseControl.join();
}
// When effects previewing is enabled, set display surface and an unknown size.
if (videoSinkProvider.isInitialized()) {
@ -875,11 +819,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
} else {
// The display surface has been removed.
reportedVideoSize = null;
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
if (videoSinkProvider.isInitialized()) {
videoSinkProvider.clearOutputSurfaceInfo();
}
}
maybeUpdateOnFrameRenderedListener();
} else if (displaySurface != null && displaySurface != placeholderSurface) {
// The display surface is set and unchanged. If we know the video size and/or have already
// rendered to the display surface, report these again immediately.
@ -987,7 +931,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
public void setPlaybackSpeed(float currentPlaybackSpeed, float targetPlaybackSpeed)
throws ExoPlaybackException {
super.setPlaybackSpeed(currentPlaybackSpeed, targetPlaybackSpeed);
frameReleaseHelper.onPlaybackSpeed(currentPlaybackSpeed);
videoFrameReleaseControl.setPlaybackSpeed(currentPlaybackSpeed);
if (videoSink != null) {
videoSink.setPlaybackSpeed(currentPlaybackSpeed);
}
@ -1104,7 +1048,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
new VideoSink.Listener() {
@Override
public void onFirstFrameRendered(VideoSink videoSink) {
maybeNotifyRenderedFirstFrame();
checkStateNotNull(displaySurface);
notifyRenderedFirstFrame();
}
@Override
public void onFrameDropped(VideoSink videoSink) {
updateDroppedBufferCounters(
/* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1);
}
@Override
@ -1245,7 +1196,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
}
decodedVideoSize =
new VideoSize(width, height, unappliedRotationDegrees, pixelWidthHeightRatio);
frameReleaseHelper.onFormatChanged(format.frameRate);
videoFrameReleaseControl.setFrameRate(format.frameRate);
if (videoSink != null && mediaFormat != null) {
onReadyToRegisterVideoSinkInputStream();
@ -1319,40 +1270,34 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
throws ExoPlaybackException {
checkNotNull(codec); // Can not render video without codec
if (initialPositionUs == C.TIME_UNSET) {
initialPositionUs = positionUs;
}
if (bufferPresentationTimeUs != lastBufferPresentationTimeUs) {
if (videoSink == null) {
frameReleaseHelper.onNextFrame(bufferPresentationTimeUs);
} // else, update the frameReleaseHelper when releasing the processed frames.
this.lastBufferPresentationTimeUs = bufferPresentationTimeUs;
}
long outputStreamOffsetUs = getOutputStreamOffsetUs();
long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
@VideoFrameReleaseControl.FrameReleaseAction
int frameReleaseAction =
videoFrameReleaseControl.getFrameReleaseAction(
bufferPresentationTimeUs,
positionUs,
elapsedRealtimeUs,
getOutputStreamStartPositionUs(),
isLastBuffer,
videoFrameReleaseInfo);
// Skip decode-only buffers, e.g. after seeking, immediately. This check must be performed after
// getting the release action from the video frame release control although not necessary.
// That's because the release control estimates the content frame rate from frame timestamps
// and we want to have this information known as early as possible, especially during seeking.
if (isDecodeOnlyBuffer && !isLastBuffer) {
skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
return true;
}
boolean isStarted = getState() == STATE_STARTED;
long earlyUs =
calculateEarlyTimeUs(
positionUs,
elapsedRealtimeUs,
bufferPresentationTimeUs,
isStarted,
getPlaybackSpeed(),
getClock());
// We are not rendering on a surface, the renderer will wait until a surface is set.
if (displaySurface == placeholderSurface) {
// Skip frames in sync with playback, so we'll be at the right frame if the mode changes.
if (isBufferLate(earlyUs)) {
if (videoFrameReleaseInfo.getEarlyUs() < 30_000) {
skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
updateVideoFrameProcessingOffsetCounters(earlyUs);
updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs());
return true;
}
return false;
@ -1368,137 +1313,86 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
return true;
}
boolean forceRenderOutputBuffer = shouldForceRender(positionUs, earlyUs);
if (forceRenderOutputBuffer) {
long releaseTimeNs = getClock().nanoTime();
notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs);
updateVideoFrameProcessingOffsetCounters(earlyUs);
return true;
}
if (!isStarted || positionUs == initialPositionUs) {
return false;
}
// Compute the buffer's desired release time in nanoseconds.
long systemTimeNs = getClock().nanoTime();
long unadjustedFrameReleaseTimeNs = systemTimeNs + (earlyUs * 1000);
// Apply a timestamp adjustment, if there is one.
long adjustedReleaseTimeNs = frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs);
earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000;
boolean treatDroppedBuffersAsSkipped = joiningDeadlineMs != C.TIME_UNSET;
if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastBuffer)
&& maybeDropBuffersToKeyframe(positionUs, treatDroppedBuffersAsSkipped)) {
return false;
} else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastBuffer)) {
if (treatDroppedBuffersAsSkipped) {
switch (frameReleaseAction) {
case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY:
long releaseTimeNs = getClock().nanoTime();
notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs);
updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs());
return true;
case VideoFrameReleaseControl.FRAME_RELEASE_SKIP:
skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
} else {
updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs());
return true;
case VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME:
if (!maybeDropBuffersToKeyframe(positionUs, /* treatDroppedBuffersAsSkipped= */ true)) {
skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs());
return true;
} else {
return false;
}
case VideoFrameReleaseControl.FRAME_RELEASE_DROP:
dropOutputBuffer(codec, bufferIndex, presentationTimeUs);
}
updateVideoFrameProcessingOffsetCounters(earlyUs);
return true;
updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs());
return true;
case VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME:
if (!maybeDropBuffersToKeyframe(positionUs, /* treatDroppedBuffersAsSkipped= */ false)) {
dropOutputBuffer(codec, bufferIndex, presentationTimeUs);
updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs());
return true;
} else {
return false;
}
case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER:
return false;
case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED:
return maybeReleaseFrame(checkStateNotNull(codec), bufferIndex, presentationTimeUs, format);
default:
throw new IllegalStateException(String.valueOf(frameReleaseAction));
}
}
private boolean maybeReleaseFrame(
MediaCodecAdapter codec, int bufferIndex, long presentationTimeUs, Format format) {
long releaseTimeNs = videoFrameReleaseInfo.getReleaseTimeNs();
long earlyUs = videoFrameReleaseInfo.getEarlyUs();
if (Util.SDK_INT >= 21) {
// Let the underlying framework time the release.
if (earlyUs < MAX_EARLY_US_THRESHOLD) {
if (shouldSkipBuffersWithIdenticalReleaseTime()
&& adjustedReleaseTimeNs == lastFrameReleaseTimeNs) {
// This frame should be displayed on the same vsync with the previous released frame. We
// are likely rendering frames at a rate higher than the screen refresh rate. Skip
// this buffer so that it's returned to MediaCodec sooner otherwise MediaCodec may not
// be able to keep decoding with this rate [b/263454203].
skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
} else {
notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format);
renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs);
}
updateVideoFrameProcessingOffsetCounters(earlyUs);
lastFrameReleaseTimeNs = adjustedReleaseTimeNs;
return true;
if (shouldSkipBuffersWithIdenticalReleaseTime() && releaseTimeNs == lastFrameReleaseTimeNs) {
// This frame should be displayed on the same vsync with the previous released frame. We
// are likely rendering frames at a rate higher than the screen refresh rate. Skip
// this buffer so that it's returned to MediaCodec sooner otherwise MediaCodec may not
// be able to keep decoding with this rate [b/263454203].
skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
} else {
notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs);
}
} else {
updateVideoFrameProcessingOffsetCounters(earlyUs);
lastFrameReleaseTimeNs = releaseTimeNs;
return true;
} else if (earlyUs < 30000) {
// We need to time the release ourselves.
if (earlyUs < 30000) {
if (earlyUs > 11000) {
// We're a little too early to render the frame. Sleep until the frame can be rendered.
// Note: The 11ms threshold was chosen fairly arbitrarily.
try {
// Subtracting 10000 rather than 11000 ensures the sleep time will be at least 1ms.
Thread.sleep((earlyUs - 10000) / 1000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
if (earlyUs > 11000) {
// We're a little too early to render the frame. Sleep until the frame can be rendered.
// Note: The 11ms threshold was chosen fairly arbitrarily.
try {
// Subtracting 10000 rather than 11000 ensures the sleep time will be at least 1ms.
Thread.sleep((earlyUs - 10000) / 1000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format);
renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
updateVideoFrameProcessingOffsetCounters(earlyUs);
return true;
}
}
// We're either not playing, or it's not time to render the frame yet.
return false;
}
/** Returns whether a buffer or a processed frame should be force rendered. */
private boolean shouldForceRender(long positionUs, long earlyUs) {
if (joiningDeadlineMs != C.TIME_UNSET) {
// No force rendering during joining.
notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
updateVideoFrameProcessingOffsetCounters(earlyUs);
return true;
} else {
// Too soon.
return false;
}
boolean isStarted = getState() == STATE_STARTED;
switch (firstFrameState) {
case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED:
return isStarted;
case C.FIRST_FRAME_NOT_RENDERED:
return true;
case C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE:
return positionUs >= getOutputStreamStartPositionUs();
case C.FIRST_FRAME_RENDERED:
long elapsedSinceLastRenderUs = msToUs(getClock().elapsedRealtime()) - lastRenderRealtimeUs;
return isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs);
default:
throw new IllegalStateException();
}
}
/**
* Calculates the time interval between the current player position and the buffer presentation
* time.
*
* @param positionUs The current media time in microseconds, measured at the start of the current
* iteration of the rendering loop.
* @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the
* start of the current iteration of the rendering loop.
* @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds,
* with {@linkplain #getOutputStreamOffsetUs() stream offset added}.
* @param isStarted Whether the playback is in {@link #STATE_STARTED}.
* @param playbackSpeed The current playback speed.
* @param clock The {@link Clock} used by the renderer.
* @return The calculated early time, in microseconds.
*/
private static long calculateEarlyTimeUs(
long positionUs,
long elapsedRealtimeUs,
long bufferPresentationTimeUs,
boolean isStarted,
float playbackSpeed,
Clock clock) {
// Calculate how early we are. In other words, the realtime duration that needs to elapse whilst
// the renderer is started before the frame should be rendered. A negative value means that
// we're already late.
// Note: Use of double rather than float is intentional for accuracy in the calculations below.
long earlyUs = (long) ((bufferPresentationTimeUs - positionUs) / (double) playbackSpeed);
if (isStarted) {
// Account for the elapsed time since the start of this iteration of the rendering loop.
earlyUs -= Util.msToUs(clock.elapsedRealtime()) - elapsedRealtimeUs;
}
return earlyUs;
}
private void notifyFrameMetadataListener(
@ -1535,7 +1429,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
@Override
protected void onProcessedStreamChange() {
super.onProcessedStreamChange();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE);
videoFrameReleaseControl.onProcessedStreamChange();
maybeUpdateOnFrameRenderedListener();
if (videoSinkProvider.isInitialized()) {
videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs());
}
@ -1552,7 +1447,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
*/
protected boolean shouldDropOutputBuffer(
long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
return isBufferLate(earlyUs) && !isLastBuffer;
return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFrame(
earlyUs, elapsedRealtimeUs, isLastBuffer);
}
/**
@ -1567,7 +1463,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
*/
protected boolean shouldDropBuffersToKeyframe(
long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
return isBufferVeryLate(earlyUs) && !isLastBuffer;
return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFramesToKeyframe(
earlyUs, elapsedRealtimeUs, isLastBuffer);
}
/**
@ -1588,8 +1485,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
* @return Returns whether to force rendering an output buffer.
*/
protected boolean shouldForceRenderOutputBuffer(long earlyUs, long elapsedSinceLastRenderUs) {
// Force render late buffers every 100ms to avoid frozen video effect.
return isBufferLate(earlyUs) && elapsedSinceLastRenderUs > 100000;
return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldForceReleaseFrame(
earlyUs, elapsedSinceLastRenderUs);
}
/**
@ -1721,7 +1618,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
decoderCounters.renderedOutputBufferCount++;
consecutiveDroppedFrameCount = 0;
if (videoSink == null) {
lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime());
maybeNotifyVideoSizeChanged(decodedVideoSize);
maybeNotifyRenderedFirstFrame();
}
@ -1745,7 +1641,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
decoderCounters.renderedOutputBufferCount++;
consecutiveDroppedFrameCount = 0;
if (videoSink == null) {
lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime());
maybeNotifyVideoSizeChanged(decodedVideoSize);
maybeNotifyRenderedFirstFrame();
}
@ -1769,15 +1664,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
}
}
private void setJoiningDeadlineMs() {
joiningDeadlineMs =
allowedJoiningTimeMs > 0
? (getClock().elapsedRealtime() + allowedJoiningTimeMs)
: C.TIME_UNSET;
}
private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) {
this.firstFrameState = min(this.firstFrameState, firstFrameState);
private void maybeUpdateOnFrameRenderedListener() {
// The first frame notification is triggered by renderOutputBuffer or renderOutputBufferV21 for
// non-tunneled playback, onQueueInputBuffer for tunneled playback prior to API level 23, and
// OnFrameRenderedListenerV23.onFrameRenderedListener for tunneled playback on API level 23 and
@ -1792,13 +1679,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
}
private void maybeNotifyRenderedFirstFrame() {
if (displaySurface != null && firstFrameState != C.FIRST_FRAME_RENDERED) {
firstFrameState = C.FIRST_FRAME_RENDERED;
eventDispatcher.renderedFirstFrame(displaySurface);
haveReportedFirstFrameRenderedForCurrentSurface = true;
if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && displaySurface != null) {
notifyRenderedFirstFrame();
}
}
@RequiresNonNull("displaySurface")
private void notifyRenderedFirstFrame() {
eventDispatcher.renderedFirstFrame(displaySurface);
haveReportedFirstFrameRenderedForCurrentSurface = true;
}
private void maybeRenotifyRenderedFirstFrame() {
if (displaySurface != null && haveReportedFirstFrameRenderedForCurrentSurface) {
eventDispatcher.renderedFirstFrame(displaySurface);
@ -1838,16 +1729,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video
}
}
private static boolean isBufferLate(long earlyUs) {
// Class a buffer as late if it should have been presented more than 30 ms ago.
return earlyUs < -30000;
}
private static boolean isBufferVeryLate(long earlyUs) {
// Class a buffer as very late if it should have been presented more than 500 ms ago.
return earlyUs < -500000;
}
@RequiresApi(29)
private static void setHdr10PlusInfoV29(MediaCodecAdapter codec, byte[] hdr10PlusInfo) {
Bundle codecParameters = new Bundle();

466
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java Normal file
View File

@ -0,0 +1,466 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.video;
import static androidx.media3.common.util.Util.msToUs;
import static java.lang.Math.min;
import static java.lang.annotation.ElementType.TYPE_USE;
import android.content.Context;
import android.os.SystemClock;
import android.view.Surface;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.Renderer;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/** Controls the releasing of video frames. */
/* package */ final class VideoFrameReleaseControl {
/**
* The frame release action returned by {@link #getFrameReleaseAction(long, long, long, long,
* boolean, FrameReleaseInfo)}.
*
* <p>One of {@link #FRAME_RELEASE_IMMEDIATELY}, {@link #FRAME_RELEASE_SCHEDULED}, {@link
* #FRAME_RELEASE_DROP}, {@link #FRAME_RELEASE_DROP_TO_KEYFRAME}, {@link ##FRAME_RELEASE_SKIP} or
* {@link #FRAME_RELEASE_TRY_AGAIN_LATER}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@UnstableApi
@IntDef({
FRAME_RELEASE_IMMEDIATELY,
FRAME_RELEASE_SCHEDULED,
FRAME_RELEASE_DROP,
FRAME_RELEASE_DROP_TO_KEYFRAME,
FRAME_RELEASE_SKIP,
FRAME_RELEASE_SKIP_TO_KEYFRAME,
FRAME_RELEASE_TRY_AGAIN_LATER
})
public @interface FrameReleaseAction {}
/** Signals a frame should be released immediately. */
public static final int FRAME_RELEASE_IMMEDIATELY = 0;
/**
* Signals a frame should be scheduled for release. The release timestamp will be returned by
* {@link FrameReleaseInfo#getReleaseTimeNs()}.
*/
public static final int FRAME_RELEASE_SCHEDULED = 1;
/** Signals a frame should be dropped. */
public static final int FRAME_RELEASE_DROP = 2;
/** Signals frames up to the next key-frame should be dropped. */
public static final int FRAME_RELEASE_DROP_TO_KEYFRAME = 3;
/** Signals that a frame should be skipped. */
public static final int FRAME_RELEASE_SKIP = 4;
/** Signals that frames up to the next key-frame should be skipped. */
public static final int FRAME_RELEASE_SKIP_TO_KEYFRAME = 5;
/** Signals that a frame should not be released and the renderer should try again later. */
public static final int FRAME_RELEASE_TRY_AGAIN_LATER = 6;
/** Per {@link FrameReleaseAction} metadata. */
public static class FrameReleaseInfo {
private long earlyUs;
private long releaseTimeNs;
/** Resets this instances state. */
public FrameReleaseInfo() {
earlyUs = C.TIME_UNSET;
releaseTimeNs = C.TIME_UNSET;
}
/**
* Returns this frame's early time compared to the playback position, before any release time
* adjustment to the screen vsync slots.
*/
public long getEarlyUs() {
return earlyUs;
}
/**
* Returns the release time for the frame, in nanoseconds, or {@link C#TIME_UNSET} if the frame
* should not be released yet.
*/
public long getReleaseTimeNs() {
return releaseTimeNs;
}
private void reset() {
earlyUs = C.TIME_UNSET;
releaseTimeNs = C.TIME_UNSET;
}
}
/** Decides whether a frame should be forced to be released, or dropped. */
public interface FrameTimingEvaluator {
/**
* Whether a frame should be forced for release.
*
* @param earlyUs The time until the buffer should be presented in microseconds. A negative
* value indicates that the buffer is late.
* @param elapsedSinceLastReleaseUs The elapsed time since the last frame was released, in
* microseconds.
* @return Whether the video frame should be force released.
*/
boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs);
/**
* Returns whether the frame should be dropped.
*
* @param earlyUs The time until the buffer should be presented in microseconds. A negative
* value indicates that the buffer is late.
* @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
* measured at the start of the current iteration of the rendering loop.
* @param isLastFrame Whether the buffer is the last buffer in the current stream.
*/
boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame);
/**
* Returns whether to drop all frames starting from this frame to the keyframe at or after the
* current playback position, if possible.
*
* @param earlyUs The time until the current buffer should be presented in microseconds. A
* negative value indicates that the buffer is late.
* @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
* measured at the start of the current iteration of the rendering loop.
* @param isLastFrame Whether the frame is the last frame in the current stream.
*/
boolean shouldDropFramesToKeyframe(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame);
/** The default timing evaluator. */
FrameTimingEvaluator DEFAULT =
new FrameTimingEvaluator() {
@Override
public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) {
// Force render late buffers every 100ms to avoid frozen video effect.
return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && elapsedSinceLastReleaseUs > 100_000;
}
@Override
public boolean shouldDropFrame(
long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastFrame;
}
@Override
public boolean shouldDropFramesToKeyframe(
long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastFrame;
}
};
}
/** The earliest time threshold, in microseconds, after which a frame is considered late. */
private static final long MIN_EARLY_US_LATE_THRESHOLD = -30_000;
/** The earliest time threshold, in microseconds, after which a frame is considered very late. */
private static final long MIN_EARLY_US_VERY_LATE_THRESHOLD = -500_000;
/** The maximum earliest time, in microseconds, to release a frame on the surface. */
private static final long MAX_EARLY_US_THRESHOLD = 50_000;
private final VideoFrameReleaseHelper frameReleaseHelper;
private final long allowedJoiningTimeMs;
private FrameTimingEvaluator frameTimingEvaluator;
private boolean started;
private @C.FirstFrameState int firstFrameState;
private long initialPositionUs;
private long lastReleaseRealtimeUs;
private long lastPresentationTimeUs;
private long joiningDeadlineMs;
private float playbackSpeed;
private Clock clock;
/**
* Creates an instance.
*
* @param applicationContext The application context.
* @param allowedJoiningTimeMs The maximum duration in milliseconds for which the renderer can
* attempt to seamlessly join an ongoing playback.
*/
public VideoFrameReleaseControl(Context applicationContext, long allowedJoiningTimeMs) {
frameReleaseHelper = new VideoFrameReleaseHelper(applicationContext);
this.allowedJoiningTimeMs = allowedJoiningTimeMs;
this.frameTimingEvaluator = FrameTimingEvaluator.DEFAULT;
firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED;
initialPositionUs = C.TIME_UNSET;
lastPresentationTimeUs = C.TIME_UNSET;
joiningDeadlineMs = C.TIME_UNSET;
playbackSpeed = 1f;
clock = Clock.DEFAULT;
}
/** Sets the {@link FrameTimingEvaluator}. */
public void setFrameTimingEvaluator(FrameTimingEvaluator frameTimingEvaluator) {
this.frameTimingEvaluator = frameTimingEvaluator;
}
/** Called when the renderer is enabled. */
public void onEnabled(boolean releaseFirstFrameBeforeStarted) {
firstFrameState =
releaseFirstFrameBeforeStarted
? C.FIRST_FRAME_NOT_RENDERED
: C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED;
}
/** Called when the renderer is disabled. */
public void onDisabled() {
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED);
}
/** Called when the renderer is started. */
public void onStarted() {
started = true;
lastReleaseRealtimeUs = msToUs(clock.elapsedRealtime());
frameReleaseHelper.onStarted();
}
/** Called when the renderer is stopped. */
public void onStopped() {
started = false;
joiningDeadlineMs = C.TIME_UNSET;
frameReleaseHelper.onStopped();
}
/** Called when the renderer processed a stream change. */
public void onProcessedStreamChange() {
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE);
}
/** Called when the display surface changed. */
public void setOutputSurface(@Nullable Surface outputSurface) {
frameReleaseHelper.onSurfaceChanged(outputSurface);
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
}
/** Sets the frame rate. */
public void setFrameRate(float frameRate) {
frameReleaseHelper.onFormatChanged(frameRate);
}
/**
* Called when a frame have been released.
*
* @return Whether this is the first released frame.
*/
public boolean onFrameReleasedIsFirstFrame() {
boolean firstFrame = firstFrameState != C.FIRST_FRAME_RENDERED;
firstFrameState = C.FIRST_FRAME_RENDERED;
lastReleaseRealtimeUs = msToUs(clock.elapsedRealtime());
return firstFrame;
}
/** Sets the clock that will be used. */
public void setClock(Clock clock) {
this.clock = clock;
}
/**
* Allows the frame control to indicate the first frame can be released before this instance is
* started.
*/
public void allowReleaseFirstFrameBeforeStarted() {
if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) {
firstFrameState = C.FIRST_FRAME_NOT_RENDERED;
}
}
/**
* Whether the release control is ready to start playback.
*
* @see Renderer#isReady()
* @param rendererReady Whether the renderer is ready.
* @return Whether the release control is ready.
*/
public boolean isReady(boolean rendererReady) {
if (rendererReady && firstFrameState == C.FIRST_FRAME_RENDERED) {
// Ready. If we were joining then we've now joined, so clear the joining deadline.
joiningDeadlineMs = C.TIME_UNSET;
return true;
} else if (joiningDeadlineMs == C.TIME_UNSET) {
// Not joining.
return false;
} else if (clock.elapsedRealtime() < joiningDeadlineMs) {
// Joining and still withing the deadline.
return true;
} else {
// The joining deadline has been exceeded. Give up and clear the deadline.
joiningDeadlineMs = C.TIME_UNSET;
return false;
}
}
/** Joins the release control to a new stream. */
public void join() {
joiningDeadlineMs =
allowedJoiningTimeMs > 0 ? (clock.elapsedRealtime() + allowedJoiningTimeMs) : C.TIME_UNSET;
}
/**
* Returns a {@link FrameReleaseAction} for a video frame which instructs a renderer what to do
* with the frame.
*
* @param presentationTimeUs The presentation time of the video frame, in microseconds.
* @param positionUs The current playback position, in microseconds.
* @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
* taken approximately at the time the playback position was {@code positionUs}.
* @param outputStreamStartPositionUs The stream's start position, in microseconds.
* @param isLastFrame Whether the frame is known to contain the last frame of the current stream.
* @param frameReleaseInfo A {@link FrameReleaseInfo} that will be filled with detailed data only
* if the method returns {@link #FRAME_RELEASE_IMMEDIATELY} or {@link
* #FRAME_RELEASE_SCHEDULED}.
* @return A {@link FrameReleaseAction} that should instruct the renderer whether to release the
* frame or not.
*/
public @FrameReleaseAction int getFrameReleaseAction(
long presentationTimeUs,
long positionUs,
long elapsedRealtimeUs,
long outputStreamStartPositionUs,
boolean isLastFrame,
FrameReleaseInfo frameReleaseInfo) {
frameReleaseInfo.reset();
if (initialPositionUs == C.TIME_UNSET) {
initialPositionUs = positionUs;
}
if (lastPresentationTimeUs != presentationTimeUs) {
frameReleaseHelper.onNextFrame(presentationTimeUs);
lastPresentationTimeUs = presentationTimeUs;
}
frameReleaseInfo.earlyUs =
calculateEarlyTimeUs(positionUs, elapsedRealtimeUs, presentationTimeUs);
if (shouldForceRelease(positionUs, frameReleaseInfo.earlyUs, outputStreamStartPositionUs)) {
return FRAME_RELEASE_IMMEDIATELY;
}
if (!started
|| positionUs == initialPositionUs
|| frameReleaseInfo.earlyUs > MAX_EARLY_US_THRESHOLD) {
return FRAME_RELEASE_TRY_AGAIN_LATER;
}
// Calculate release time and and adjust earlyUs to screen vsync.
long systemTimeNs = clock.nanoTime();
frameReleaseInfo.releaseTimeNs =
frameReleaseHelper.adjustReleaseTime(systemTimeNs + (frameReleaseInfo.earlyUs * 1_000));
frameReleaseInfo.earlyUs = (frameReleaseInfo.releaseTimeNs - systemTimeNs) / 1_000;
// While joining, late frames are skipped.
boolean treatDropAsSkip = joiningDeadlineMs != C.TIME_UNSET;
if (frameTimingEvaluator.shouldDropFramesToKeyframe(
frameReleaseInfo.earlyUs, elapsedRealtimeUs, isLastFrame)) {
return treatDropAsSkip ? FRAME_RELEASE_SKIP_TO_KEYFRAME : FRAME_RELEASE_DROP_TO_KEYFRAME;
} else if (frameTimingEvaluator.shouldDropFrame(
frameReleaseInfo.earlyUs, elapsedRealtimeUs, isLastFrame)) {
// While joining, dropped buffers are considered skipped.
return treatDropAsSkip ? FRAME_RELEASE_SKIP : FRAME_RELEASE_DROP;
}
return FRAME_RELEASE_SCHEDULED;
}
/** Resets the release control. */
public void reset() {
frameReleaseHelper.onPositionReset();
lastPresentationTimeUs = C.TIME_UNSET;
initialPositionUs = C.TIME_UNSET;
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
joiningDeadlineMs = C.TIME_UNSET;
}
/**
* Change the {@link C.VideoChangeFrameRateStrategy}, used when calling {@link
* Surface#setFrameRate}.
*/
public void setChangeFrameRateStrategy(
@C.VideoChangeFrameRateStrategy int changeFrameRateStrategy) {
frameReleaseHelper.setChangeFrameRateStrategy(changeFrameRateStrategy);
}
/** Sets the playback speed. Called when the renderer playback speed changes. */
public void setPlaybackSpeed(float speed) {
this.playbackSpeed = speed;
frameReleaseHelper.onPlaybackSpeed(speed);
}
private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) {
this.firstFrameState = min(this.firstFrameState, firstFrameState);
}
/**
* Calculates the time interval between the current player position and the frame presentation
* time.
*
* @param positionUs The current media time in microseconds, measured at the start of the current
* iteration of the rendering loop.
* @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the
* start of the current iteration of the rendering loop.
* @param framePresentationTimeUs The presentation time of the frame in microseconds.
* @return The calculated early time, in microseconds.
*/
private long calculateEarlyTimeUs(
long positionUs, long elapsedRealtimeUs, long framePresentationTimeUs) {
// Calculate how early we are. In other words, the realtime duration that needs to elapse whilst
// the renderer is started before the frame should be rendered. A negative value means that
// we're already late.
// Note: Use of double rather than float is intentional for accuracy in the calculations below.
long earlyUs = (long) ((framePresentationTimeUs - positionUs) / (double) playbackSpeed);
if (started) {
// Account for the elapsed time since the start of this iteration of the rendering loop.
earlyUs -= Util.msToUs(clock.elapsedRealtime()) - elapsedRealtimeUs;
}
return earlyUs;
}
/** Returns whether a frame should be force released. */
private boolean shouldForceRelease(
long positionUs, long earlyUs, long outputStreamStartPositionUs) {
if (joiningDeadlineMs != C.TIME_UNSET) {
// No force releasing during joining.
return false;
}
switch (firstFrameState) {
case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED:
return started;
case C.FIRST_FRAME_NOT_RENDERED:
return true;
case C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE:
return positionUs >= outputStreamStartPositionUs;
case C.FIRST_FRAME_RENDERED:
long elapsedTimeSinceLastReleaseUs =
msToUs(clock.elapsedRealtime()) - lastReleaseRealtimeUs;
return started
&& frameTimingEvaluator.shouldForceReleaseFrame(earlyUs, elapsedTimeSinceLastReleaseUs);
default:
throw new IllegalStateException();
}
}
}

46
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java
View File

@ -55,6 +55,9 @@ import java.util.concurrent.Executor;
/** Called when the sink renderers the first frame. */
void onFirstFrameRendered(VideoSink videoSink);
/** Called when the sink dropped a frame. */
void onFrameDropped(VideoSink videoSink);
/** Called when the output video size changed. */
void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize);
@ -62,49 +65,6 @@ import java.util.concurrent.Executor;
void onError(VideoSink videoSink, VideoSinkException videoSinkException);
}
/** Controls the rendering of video frames. */
interface RenderControl {
/** Signals a frame must be rendered immediately. */
long RENDER_TIME_IMMEDIATELY = -1;
/** Signals a frame must be dropped. */
long RENDER_TIME_DROP = -2;
/** Signals that a frame should not be rendered yet. */
long RENDER_TIME_TRY_AGAIN_LATER = -3;
/**
* Returns the render timestamp, in nanoseconds, associated with this video frames or one of the
* {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not
* rendered yet.
*
* @param presentationTimeUs The presentation time of the video frame, in microseconds.
* @param positionUs The current playback position, in microseconds.
* @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
* taken approximately at the time the playback position was {@code positionUs}.
* @param playbackSpeed The current playback speed.
* @return The render timestamp, in nanoseconds, associated with this frame, or one of the
* {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not
* rendered yet.
*/
long getFrameRenderTimeNs(
long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed);
/**
* Informs the rendering control that a video frame will be rendered. Call this method before
* rendering a frame.
*
* @param presentationTimeUs The frame's presentation time, in microseconds.
*/
void onNextFrame(long presentationTimeUs);
/** Informs the rendering control that a video frame was rendered. */
void onFrameRendered();
/** Informs the rendering control that a video frame was dropped. */
void onFrameDropped();
}
/**
* Specifies how the input frames are made available to the video sink. One of {@link
* #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}.

24
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java
View File

@ -132,11 +132,13 @@ public final class CompositingVideoSinkProviderTest {
}
private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() {
VideoSink.RenderControl renderControl = new TestRenderControl();
VideoFrameReleaseControl releaseControl =
new VideoFrameReleaseControl(
ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0);
return new CompositingVideoSinkProvider(
ApplicationProvider.getApplicationContext(),
new TestPreviewingVideoGraphFactory(),
renderControl);
releaseControl);
}
private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory {
@ -161,22 +163,4 @@ public final class CompositingVideoSinkProviderTest {
return previewingVideoGraph;
}
}
private static class TestRenderControl implements VideoSink.RenderControl {
@Override
public long getFrameRenderTimeNs(
long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) {
return presentationTimeUs;
}
@Override
public void onNextFrame(long presentationTimeUs) {}
@Override
public void onFrameRendered() {}
@Override
public void onFrameDropped() {}
}
}

505
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java Normal file
View File

@ -0,0 +1,505 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.video;
import static com.google.common.truth.Truth.assertThat;
import androidx.media3.test.utils.FakeClock;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Unit tests for {@link VideoFrameReleaseControl}. */
@RunWith(AndroidJUnit4.class)
public class VideoFrameReleaseControlTest {
@Test
public void isReady_onNewInstance_returnsFalse() {
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isFalse();
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse();
}
@Test
public void isReady_afterReleasingFrame_returnsTrue() {
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue();
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isTrue();
}
@Test
public void isReady_withinJoiningDeadline_returnsTrue() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.join();
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
}
@Test
public void isReady_joiningDeadlineExceeded_returnsFalse() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.join();
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
clock.advanceTime(/* timeDiffMs= */ 101);
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse();
}
@Test
public void onFrameReleasedIsFirstFrame_resetsAfterOnEnabled() {
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue();
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue();
}
@Test
public void onFrameReleasedIsFirstFrame_resetsAfterOnProcessedStreamChange() {
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue();
videoFrameReleaseControl.onProcessedStreamChange();
assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue();
}
@Test
public void onFrameReleasedIsFirstFrame_resetsAfterSetOutputSurface() {
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue();
videoFrameReleaseControl.setOutputSurface(/* outputSurface= */ null);
assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue();
}
@Test
public void isReady_afterReset_returnsFalse() {
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isTrue();
videoFrameReleaseControl.reset();
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isFalse();
}
@Test
public void getFrameReleaseAction_firstFrameAllowedBeforeStart_returnsReleaseImmediately() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
}
@Test
public void getFrameReleaseAction_firstFrameNotAllowedBeforeStart_returnsTryAgainLater() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ false);
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER);
}
@Test
public void
getFrameReleaseAction_firstFrameNotAllowedBeforeStartAndStarted_returnsReleaseImmediately() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ false);
videoFrameReleaseControl.onStarted();
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
}
@Test
public void getFrameReleaseAction_secondFrameAndNotStarted_returnsTryAgainLater() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
// Second frame
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 10_000,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER);
}
@Test
public void getFrameReleaseAction_secondFrameAndStarted_returnsScheduled() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
// Second frame
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 10_000,
/* positionUs= */ 1,
/* elapsedRealtimeUs= */ 1,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED);
}
@Test
public void getFrameReleaseAction_secondFrameEarly_returnsTryAgainLater() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
clock.advanceTime(/* timeDiffMs= */ 10);
// Second frame is 51 ms too soon.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 61_000,
/* positionUs= */ 10_000,
/* elapsedRealtimeUs= */ 10_000,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER);
assertThat(frameReleaseInfo.getEarlyUs()).isEqualTo(51_000);
}
@Test
public void getFrameReleaseAction_frameLate_returnsDrop() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
clock.advanceTime(/* timeDiffMs= */ 40);
// Second frame is 31 ms late.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 9_000,
/* positionUs= */ 40_000,
/* elapsedRealtimeUs= */ 40_000,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP);
}
@Test
public void getFrameReleaseAction_frameLateWhileJoining_returnsSkip() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 1234);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
clock.advanceTime(/* timeDiffMs= */ 40);
// Start joining.
videoFrameReleaseControl.join();
// Second frame is 31 ms late.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 9_000,
/* positionUs= */ 40_000,
/* elapsedRealtimeUs= */ 40_000,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP);
}
@Test
public void getFrameReleaseAction_frameVeryLate_returnsDropToKeyframe() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
new VideoFrameReleaseControl(
ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.setFrameTimingEvaluator(
new TestFrameTimingEvaluator(/* shouldForceRelease= */ false));
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
clock.advanceTime(/* timeDiffMs= */ 1_000);
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 1_000,
/* positionUs= */ 1_000_000,
/* elapsedRealtimeUs= */ 1_000_000,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME);
}
@Test
public void getFrameReleaseAction_frameVeryLateAndJoining_returnsSkipToFrame() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 1234);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
clock.advanceTime(/* timeDiffMs= */ 1_000);
videoFrameReleaseControl.join();
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 1_000,
/* positionUs= */ 1_000_000,
/* elapsedRealtimeUs= */ 1_000_000,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME);
}
@Test
public void
getFrameReleaseAction_frameVeryLateAndFrameTimeEvaluatorForcesFrameRelease_returnsReleaseImmediately() {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
new VideoFrameReleaseControl(
ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0);
videoFrameReleaseControl.setFrameTimingEvaluator(
/* frameTimingEvaluator= */ new TestFrameTimingEvaluator(/* shouldForceRelease= */ true));
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();
// First frame released.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
clock.advanceTime(/* timeDiffMs= */ 1_000);
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 1_000,
/* positionUs= */ 1_000_000,
/* elapsedRealtimeUs= */ 1_000_000,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
}
private static VideoFrameReleaseControl createVideoFrameReleaseControl() {
return createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 0);
}
private static VideoFrameReleaseControl createVideoFrameReleaseControl(
long allowedJoiningTimeMs) {
return new VideoFrameReleaseControl(
ApplicationProvider.getApplicationContext(), allowedJoiningTimeMs);
}
private static class TestFrameTimingEvaluator
implements VideoFrameReleaseControl.FrameTimingEvaluator {
private final boolean shouldForceRelease;
public TestFrameTimingEvaluator(boolean shouldForceRelease) {
this.shouldForceRelease = shouldForceRelease;
}
@Override
public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) {
return shouldForceRelease;
}
@Override
public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFrame(
earlyUs, elapsedRealtimeUs, isLastFrame);
}
@Override
public boolean shouldDropFramesToKeyframe(
long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFramesToKeyframe(
earlyUs, elapsedRealtimeUs, isLastFrame);
}
}
}