diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java index 101dc6f258..5404a20545 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java @@ -92,6 +92,7 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { this.index = index; this.playerId = playerId; this.clock = clock; + onInit(); } @Override @@ -263,6 +264,11 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { // Methods to be overridden by subclasses. + /** Called when the renderer is initialized. */ + protected void onInit() { + // Do nothing + } + /** * Called when the renderer is enabled. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index e5bb66e70c..9489f3ee94 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -40,6 +40,7 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.VideoSize; +import androidx.media3.common.util.Clock; import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.Size; import androidx.media3.common.util.TimedValueQueue; @@ -62,7 +63,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Context context; private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; - private final VideoSink.RenderControl renderControl; + private final VideoFrameReleaseControl videoFrameReleaseControl; @Nullable private VideoSinkImpl videoSinkImpl; @Nullable private List videoEffects; @@ -73,7 +74,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public CompositingVideoSinkProvider( Context context, VideoFrameProcessor.Factory videoFrameProcessorFactory, - VideoSink.RenderControl renderControl) { + VideoFrameReleaseControl renderControl) { this( context, new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory), @@ -84,10 +85,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* package */ CompositingVideoSinkProvider( Context context, PreviewingVideoGraph.Factory previewingVideoGraphFactory, - VideoSink.RenderControl renderControl) { + VideoFrameReleaseControl releaseControl) { this.context = context; this.previewingVideoGraphFactory = previewingVideoGraphFactory; - this.renderControl = renderControl; + this.videoFrameReleaseControl = releaseControl; } @Override @@ -97,7 +98,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; try { videoSinkImpl = - new VideoSinkImpl(context, previewingVideoGraphFactory, renderControl, sourceFormat); + new VideoSinkImpl( + context, previewingVideoGraphFactory, videoFrameReleaseControl, sourceFormat); } catch (VideoFrameProcessingException e) { throw new VideoSink.VideoSinkException(e, sourceFormat); } @@ -173,7 +175,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listener { private final Context context; - private final VideoSink.RenderControl renderControl; + private final VideoFrameReleaseControl videoFrameReleaseControl; + private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo; private final VideoFrameProcessor videoFrameProcessor; private final LongArrayQueue processedFramesBufferTimestampsUs; private final TimedValueQueue streamOffsets; @@ -207,11 +210,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private VideoSize processedFrameSize; private VideoSize reportedVideoSize; private boolean pendingVideoSizeChange; - private boolean renderedFirstFrame; private long inputStreamOffsetUs; private boolean pendingInputStreamOffsetChange; private long outputStreamOffsetUs; - private float playbackSpeed; // TODO b/292111083 - Remove the field and trigger the callback on every video size change. private boolean onVideoSizeChangedCalled; @@ -220,11 +221,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public VideoSinkImpl( Context context, PreviewingVideoGraph.Factory previewingVideoGraphFactory, - RenderControl renderControl, + VideoFrameReleaseControl videoFrameReleaseControl, Format sourceFormat) throws VideoFrameProcessingException { this.context = context; - this.renderControl = renderControl; + this.videoFrameReleaseControl = videoFrameReleaseControl; + videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); processedFramesBufferTimestampsUs = new LongArrayQueue(); streamOffsets = new TimedValueQueue<>(); videoSizeChanges = new TimedValueQueue<>(); @@ -237,7 +239,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; lastCodecBufferPresentationTimestampUs = C.TIME_UNSET; processedFrameSize = VideoSize.UNKNOWN; reportedVideoSize = VideoSize.UNKNOWN; - playbackSpeed = 1f; // Playback thread handler. handler = Util.createHandlerForCurrentLooper(); @@ -293,7 +294,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; processedFramesBufferTimestampsUs.clear(); streamOffsets.clear(); handler.removeCallbacksAndMessages(/* token= */ null); - renderedFirstFrame = false; + videoFrameReleaseControl.reset(); if (registeredLastFrame) { registeredLastFrame = false; processedLastFrame = false; @@ -303,7 +304,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public boolean isReady() { - return renderedFirstFrame; + return videoFrameReleaseControl.isReady(/* rendererReady= */ true); } @Override @@ -385,47 +386,50 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long bufferPresentationTimeUs = processedFramesBufferTimestampsUs.element(); // check whether this buffer comes with a new stream offset. if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) { - renderedFirstFrame = false; + videoFrameReleaseControl.onProcessedStreamChange(); } long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1; - long frameRenderTimeNs = - renderControl.getFrameRenderTimeNs( - bufferPresentationTimeUs, positionUs, elapsedRealtimeUs, playbackSpeed); - if (frameRenderTimeNs == RenderControl.RENDER_TIME_TRY_AGAIN_LATER) { - return; - } else if (framePresentationTimeUs == RenderControl.RENDER_TIME_DROP) { - // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush - // VideoFrameProcessor input frames in this case. - releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); - continue; + @VideoFrameReleaseControl.FrameReleaseAction + int frameReleaseAction = + videoFrameReleaseControl.getFrameReleaseAction( + bufferPresentationTimeUs, + positionUs, + elapsedRealtimeUs, + outputStreamOffsetUs, + isLastFrame, + videoFrameReleaseInfo); + switch (frameReleaseAction) { + case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER: + return; + case VideoFrameReleaseControl.FRAME_RELEASE_SKIP: + case VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME: + // Skipped frames are dropped. + case VideoFrameReleaseControl.FRAME_RELEASE_DROP: + case VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME: + // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush + // VideoFrameProcessor input frames in this case. + dropFrame(isLastFrame); + break; + case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: + case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED: + renderFrame( + framePresentationTimeUs, bufferPresentationTimeUs, frameReleaseAction, isLastFrame); + break; + default: + throw new IllegalStateException(String.valueOf(frameReleaseAction)); } - renderControl.onNextFrame(bufferPresentationTimeUs); - if (videoFrameMetadataListener != null) { - videoFrameMetadataListener.onVideoFrameAboutToBeRendered( - framePresentationTimeUs, - frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY - ? System.nanoTime() - : frameRenderTimeNs, - checkNotNull(inputFormat), - /* mediaFormat= */ null); - } - releaseProcessedFrameInternal( - frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY - ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY - : frameRenderTimeNs, - isLastFrame); - - maybeNotifyVideoSizeChanged(bufferPresentationTimeUs); } } @Override public void setPlaybackSpeed(float speed) { checkArgument(speed >= 0.0); - this.playbackSpeed = speed; + videoFrameReleaseControl.setPlaybackSpeed(speed); } + // VideoGraph.Listener methods + @Override public void onOutputSizeChanged(int width, int height) { VideoSize newVideoSize = new VideoSize(width, height); @@ -477,12 +481,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new IllegalStateException(); } + // Other methods + public void release() { videoFrameProcessor.release(); handler.removeCallbacksAndMessages(/* token= */ null); streamOffsets.clear(); processedFramesBufferTimestampsUs.clear(); - renderedFirstFrame = false; } /** Sets the {@linkplain Effect video effects} to apply immediately. */ @@ -541,18 +546,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; && currentSurfaceAndSize.second.equals(outputResolution)) { return; } - renderedFirstFrame = - currentSurfaceAndSize == null || currentSurfaceAndSize.first.equals(outputSurface); + videoFrameReleaseControl.setOutputSurface(outputSurface); currentSurfaceAndSize = Pair.create(outputSurface, outputResolution); videoFrameProcessor.setOutputSurfaceInfo( new SurfaceInfo( outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); } + /** Clears the output surface info. */ public void clearOutputSurfaceInfo() { videoFrameProcessor.setOutputSurfaceInfo(null); currentSurfaceAndSize = null; - renderedFirstFrame = false; } private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) { @@ -565,21 +569,52 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return updatedOffset; } + private void dropFrame(boolean isLastFrame) { + if (listenerExecutor != null) { + listenerExecutor.execute( + () -> { + if (listener != null) { + listener.onFrameDropped(this); + } + }); + } + releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); + } + + private void renderFrame( + long framePresentationTimeUs, + long bufferPresentationTimeUs, + @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction, + boolean isLastFrame) { + if (videoFrameMetadataListener != null) { + videoFrameMetadataListener.onVideoFrameAboutToBeRendered( + framePresentationTimeUs, + frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY + ? Clock.DEFAULT.nanoTime() + : videoFrameReleaseInfo.getReleaseTimeNs(), + checkNotNull(inputFormat), + /* mediaFormat= */ null); + } + if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && listenerExecutor != null) { + listenerExecutor.execute( + () -> { + if (listener != null) { + listener.onFirstFrameRendered(/* videoSink= */ this); + } + }); + } + releaseProcessedFrameInternal( + frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY + ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY + : videoFrameReleaseInfo.getReleaseTimeNs(), + isLastFrame); + + maybeNotifyVideoSizeChanged(bufferPresentationTimeUs); + } + private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) { videoFrameProcessor.renderOutputFrame(releaseTimeNs); processedFramesBufferTimestampsUs.remove(); - if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME) { - renderControl.onFrameDropped(); - } else { - renderControl.onFrameRendered(); - if (!renderedFirstFrame) { - if (listener != null) { - checkNotNull(listenerExecutor) - .execute(() -> checkNotNull(listener).onFirstFrameRendered(this)); - } - renderedFirstFrame = true; - } - } if (isLastFrame) { releasedLastFrame = true; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 323e1198b8..75f42f91fc 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -18,7 +18,7 @@ package androidx.media3.exoplayer.video; import static android.view.Display.DEFAULT_DISPLAY; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Util.msToUs; +import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_VIDEO_MAX_RESOLUTION_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO; @@ -38,7 +38,6 @@ import android.media.MediaFormat; import android.os.Bundle; import android.os.Handler; import android.os.Message; -import android.os.SystemClock; import android.util.Pair; import android.view.Display; import android.view.Surface; @@ -57,7 +56,6 @@ import androidx.media3.common.PlaybackException; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; -import androidx.media3.common.util.Clock; import androidx.media3.common.util.Log; import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.Size; @@ -88,8 +86,8 @@ import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.Executor; -import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * Decodes and renders video using {@link MediaCodec}. @@ -115,7 +113,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * */ @UnstableApi -public class MediaCodecVideoRenderer extends MediaCodecRenderer implements VideoSink.RenderControl { +public class MediaCodecVideoRenderer extends MediaCodecRenderer + implements VideoFrameReleaseControl.FrameTimingEvaluator { private static final String TAG = "MediaCodecVideoRenderer"; private static final String KEY_CROP_LEFT = "crop-left"; @@ -139,19 +138,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video /** The minimum input buffer size for HEVC. */ private static final int HEVC_MAX_INPUT_SIZE_THRESHOLD = 2 * 1024 * 1024; - /** The maximum earliest time, in microseconds, to release a frame on the surface. */ - private static final long MAX_EARLY_US_THRESHOLD = 50_000; - private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround; private static boolean deviceNeedsSetOutputSurfaceWorkaround; private final Context context; - private final VideoFrameReleaseHelper frameReleaseHelper; private final VideoSinkProvider videoSinkProvider; private final EventDispatcher eventDispatcher; - private final long allowedJoiningTimeMs; private final int maxDroppedFramesToNotify; private final boolean deviceNeedsNoPostProcessWorkaround; + private final VideoFrameReleaseControl videoFrameReleaseControl; + private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo; private @MonotonicNonNull CodecMaxValues codecMaxValues; private boolean codecNeedsSetOutputSurfaceWorkaround; @@ -160,15 +156,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video @Nullable private PlaceholderSurface placeholderSurface; private boolean haveReportedFirstFrameRenderedForCurrentSurface; private @C.VideoScalingMode int scalingMode; - private @C.FirstFrameState int firstFrameState; - private long initialPositionUs; - private long joiningDeadlineMs; private long droppedFrameAccumulationStartTimeMs; private int droppedFrames; private int consecutiveDroppedFrameCount; private int buffersInCodecCount; - private long lastBufferPresentationTimeUs; - private long lastRenderRealtimeUs; private long totalVideoFrameProcessingOffsetUs; private int videoFrameProcessingOffsetCount; private long lastFrameReleaseTimeNs; @@ -393,23 +384,42 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video mediaCodecSelector, enableDecoderFallback, assumedMinimumCodecOperatingRate); - this.allowedJoiningTimeMs = allowedJoiningTimeMs; this.maxDroppedFramesToNotify = maxDroppedFramesToNotify; this.context = context.getApplicationContext(); - frameReleaseHelper = new VideoFrameReleaseHelper(this.context); + + videoFrameReleaseControl = new VideoFrameReleaseControl(this.context, allowedJoiningTimeMs); + videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); eventDispatcher = new EventDispatcher(eventHandler, eventListener); - @SuppressWarnings("nullness:assignment") - VideoSink.@Initialized RenderControl renderControl = this; videoSinkProvider = - new CompositingVideoSinkProvider(context, videoFrameProcessorFactory, renderControl); + new CompositingVideoSinkProvider( + context, videoFrameProcessorFactory, videoFrameReleaseControl); deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround(); - joiningDeadlineMs = C.TIME_UNSET; scalingMode = C.VIDEO_SCALING_MODE_DEFAULT; decodedVideoSize = VideoSize.UNKNOWN; tunnelingAudioSessionId = C.AUDIO_SESSION_ID_UNSET; - firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; + reportedVideoSize = null; } + // FrameTimingEvaluator methods + + @Override + public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { + return shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastReleaseUs); + } + + @Override + public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { + return shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastFrame); + } + + @Override + public boolean shouldDropFramesToKeyframe( + long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { + return shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastFrame); + } + + // Renderer methods + @Override public String getName() { return TAG; @@ -523,53 +533,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video format); } - // RenderControl implementation - - @Override - public long getFrameRenderTimeNs( - long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) { - long earlyUs = - calculateEarlyTimeUs( - positionUs, - elapsedRealtimeUs, - presentationTimeUs, - getState() == STATE_STARTED, - playbackSpeed, - getClock()); - if (isBufferLate(earlyUs)) { - return VideoSink.RenderControl.RENDER_TIME_DROP; - } - if (shouldForceRender(positionUs, earlyUs)) { - return VideoSink.RenderControl.RENDER_TIME_IMMEDIATELY; - } - - if (getState() != STATE_STARTED - || positionUs == initialPositionUs - || earlyUs > MAX_EARLY_US_THRESHOLD) { - return VideoSink.RenderControl.RENDER_TIME_TRY_AGAIN_LATER; - } - // Compute the buffer's desired release time in nanoseconds. - long unadjustedFrameReleaseTimeNs = getClock().nanoTime() + (earlyUs * 1000); - // Apply a timestamp adjustment, if there is one. - return frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); - } - - @Override - public void onNextFrame(long presentationTimeUs) { - frameReleaseHelper.onNextFrame(presentationTimeUs); - } - - @Override - public void onFrameRendered() { - lastRenderRealtimeUs = Util.msToUs(getClock().elapsedRealtime()); - } - - @Override - public void onFrameDropped() { - updateDroppedBufferCounters( - /* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1); - } - // Other methods /** @@ -634,6 +597,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } } + @Override + protected void onInit() { + super.onInit(); + videoFrameReleaseControl.setFrameTimingEvaluator(/* frameTimingEvaluator= */ this); + videoFrameReleaseControl.setClock(getClock()); + } + @Override protected void onEnabled(boolean joining, boolean mayRenderStartOfStream) throws ExoPlaybackException { @@ -645,17 +615,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video releaseCodec(); } eventDispatcher.enabled(decoderCounters); - firstFrameState = - mayRenderStartOfStream - ? C.FIRST_FRAME_NOT_RENDERED - : C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; + videoFrameReleaseControl.onEnabled(mayRenderStartOfStream); } @Override public void enableMayRenderStartOfStream() { - if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) { - firstFrameState = C.FIRST_FRAME_NOT_RENDERED; - } + videoFrameReleaseControl.allowReleaseFirstFrameBeforeStarted(); } @Override @@ -666,21 +631,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video videoSink.flush(); } super.onPositionReset(positionUs, joining); - if (videoSinkProvider.isInitialized()) { videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); } - - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); - frameReleaseHelper.onPositionReset(); - lastBufferPresentationTimeUs = C.TIME_UNSET; - initialPositionUs = C.TIME_UNSET; - consecutiveDroppedFrameCount = 0; + videoFrameReleaseControl.reset(); if (joining) { - setJoiningDeadlineMs(); - } else { - joiningDeadlineMs = C.TIME_UNSET; + videoFrameReleaseControl.join(); } + maybeUpdateOnFrameRenderedListener(); + consecutiveDroppedFrameCount = 0; } @Override @@ -690,26 +649,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video @Override public boolean isReady() { - if (super.isReady() - && (videoSink == null || videoSink.isReady()) - && (firstFrameState == C.FIRST_FRAME_RENDERED - || (placeholderSurface != null && displaySurface == placeholderSurface) + boolean readyToReleaseFrames = super.isReady() && (videoSink == null || videoSink.isReady()); + if (readyToReleaseFrames + && ((placeholderSurface != null && displaySurface == placeholderSurface) || getCodec() == null || tunneling)) { - // Ready. If we were joining then we've now joined, so clear the joining deadline. - joiningDeadlineMs = C.TIME_UNSET; + // Not releasing frames. return true; - } else if (joiningDeadlineMs == C.TIME_UNSET) { - // Not joining. - return false; - } else if (getClock().elapsedRealtime() < joiningDeadlineMs) { - // Joining and still within the joining deadline. - return true; - } else { - // The joining deadline has been exceeded. Give up and clear the deadline. - joiningDeadlineMs = C.TIME_UNSET; - return false; } + return videoFrameReleaseControl.isReady(readyToReleaseFrames); } @Override @@ -718,25 +666,24 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video droppedFrames = 0; long elapsedRealtimeMs = getClock().elapsedRealtime(); droppedFrameAccumulationStartTimeMs = elapsedRealtimeMs; - lastRenderRealtimeUs = msToUs(elapsedRealtimeMs); totalVideoFrameProcessingOffsetUs = 0; videoFrameProcessingOffsetCount = 0; - frameReleaseHelper.onStarted(); + videoFrameReleaseControl.onStarted(); } @Override protected void onStopped() { - joiningDeadlineMs = C.TIME_UNSET; maybeNotifyDroppedFrames(); maybeNotifyVideoFrameProcessingOffset(); - frameReleaseHelper.onStopped(); + videoFrameReleaseControl.onStopped(); super.onStopped(); } @Override protected void onDisabled() { reportedVideoSize = null; - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED); + videoFrameReleaseControl.onDisabled(); + maybeUpdateOnFrameRenderedListener(); haveReportedFirstFrameRenderedForCurrentSurface = false; tunnelingOnFrameRenderedListener = null; try { @@ -783,7 +730,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } break; case MSG_SET_CHANGE_FRAME_RATE_STRATEGY: - frameReleaseHelper.setChangeFrameRateStrategy((int) checkNotNull(message)); + videoFrameReleaseControl.setChangeFrameRateStrategy((int) checkNotNull(message)); break; case MSG_SET_VIDEO_FRAME_METADATA_LISTENER: frameMetadataListener = (VideoFrameMetadataListener) checkNotNull(message); @@ -843,7 +790,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video // We only need to update the codec if the display surface has changed. if (this.displaySurface != displaySurface) { this.displaySurface = displaySurface; - frameReleaseHelper.onSurfaceChanged(displaySurface); + videoFrameReleaseControl.setOutputSurface(displaySurface); haveReportedFirstFrameRenderedForCurrentSurface = false; @State int state = getState(); @@ -862,11 +809,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video if (displaySurface != null && displaySurface != placeholderSurface) { // If we know the video size, report it again immediately. maybeRenotifyVideoSizeChanged(); - // We haven't rendered to the new display surface yet. - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); if (state == STATE_STARTED) { - // Set joining deadline to report MediaCodecVideoRenderer is ready. - setJoiningDeadlineMs(); + videoFrameReleaseControl.join(); } // When effects previewing is enabled, set display surface and an unknown size. if (videoSinkProvider.isInitialized()) { @@ -875,11 +819,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } else { // The display surface has been removed. reportedVideoSize = null; - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); if (videoSinkProvider.isInitialized()) { videoSinkProvider.clearOutputSurfaceInfo(); } } + maybeUpdateOnFrameRenderedListener(); } else if (displaySurface != null && displaySurface != placeholderSurface) { // The display surface is set and unchanged. If we know the video size and/or have already // rendered to the display surface, report these again immediately. @@ -987,7 +931,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video public void setPlaybackSpeed(float currentPlaybackSpeed, float targetPlaybackSpeed) throws ExoPlaybackException { super.setPlaybackSpeed(currentPlaybackSpeed, targetPlaybackSpeed); - frameReleaseHelper.onPlaybackSpeed(currentPlaybackSpeed); + videoFrameReleaseControl.setPlaybackSpeed(currentPlaybackSpeed); if (videoSink != null) { videoSink.setPlaybackSpeed(currentPlaybackSpeed); } @@ -1104,7 +1048,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video new VideoSink.Listener() { @Override public void onFirstFrameRendered(VideoSink videoSink) { - maybeNotifyRenderedFirstFrame(); + checkStateNotNull(displaySurface); + notifyRenderedFirstFrame(); + } + + @Override + public void onFrameDropped(VideoSink videoSink) { + updateDroppedBufferCounters( + /* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1); } @Override @@ -1245,7 +1196,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } decodedVideoSize = new VideoSize(width, height, unappliedRotationDegrees, pixelWidthHeightRatio); - frameReleaseHelper.onFormatChanged(format.frameRate); + videoFrameReleaseControl.setFrameRate(format.frameRate); if (videoSink != null && mediaFormat != null) { onReadyToRegisterVideoSinkInputStream(); @@ -1319,40 +1270,34 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video throws ExoPlaybackException { checkNotNull(codec); // Can not render video without codec - if (initialPositionUs == C.TIME_UNSET) { - initialPositionUs = positionUs; - } - - if (bufferPresentationTimeUs != lastBufferPresentationTimeUs) { - if (videoSink == null) { - frameReleaseHelper.onNextFrame(bufferPresentationTimeUs); - } // else, update the frameReleaseHelper when releasing the processed frames. - this.lastBufferPresentationTimeUs = bufferPresentationTimeUs; - } - long outputStreamOffsetUs = getOutputStreamOffsetUs(); long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; + @VideoFrameReleaseControl.FrameReleaseAction + int frameReleaseAction = + videoFrameReleaseControl.getFrameReleaseAction( + bufferPresentationTimeUs, + positionUs, + elapsedRealtimeUs, + getOutputStreamStartPositionUs(), + isLastBuffer, + videoFrameReleaseInfo); + + // Skip decode-only buffers, e.g. after seeking, immediately. This check must be performed after + // getting the release action from the video frame release control although not necessary. + // That's because the release control estimates the content frame rate from frame timestamps + // and we want to have this information known as early as possible, especially during seeking. if (isDecodeOnlyBuffer && !isLastBuffer) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } - boolean isStarted = getState() == STATE_STARTED; - long earlyUs = - calculateEarlyTimeUs( - positionUs, - elapsedRealtimeUs, - bufferPresentationTimeUs, - isStarted, - getPlaybackSpeed(), - getClock()); - + // We are not rendering on a surface, the renderer will wait until a surface is set. if (displaySurface == placeholderSurface) { // Skip frames in sync with playback, so we'll be at the right frame if the mode changes. - if (isBufferLate(earlyUs)) { + if (videoFrameReleaseInfo.getEarlyUs() < 30_000) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(earlyUs); + updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); return true; } return false; @@ -1368,137 +1313,86 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video return true; } - boolean forceRenderOutputBuffer = shouldForceRender(positionUs, earlyUs); - if (forceRenderOutputBuffer) { - long releaseTimeNs = getClock().nanoTime(); - notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); - renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); - updateVideoFrameProcessingOffsetCounters(earlyUs); - return true; - } - - if (!isStarted || positionUs == initialPositionUs) { - return false; - } - - // Compute the buffer's desired release time in nanoseconds. - long systemTimeNs = getClock().nanoTime(); - long unadjustedFrameReleaseTimeNs = systemTimeNs + (earlyUs * 1000); - // Apply a timestamp adjustment, if there is one. - long adjustedReleaseTimeNs = frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); - earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000; - boolean treatDroppedBuffersAsSkipped = joiningDeadlineMs != C.TIME_UNSET; - if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastBuffer) - && maybeDropBuffersToKeyframe(positionUs, treatDroppedBuffersAsSkipped)) { - return false; - } else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastBuffer)) { - if (treatDroppedBuffersAsSkipped) { + switch (frameReleaseAction) { + case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: + long releaseTimeNs = getClock().nanoTime(); + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); + renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); + updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); + return true; + case VideoFrameReleaseControl.FRAME_RELEASE_SKIP: skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - } else { + updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); + return true; + case VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME: + if (!maybeDropBuffersToKeyframe(positionUs, /* treatDroppedBuffersAsSkipped= */ true)) { + skipOutputBuffer(codec, bufferIndex, presentationTimeUs); + updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); + return true; + } else { + return false; + } + case VideoFrameReleaseControl.FRAME_RELEASE_DROP: dropOutputBuffer(codec, bufferIndex, presentationTimeUs); - } - updateVideoFrameProcessingOffsetCounters(earlyUs); - return true; + updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); + return true; + case VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME: + if (!maybeDropBuffersToKeyframe(positionUs, /* treatDroppedBuffersAsSkipped= */ false)) { + dropOutputBuffer(codec, bufferIndex, presentationTimeUs); + updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); + return true; + } else { + return false; + } + case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER: + return false; + case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED: + return maybeReleaseFrame(checkStateNotNull(codec), bufferIndex, presentationTimeUs, format); + default: + throw new IllegalStateException(String.valueOf(frameReleaseAction)); } + } + private boolean maybeReleaseFrame( + MediaCodecAdapter codec, int bufferIndex, long presentationTimeUs, Format format) { + long releaseTimeNs = videoFrameReleaseInfo.getReleaseTimeNs(); + long earlyUs = videoFrameReleaseInfo.getEarlyUs(); if (Util.SDK_INT >= 21) { // Let the underlying framework time the release. - if (earlyUs < MAX_EARLY_US_THRESHOLD) { - if (shouldSkipBuffersWithIdenticalReleaseTime() - && adjustedReleaseTimeNs == lastFrameReleaseTimeNs) { - // This frame should be displayed on the same vsync with the previous released frame. We - // are likely rendering frames at a rate higher than the screen refresh rate. Skip - // this buffer so that it's returned to MediaCodec sooner otherwise MediaCodec may not - // be able to keep decoding with this rate [b/263454203]. - skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - } else { - notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format); - renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs); - } - updateVideoFrameProcessingOffsetCounters(earlyUs); - lastFrameReleaseTimeNs = adjustedReleaseTimeNs; - return true; + if (shouldSkipBuffersWithIdenticalReleaseTime() && releaseTimeNs == lastFrameReleaseTimeNs) { + // This frame should be displayed on the same vsync with the previous released frame. We + // are likely rendering frames at a rate higher than the screen refresh rate. Skip + // this buffer so that it's returned to MediaCodec sooner otherwise MediaCodec may not + // be able to keep decoding with this rate [b/263454203]. + skipOutputBuffer(codec, bufferIndex, presentationTimeUs); + } else { + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); + renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs); } - } else { + updateVideoFrameProcessingOffsetCounters(earlyUs); + lastFrameReleaseTimeNs = releaseTimeNs; + return true; + } else if (earlyUs < 30000) { // We need to time the release ourselves. - if (earlyUs < 30000) { - if (earlyUs > 11000) { - // We're a little too early to render the frame. Sleep until the frame can be rendered. - // Note: The 11ms threshold was chosen fairly arbitrarily. - try { - // Subtracting 10000 rather than 11000 ensures the sleep time will be at least 1ms. - Thread.sleep((earlyUs - 10000) / 1000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } + if (earlyUs > 11000) { + // We're a little too early to render the frame. Sleep until the frame can be rendered. + // Note: The 11ms threshold was chosen fairly arbitrarily. + try { + // Subtracting 10000 rather than 11000 ensures the sleep time will be at least 1ms. + Thread.sleep((earlyUs - 10000) / 1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; } - notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format); - renderOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(earlyUs); - return true; } - } - - // We're either not playing, or it's not time to render the frame yet. - return false; - } - - /** Returns whether a buffer or a processed frame should be force rendered. */ - private boolean shouldForceRender(long positionUs, long earlyUs) { - if (joiningDeadlineMs != C.TIME_UNSET) { - // No force rendering during joining. + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); + renderOutputBuffer(codec, bufferIndex, presentationTimeUs); + updateVideoFrameProcessingOffsetCounters(earlyUs); + return true; + } else { + // Too soon. return false; } - boolean isStarted = getState() == STATE_STARTED; - switch (firstFrameState) { - case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED: - return isStarted; - case C.FIRST_FRAME_NOT_RENDERED: - return true; - case C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE: - return positionUs >= getOutputStreamStartPositionUs(); - case C.FIRST_FRAME_RENDERED: - long elapsedSinceLastRenderUs = msToUs(getClock().elapsedRealtime()) - lastRenderRealtimeUs; - return isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs); - default: - throw new IllegalStateException(); - } - } - - /** - * Calculates the time interval between the current player position and the buffer presentation - * time. - * - * @param positionUs The current media time in microseconds, measured at the start of the current - * iteration of the rendering loop. - * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the - * start of the current iteration of the rendering loop. - * @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds, - * with {@linkplain #getOutputStreamOffsetUs() stream offset added}. - * @param isStarted Whether the playback is in {@link #STATE_STARTED}. - * @param playbackSpeed The current playback speed. - * @param clock The {@link Clock} used by the renderer. - * @return The calculated early time, in microseconds. - */ - private static long calculateEarlyTimeUs( - long positionUs, - long elapsedRealtimeUs, - long bufferPresentationTimeUs, - boolean isStarted, - float playbackSpeed, - Clock clock) { - // Calculate how early we are. In other words, the realtime duration that needs to elapse whilst - // the renderer is started before the frame should be rendered. A negative value means that - // we're already late. - // Note: Use of double rather than float is intentional for accuracy in the calculations below. - long earlyUs = (long) ((bufferPresentationTimeUs - positionUs) / (double) playbackSpeed); - if (isStarted) { - // Account for the elapsed time since the start of this iteration of the rendering loop. - earlyUs -= Util.msToUs(clock.elapsedRealtime()) - elapsedRealtimeUs; - } - - return earlyUs; } private void notifyFrameMetadataListener( @@ -1535,7 +1429,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video @Override protected void onProcessedStreamChange() { super.onProcessedStreamChange(); - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE); + videoFrameReleaseControl.onProcessedStreamChange(); + maybeUpdateOnFrameRenderedListener(); if (videoSinkProvider.isInitialized()) { videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); } @@ -1552,7 +1447,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video */ protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { - return isBufferLate(earlyUs) && !isLastBuffer; + return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFrame( + earlyUs, elapsedRealtimeUs, isLastBuffer); } /** @@ -1567,7 +1463,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video */ protected boolean shouldDropBuffersToKeyframe( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { - return isBufferVeryLate(earlyUs) && !isLastBuffer; + return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFramesToKeyframe( + earlyUs, elapsedRealtimeUs, isLastBuffer); } /** @@ -1588,8 +1485,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video * @return Returns whether to force rendering an output buffer. */ protected boolean shouldForceRenderOutputBuffer(long earlyUs, long elapsedSinceLastRenderUs) { - // Force render late buffers every 100ms to avoid frozen video effect. - return isBufferLate(earlyUs) && elapsedSinceLastRenderUs > 100000; + return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldForceReleaseFrame( + earlyUs, elapsedSinceLastRenderUs); } /** @@ -1721,7 +1618,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; if (videoSink == null) { - lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime()); maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); } @@ -1745,7 +1641,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; if (videoSink == null) { - lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime()); maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); } @@ -1769,15 +1664,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } } - private void setJoiningDeadlineMs() { - joiningDeadlineMs = - allowedJoiningTimeMs > 0 - ? (getClock().elapsedRealtime() + allowedJoiningTimeMs) - : C.TIME_UNSET; - } - - private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) { - this.firstFrameState = min(this.firstFrameState, firstFrameState); + private void maybeUpdateOnFrameRenderedListener() { // The first frame notification is triggered by renderOutputBuffer or renderOutputBufferV21 for // non-tunneled playback, onQueueInputBuffer for tunneled playback prior to API level 23, and // OnFrameRenderedListenerV23.onFrameRenderedListener for tunneled playback on API level 23 and @@ -1792,13 +1679,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } private void maybeNotifyRenderedFirstFrame() { - if (displaySurface != null && firstFrameState != C.FIRST_FRAME_RENDERED) { - firstFrameState = C.FIRST_FRAME_RENDERED; - eventDispatcher.renderedFirstFrame(displaySurface); - haveReportedFirstFrameRenderedForCurrentSurface = true; + if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && displaySurface != null) { + notifyRenderedFirstFrame(); } } + @RequiresNonNull("displaySurface") + private void notifyRenderedFirstFrame() { + eventDispatcher.renderedFirstFrame(displaySurface); + haveReportedFirstFrameRenderedForCurrentSurface = true; + } + private void maybeRenotifyRenderedFirstFrame() { if (displaySurface != null && haveReportedFirstFrameRenderedForCurrentSurface) { eventDispatcher.renderedFirstFrame(displaySurface); @@ -1838,16 +1729,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } } - private static boolean isBufferLate(long earlyUs) { - // Class a buffer as late if it should have been presented more than 30 ms ago. - return earlyUs < -30000; - } - - private static boolean isBufferVeryLate(long earlyUs) { - // Class a buffer as very late if it should have been presented more than 500 ms ago. - return earlyUs < -500000; - } - @RequiresApi(29) private static void setHdr10PlusInfoV29(MediaCodecAdapter codec, byte[] hdr10PlusInfo) { Bundle codecParameters = new Bundle(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java new file mode 100644 index 0000000000..4e99c405d2 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java @@ -0,0 +1,466 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.video; + +import static androidx.media3.common.util.Util.msToUs; +import static java.lang.Math.min; +import static java.lang.annotation.ElementType.TYPE_USE; + +import android.content.Context; +import android.os.SystemClock; +import android.view.Surface; +import androidx.annotation.IntDef; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.util.Clock; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.Renderer; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Controls the releasing of video frames. */ +/* package */ final class VideoFrameReleaseControl { + + /** + * The frame release action returned by {@link #getFrameReleaseAction(long, long, long, long, + * boolean, FrameReleaseInfo)}. + * + *

One of {@link #FRAME_RELEASE_IMMEDIATELY}, {@link #FRAME_RELEASE_SCHEDULED}, {@link + * #FRAME_RELEASE_DROP}, {@link #FRAME_RELEASE_DROP_TO_KEYFRAME}, {@link ##FRAME_RELEASE_SKIP} or + * {@link #FRAME_RELEASE_TRY_AGAIN_LATER}. + */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @UnstableApi + @IntDef({ + FRAME_RELEASE_IMMEDIATELY, + FRAME_RELEASE_SCHEDULED, + FRAME_RELEASE_DROP, + FRAME_RELEASE_DROP_TO_KEYFRAME, + FRAME_RELEASE_SKIP, + FRAME_RELEASE_SKIP_TO_KEYFRAME, + FRAME_RELEASE_TRY_AGAIN_LATER + }) + public @interface FrameReleaseAction {} + + /** Signals a frame should be released immediately. */ + public static final int FRAME_RELEASE_IMMEDIATELY = 0; + + /** + * Signals a frame should be scheduled for release. The release timestamp will be returned by + * {@link FrameReleaseInfo#getReleaseTimeNs()}. + */ + public static final int FRAME_RELEASE_SCHEDULED = 1; + + /** Signals a frame should be dropped. */ + public static final int FRAME_RELEASE_DROP = 2; + + /** Signals frames up to the next key-frame should be dropped. */ + public static final int FRAME_RELEASE_DROP_TO_KEYFRAME = 3; + + /** Signals that a frame should be skipped. */ + public static final int FRAME_RELEASE_SKIP = 4; + + /** Signals that frames up to the next key-frame should be skipped. */ + public static final int FRAME_RELEASE_SKIP_TO_KEYFRAME = 5; + + /** Signals that a frame should not be released and the renderer should try again later. */ + public static final int FRAME_RELEASE_TRY_AGAIN_LATER = 6; + + /** Per {@link FrameReleaseAction} metadata. */ + public static class FrameReleaseInfo { + private long earlyUs; + private long releaseTimeNs; + + /** Resets this instances state. */ + public FrameReleaseInfo() { + earlyUs = C.TIME_UNSET; + releaseTimeNs = C.TIME_UNSET; + } + + /** + * Returns this frame's early time compared to the playback position, before any release time + * adjustment to the screen vsync slots. + */ + public long getEarlyUs() { + return earlyUs; + } + + /** + * Returns the release time for the frame, in nanoseconds, or {@link C#TIME_UNSET} if the frame + * should not be released yet. + */ + public long getReleaseTimeNs() { + return releaseTimeNs; + } + + private void reset() { + earlyUs = C.TIME_UNSET; + releaseTimeNs = C.TIME_UNSET; + } + } + + /** Decides whether a frame should be forced to be released, or dropped. */ + public interface FrameTimingEvaluator { + /** + * Whether a frame should be forced for release. + * + * @param earlyUs The time until the buffer should be presented in microseconds. A negative + * value indicates that the buffer is late. + * @param elapsedSinceLastReleaseUs The elapsed time since the last frame was released, in + * microseconds. + * @return Whether the video frame should be force released. + */ + boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs); + + /** + * Returns whether the frame should be dropped. + * + * @param earlyUs The time until the buffer should be presented in microseconds. A negative + * value indicates that the buffer is late. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * measured at the start of the current iteration of the rendering loop. + * @param isLastFrame Whether the buffer is the last buffer in the current stream. + */ + boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame); + + /** + * Returns whether to drop all frames starting from this frame to the keyframe at or after the + * current playback position, if possible. + * + * @param earlyUs The time until the current buffer should be presented in microseconds. A + * negative value indicates that the buffer is late. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * measured at the start of the current iteration of the rendering loop. + * @param isLastFrame Whether the frame is the last frame in the current stream. + */ + boolean shouldDropFramesToKeyframe(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame); + + /** The default timing evaluator. */ + FrameTimingEvaluator DEFAULT = + new FrameTimingEvaluator() { + @Override + public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { + // Force render late buffers every 100ms to avoid frozen video effect. + return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && elapsedSinceLastReleaseUs > 100_000; + } + + @Override + public boolean shouldDropFrame( + long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { + return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastFrame; + } + + @Override + public boolean shouldDropFramesToKeyframe( + long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { + return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastFrame; + } + }; + } + + /** The earliest time threshold, in microseconds, after which a frame is considered late. */ + private static final long MIN_EARLY_US_LATE_THRESHOLD = -30_000; + + /** The earliest time threshold, in microseconds, after which a frame is considered very late. */ + private static final long MIN_EARLY_US_VERY_LATE_THRESHOLD = -500_000; + + /** The maximum earliest time, in microseconds, to release a frame on the surface. */ + private static final long MAX_EARLY_US_THRESHOLD = 50_000; + + private final VideoFrameReleaseHelper frameReleaseHelper; + private final long allowedJoiningTimeMs; + + private FrameTimingEvaluator frameTimingEvaluator; + private boolean started; + private @C.FirstFrameState int firstFrameState; + private long initialPositionUs; + private long lastReleaseRealtimeUs; + private long lastPresentationTimeUs; + private long joiningDeadlineMs; + private float playbackSpeed; + private Clock clock; + + /** + * Creates an instance. + * + * @param applicationContext The application context. + * @param allowedJoiningTimeMs The maximum duration in milliseconds for which the renderer can + * attempt to seamlessly join an ongoing playback. + */ + public VideoFrameReleaseControl(Context applicationContext, long allowedJoiningTimeMs) { + frameReleaseHelper = new VideoFrameReleaseHelper(applicationContext); + this.allowedJoiningTimeMs = allowedJoiningTimeMs; + this.frameTimingEvaluator = FrameTimingEvaluator.DEFAULT; + firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; + initialPositionUs = C.TIME_UNSET; + lastPresentationTimeUs = C.TIME_UNSET; + joiningDeadlineMs = C.TIME_UNSET; + playbackSpeed = 1f; + clock = Clock.DEFAULT; + } + + /** Sets the {@link FrameTimingEvaluator}. */ + public void setFrameTimingEvaluator(FrameTimingEvaluator frameTimingEvaluator) { + this.frameTimingEvaluator = frameTimingEvaluator; + } + + /** Called when the renderer is enabled. */ + public void onEnabled(boolean releaseFirstFrameBeforeStarted) { + firstFrameState = + releaseFirstFrameBeforeStarted + ? C.FIRST_FRAME_NOT_RENDERED + : C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; + } + + /** Called when the renderer is disabled. */ + public void onDisabled() { + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED); + } + + /** Called when the renderer is started. */ + public void onStarted() { + started = true; + lastReleaseRealtimeUs = msToUs(clock.elapsedRealtime()); + frameReleaseHelper.onStarted(); + } + + /** Called when the renderer is stopped. */ + public void onStopped() { + started = false; + joiningDeadlineMs = C.TIME_UNSET; + frameReleaseHelper.onStopped(); + } + + /** Called when the renderer processed a stream change. */ + public void onProcessedStreamChange() { + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE); + } + + /** Called when the display surface changed. */ + public void setOutputSurface(@Nullable Surface outputSurface) { + frameReleaseHelper.onSurfaceChanged(outputSurface); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); + } + + /** Sets the frame rate. */ + public void setFrameRate(float frameRate) { + frameReleaseHelper.onFormatChanged(frameRate); + } + + /** + * Called when a frame have been released. + * + * @return Whether this is the first released frame. + */ + public boolean onFrameReleasedIsFirstFrame() { + boolean firstFrame = firstFrameState != C.FIRST_FRAME_RENDERED; + firstFrameState = C.FIRST_FRAME_RENDERED; + lastReleaseRealtimeUs = msToUs(clock.elapsedRealtime()); + return firstFrame; + } + + /** Sets the clock that will be used. */ + public void setClock(Clock clock) { + this.clock = clock; + } + + /** + * Allows the frame control to indicate the first frame can be released before this instance is + * started. + */ + public void allowReleaseFirstFrameBeforeStarted() { + if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) { + firstFrameState = C.FIRST_FRAME_NOT_RENDERED; + } + } + + /** + * Whether the release control is ready to start playback. + * + * @see Renderer#isReady() + * @param rendererReady Whether the renderer is ready. + * @return Whether the release control is ready. + */ + public boolean isReady(boolean rendererReady) { + if (rendererReady && firstFrameState == C.FIRST_FRAME_RENDERED) { + // Ready. If we were joining then we've now joined, so clear the joining deadline. + joiningDeadlineMs = C.TIME_UNSET; + return true; + } else if (joiningDeadlineMs == C.TIME_UNSET) { + // Not joining. + return false; + } else if (clock.elapsedRealtime() < joiningDeadlineMs) { + // Joining and still withing the deadline. + return true; + } else { + // The joining deadline has been exceeded. Give up and clear the deadline. + joiningDeadlineMs = C.TIME_UNSET; + return false; + } + } + + /** Joins the release control to a new stream. */ + public void join() { + joiningDeadlineMs = + allowedJoiningTimeMs > 0 ? (clock.elapsedRealtime() + allowedJoiningTimeMs) : C.TIME_UNSET; + } + + /** + * Returns a {@link FrameReleaseAction} for a video frame which instructs a renderer what to do + * with the frame. + * + * @param presentationTimeUs The presentation time of the video frame, in microseconds. + * @param positionUs The current playback position, in microseconds. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * taken approximately at the time the playback position was {@code positionUs}. + * @param outputStreamStartPositionUs The stream's start position, in microseconds. + * @param isLastFrame Whether the frame is known to contain the last frame of the current stream. + * @param frameReleaseInfo A {@link FrameReleaseInfo} that will be filled with detailed data only + * if the method returns {@link #FRAME_RELEASE_IMMEDIATELY} or {@link + * #FRAME_RELEASE_SCHEDULED}. + * @return A {@link FrameReleaseAction} that should instruct the renderer whether to release the + * frame or not. + */ + public @FrameReleaseAction int getFrameReleaseAction( + long presentationTimeUs, + long positionUs, + long elapsedRealtimeUs, + long outputStreamStartPositionUs, + boolean isLastFrame, + FrameReleaseInfo frameReleaseInfo) { + frameReleaseInfo.reset(); + + if (initialPositionUs == C.TIME_UNSET) { + initialPositionUs = positionUs; + } + if (lastPresentationTimeUs != presentationTimeUs) { + frameReleaseHelper.onNextFrame(presentationTimeUs); + lastPresentationTimeUs = presentationTimeUs; + } + + frameReleaseInfo.earlyUs = + calculateEarlyTimeUs(positionUs, elapsedRealtimeUs, presentationTimeUs); + + if (shouldForceRelease(positionUs, frameReleaseInfo.earlyUs, outputStreamStartPositionUs)) { + return FRAME_RELEASE_IMMEDIATELY; + } + if (!started + || positionUs == initialPositionUs + || frameReleaseInfo.earlyUs > MAX_EARLY_US_THRESHOLD) { + return FRAME_RELEASE_TRY_AGAIN_LATER; + } + + // Calculate release time and and adjust earlyUs to screen vsync. + long systemTimeNs = clock.nanoTime(); + frameReleaseInfo.releaseTimeNs = + frameReleaseHelper.adjustReleaseTime(systemTimeNs + (frameReleaseInfo.earlyUs * 1_000)); + frameReleaseInfo.earlyUs = (frameReleaseInfo.releaseTimeNs - systemTimeNs) / 1_000; + // While joining, late frames are skipped. + boolean treatDropAsSkip = joiningDeadlineMs != C.TIME_UNSET; + if (frameTimingEvaluator.shouldDropFramesToKeyframe( + frameReleaseInfo.earlyUs, elapsedRealtimeUs, isLastFrame)) { + return treatDropAsSkip ? FRAME_RELEASE_SKIP_TO_KEYFRAME : FRAME_RELEASE_DROP_TO_KEYFRAME; + } else if (frameTimingEvaluator.shouldDropFrame( + frameReleaseInfo.earlyUs, elapsedRealtimeUs, isLastFrame)) { + // While joining, dropped buffers are considered skipped. + return treatDropAsSkip ? FRAME_RELEASE_SKIP : FRAME_RELEASE_DROP; + } + return FRAME_RELEASE_SCHEDULED; + } + + /** Resets the release control. */ + public void reset() { + frameReleaseHelper.onPositionReset(); + lastPresentationTimeUs = C.TIME_UNSET; + initialPositionUs = C.TIME_UNSET; + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); + joiningDeadlineMs = C.TIME_UNSET; + } + + /** + * Change the {@link C.VideoChangeFrameRateStrategy}, used when calling {@link + * Surface#setFrameRate}. + */ + public void setChangeFrameRateStrategy( + @C.VideoChangeFrameRateStrategy int changeFrameRateStrategy) { + frameReleaseHelper.setChangeFrameRateStrategy(changeFrameRateStrategy); + } + + /** Sets the playback speed. Called when the renderer playback speed changes. */ + public void setPlaybackSpeed(float speed) { + this.playbackSpeed = speed; + frameReleaseHelper.onPlaybackSpeed(speed); + } + + private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) { + this.firstFrameState = min(this.firstFrameState, firstFrameState); + } + + /** + * Calculates the time interval between the current player position and the frame presentation + * time. + * + * @param positionUs The current media time in microseconds, measured at the start of the current + * iteration of the rendering loop. + * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the + * start of the current iteration of the rendering loop. + * @param framePresentationTimeUs The presentation time of the frame in microseconds. + * @return The calculated early time, in microseconds. + */ + private long calculateEarlyTimeUs( + long positionUs, long elapsedRealtimeUs, long framePresentationTimeUs) { + // Calculate how early we are. In other words, the realtime duration that needs to elapse whilst + // the renderer is started before the frame should be rendered. A negative value means that + // we're already late. + // Note: Use of double rather than float is intentional for accuracy in the calculations below. + long earlyUs = (long) ((framePresentationTimeUs - positionUs) / (double) playbackSpeed); + if (started) { + // Account for the elapsed time since the start of this iteration of the rendering loop. + earlyUs -= Util.msToUs(clock.elapsedRealtime()) - elapsedRealtimeUs; + } + + return earlyUs; + } + + /** Returns whether a frame should be force released. */ + private boolean shouldForceRelease( + long positionUs, long earlyUs, long outputStreamStartPositionUs) { + if (joiningDeadlineMs != C.TIME_UNSET) { + // No force releasing during joining. + return false; + } + switch (firstFrameState) { + case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED: + return started; + case C.FIRST_FRAME_NOT_RENDERED: + return true; + case C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE: + return positionUs >= outputStreamStartPositionUs; + case C.FIRST_FRAME_RENDERED: + long elapsedTimeSinceLastReleaseUs = + msToUs(clock.elapsedRealtime()) - lastReleaseRealtimeUs; + return started + && frameTimingEvaluator.shouldForceReleaseFrame(earlyUs, elapsedTimeSinceLastReleaseUs); + default: + throw new IllegalStateException(); + } + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java index ce43566b22..0f441fbcb6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java @@ -55,6 +55,9 @@ import java.util.concurrent.Executor; /** Called when the sink renderers the first frame. */ void onFirstFrameRendered(VideoSink videoSink); + /** Called when the sink dropped a frame. */ + void onFrameDropped(VideoSink videoSink); + /** Called when the output video size changed. */ void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize); @@ -62,49 +65,6 @@ import java.util.concurrent.Executor; void onError(VideoSink videoSink, VideoSinkException videoSinkException); } - /** Controls the rendering of video frames. */ - interface RenderControl { - /** Signals a frame must be rendered immediately. */ - long RENDER_TIME_IMMEDIATELY = -1; - - /** Signals a frame must be dropped. */ - long RENDER_TIME_DROP = -2; - - /** Signals that a frame should not be rendered yet. */ - long RENDER_TIME_TRY_AGAIN_LATER = -3; - - /** - * Returns the render timestamp, in nanoseconds, associated with this video frames or one of the - * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not - * rendered yet. - * - * @param presentationTimeUs The presentation time of the video frame, in microseconds. - * @param positionUs The current playback position, in microseconds. - * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, - * taken approximately at the time the playback position was {@code positionUs}. - * @param playbackSpeed The current playback speed. - * @return The render timestamp, in nanoseconds, associated with this frame, or one of the - * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not - * rendered yet. - */ - long getFrameRenderTimeNs( - long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed); - - /** - * Informs the rendering control that a video frame will be rendered. Call this method before - * rendering a frame. - * - * @param presentationTimeUs The frame's presentation time, in microseconds. - */ - void onNextFrame(long presentationTimeUs); - - /** Informs the rendering control that a video frame was rendered. */ - void onFrameRendered(); - - /** Informs the rendering control that a video frame was dropped. */ - void onFrameDropped(); - } - /** * Specifies how the input frames are made available to the video sink. One of {@link * #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java index 871425451c..9627d2030f 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java @@ -132,11 +132,13 @@ public final class CompositingVideoSinkProviderTest { } private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() { - VideoSink.RenderControl renderControl = new TestRenderControl(); + VideoFrameReleaseControl releaseControl = + new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0); return new CompositingVideoSinkProvider( ApplicationProvider.getApplicationContext(), new TestPreviewingVideoGraphFactory(), - renderControl); + releaseControl); } private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory { @@ -161,22 +163,4 @@ public final class CompositingVideoSinkProviderTest { return previewingVideoGraph; } } - - private static class TestRenderControl implements VideoSink.RenderControl { - - @Override - public long getFrameRenderTimeNs( - long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) { - return presentationTimeUs; - } - - @Override - public void onNextFrame(long presentationTimeUs) {} - - @Override - public void onFrameRendered() {} - - @Override - public void onFrameDropped() {} - } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java new file mode 100644 index 0000000000..cbfa2d7fda --- /dev/null +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java @@ -0,0 +1,505 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.video; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.media3.test.utils.FakeClock; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link VideoFrameReleaseControl}. */ +@RunWith(AndroidJUnit4.class) +public class VideoFrameReleaseControlTest { + @Test + public void isReady_onNewInstance_returnsFalse() { + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isFalse(); + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse(); + } + + @Test + public void isReady_afterReleasingFrame_returnsTrue() { + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + + assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isTrue(); + } + + @Test + public void isReady_withinJoiningDeadline_returnsTrue() { + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100); + videoFrameReleaseControl.setClock(clock); + + videoFrameReleaseControl.join(); + + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue(); + } + + @Test + public void isReady_joiningDeadlineExceeded_returnsFalse() { + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100); + videoFrameReleaseControl.setClock(clock); + + videoFrameReleaseControl.join(); + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue(); + + clock.advanceTime(/* timeDiffMs= */ 101); + + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse(); + } + + @Test + public void onFrameReleasedIsFirstFrame_resetsAfterOnEnabled() { + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + + assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); + } + + @Test + public void onFrameReleasedIsFirstFrame_resetsAfterOnProcessedStreamChange() { + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + + assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); + videoFrameReleaseControl.onProcessedStreamChange(); + + assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); + } + + @Test + public void onFrameReleasedIsFirstFrame_resetsAfterSetOutputSurface() { + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + + assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); + videoFrameReleaseControl.setOutputSurface(/* outputSurface= */ null); + + assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); + } + + @Test + public void isReady_afterReset_returnsFalse() { + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isTrue(); + videoFrameReleaseControl.reset(); + + assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isFalse(); + } + + @Test + public void getFrameReleaseAction_firstFrameAllowedBeforeStart_returnsReleaseImmediately() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + } + + @Test + public void getFrameReleaseAction_firstFrameNotAllowedBeforeStart_returnsTryAgainLater() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ false); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); + } + + @Test + public void + getFrameReleaseAction_firstFrameNotAllowedBeforeStartAndStarted_returnsReleaseImmediately() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ false); + + videoFrameReleaseControl.onStarted(); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + } + + @Test + public void getFrameReleaseAction_secondFrameAndNotStarted_returnsTryAgainLater() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + // First frame released. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + + // Second frame + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 10_000, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); + } + + @Test + public void getFrameReleaseAction_secondFrameAndStarted_returnsScheduled() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameReleaseControl.onStarted(); + + // First frame released. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + + // Second frame + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 10_000, + /* positionUs= */ 1, + /* elapsedRealtimeUs= */ 1, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED); + } + + @Test + public void getFrameReleaseAction_secondFrameEarly_returnsTryAgainLater() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + videoFrameReleaseControl.onStarted(); + + // First frame released. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + clock.advanceTime(/* timeDiffMs= */ 10); + + // Second frame is 51 ms too soon. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 61_000, + /* positionUs= */ 10_000, + /* elapsedRealtimeUs= */ 10_000, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); + assertThat(frameReleaseInfo.getEarlyUs()).isEqualTo(51_000); + } + + @Test + public void getFrameReleaseAction_frameLate_returnsDrop() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + videoFrameReleaseControl.onStarted(); + + // First frame released. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + clock.advanceTime(/* timeDiffMs= */ 40); + + // Second frame is 31 ms late. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 9_000, + /* positionUs= */ 40_000, + /* elapsedRealtimeUs= */ 40_000, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP); + } + + @Test + public void getFrameReleaseAction_frameLateWhileJoining_returnsSkip() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 1234); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + videoFrameReleaseControl.onStarted(); + + // First frame released. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + clock.advanceTime(/* timeDiffMs= */ 40); + + // Start joining. + videoFrameReleaseControl.join(); + + // Second frame is 31 ms late. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 9_000, + /* positionUs= */ 40_000, + /* elapsedRealtimeUs= */ 40_000, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP); + } + + @Test + public void getFrameReleaseAction_frameVeryLate_returnsDropToKeyframe() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.setFrameTimingEvaluator( + new TestFrameTimingEvaluator(/* shouldForceRelease= */ false)); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + videoFrameReleaseControl.onStarted(); + + // First frame released. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + clock.advanceTime(/* timeDiffMs= */ 1_000); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 1_000, + /* positionUs= */ 1_000_000, + /* elapsedRealtimeUs= */ 1_000_000, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME); + } + + @Test + public void getFrameReleaseAction_frameVeryLateAndJoining_returnsSkipToFrame() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 1234); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + videoFrameReleaseControl.onStarted(); + + // First frame released. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + clock.advanceTime(/* timeDiffMs= */ 1_000); + videoFrameReleaseControl.join(); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 1_000, + /* positionUs= */ 1_000_000, + /* elapsedRealtimeUs= */ 1_000_000, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME); + } + + @Test + public void + getFrameReleaseAction_frameVeryLateAndFrameTimeEvaluatorForcesFrameRelease_returnsReleaseImmediately() { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0); + videoFrameReleaseControl.setFrameTimingEvaluator( + /* frameTimingEvaluator= */ new TestFrameTimingEvaluator(/* shouldForceRelease= */ true)); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + + videoFrameReleaseControl.onStarted(); + + // First frame released. + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); + clock.advanceTime(/* timeDiffMs= */ 1_000); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 1_000, + /* positionUs= */ 1_000_000, + /* elapsedRealtimeUs= */ 1_000_000, + /* outputStreamStartPositionUs= */ 0, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + } + + private static VideoFrameReleaseControl createVideoFrameReleaseControl() { + return createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 0); + } + + private static VideoFrameReleaseControl createVideoFrameReleaseControl( + long allowedJoiningTimeMs) { + return new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), allowedJoiningTimeMs); + } + + private static class TestFrameTimingEvaluator + implements VideoFrameReleaseControl.FrameTimingEvaluator { + + private final boolean shouldForceRelease; + + public TestFrameTimingEvaluator(boolean shouldForceRelease) { + this.shouldForceRelease = shouldForceRelease; + } + + @Override + public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { + return shouldForceRelease; + } + + @Override + public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { + return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFrame( + earlyUs, elapsedRealtimeUs, isLastFrame); + } + + @Override + public boolean shouldDropFramesToKeyframe( + long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { + return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFramesToKeyframe( + earlyUs, elapsedRealtimeUs, isLastFrame); + } + } +}