diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java index 5404a20545..101dc6f258 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java @@ -92,7 +92,6 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { this.index = index; this.playerId = playerId; this.clock = clock; - onInit(); } @Override @@ -264,11 +263,6 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { // Methods to be overridden by subclasses. - /** Called when the renderer is initialized. */ - protected void onInit() { - // Do nothing - } - /** * Called when the renderer is enabled. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 9489f3ee94..e5bb66e70c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -40,7 +40,6 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.VideoSize; -import androidx.media3.common.util.Clock; import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.Size; import androidx.media3.common.util.TimedValueQueue; @@ -63,7 +62,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Context context; private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; - private final VideoFrameReleaseControl videoFrameReleaseControl; + private final VideoSink.RenderControl renderControl; @Nullable private VideoSinkImpl videoSinkImpl; @Nullable private List videoEffects; @@ -74,7 +73,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public CompositingVideoSinkProvider( Context context, VideoFrameProcessor.Factory videoFrameProcessorFactory, - VideoFrameReleaseControl renderControl) { + VideoSink.RenderControl renderControl) { this( context, new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory), @@ -85,10 +84,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* package */ CompositingVideoSinkProvider( Context context, PreviewingVideoGraph.Factory previewingVideoGraphFactory, - VideoFrameReleaseControl releaseControl) { + VideoSink.RenderControl renderControl) { this.context = context; this.previewingVideoGraphFactory = previewingVideoGraphFactory; - this.videoFrameReleaseControl = releaseControl; + this.renderControl = renderControl; } @Override @@ -98,8 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; try { videoSinkImpl = - new VideoSinkImpl( - context, previewingVideoGraphFactory, videoFrameReleaseControl, sourceFormat); + new VideoSinkImpl(context, previewingVideoGraphFactory, renderControl, sourceFormat); } catch (VideoFrameProcessingException e) { throw new VideoSink.VideoSinkException(e, sourceFormat); } @@ -175,8 +173,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listener { private final Context context; - private final VideoFrameReleaseControl videoFrameReleaseControl; - private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo; + private final VideoSink.RenderControl renderControl; private final VideoFrameProcessor videoFrameProcessor; private final LongArrayQueue processedFramesBufferTimestampsUs; private final TimedValueQueue streamOffsets; @@ -210,9 +207,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private VideoSize processedFrameSize; private VideoSize reportedVideoSize; private boolean pendingVideoSizeChange; + private boolean renderedFirstFrame; private long inputStreamOffsetUs; private boolean pendingInputStreamOffsetChange; private long outputStreamOffsetUs; + private float playbackSpeed; // TODO b/292111083 - Remove the field and trigger the callback on every video size change. private boolean onVideoSizeChangedCalled; @@ -221,12 +220,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public VideoSinkImpl( Context context, PreviewingVideoGraph.Factory previewingVideoGraphFactory, - VideoFrameReleaseControl videoFrameReleaseControl, + RenderControl renderControl, Format sourceFormat) throws VideoFrameProcessingException { this.context = context; - this.videoFrameReleaseControl = videoFrameReleaseControl; - videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); + this.renderControl = renderControl; processedFramesBufferTimestampsUs = new LongArrayQueue(); streamOffsets = new TimedValueQueue<>(); videoSizeChanges = new TimedValueQueue<>(); @@ -239,6 +237,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; lastCodecBufferPresentationTimestampUs = C.TIME_UNSET; processedFrameSize = VideoSize.UNKNOWN; reportedVideoSize = VideoSize.UNKNOWN; + playbackSpeed = 1f; // Playback thread handler. handler = Util.createHandlerForCurrentLooper(); @@ -294,7 +293,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; processedFramesBufferTimestampsUs.clear(); streamOffsets.clear(); handler.removeCallbacksAndMessages(/* token= */ null); - videoFrameReleaseControl.reset(); + renderedFirstFrame = false; if (registeredLastFrame) { registeredLastFrame = false; processedLastFrame = false; @@ -304,7 +303,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public boolean isReady() { - return videoFrameReleaseControl.isReady(/* rendererReady= */ true); + return renderedFirstFrame; } @Override @@ -386,50 +385,47 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long bufferPresentationTimeUs = processedFramesBufferTimestampsUs.element(); // check whether this buffer comes with a new stream offset. if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) { - videoFrameReleaseControl.onProcessedStreamChange(); + renderedFirstFrame = false; } long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1; - @VideoFrameReleaseControl.FrameReleaseAction - int frameReleaseAction = - videoFrameReleaseControl.getFrameReleaseAction( - bufferPresentationTimeUs, - positionUs, - elapsedRealtimeUs, - outputStreamOffsetUs, - isLastFrame, - videoFrameReleaseInfo); - switch (frameReleaseAction) { - case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER: - return; - case VideoFrameReleaseControl.FRAME_RELEASE_SKIP: - case VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME: - // Skipped frames are dropped. - case VideoFrameReleaseControl.FRAME_RELEASE_DROP: - case VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME: - // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush - // VideoFrameProcessor input frames in this case. - dropFrame(isLastFrame); - break; - case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: - case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED: - renderFrame( - framePresentationTimeUs, bufferPresentationTimeUs, frameReleaseAction, isLastFrame); - break; - default: - throw new IllegalStateException(String.valueOf(frameReleaseAction)); + long frameRenderTimeNs = + renderControl.getFrameRenderTimeNs( + bufferPresentationTimeUs, positionUs, elapsedRealtimeUs, playbackSpeed); + if (frameRenderTimeNs == RenderControl.RENDER_TIME_TRY_AGAIN_LATER) { + return; + } else if (framePresentationTimeUs == RenderControl.RENDER_TIME_DROP) { + // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush + // VideoFrameProcessor input frames in this case. + releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); + continue; } + renderControl.onNextFrame(bufferPresentationTimeUs); + if (videoFrameMetadataListener != null) { + videoFrameMetadataListener.onVideoFrameAboutToBeRendered( + framePresentationTimeUs, + frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY + ? System.nanoTime() + : frameRenderTimeNs, + checkNotNull(inputFormat), + /* mediaFormat= */ null); + } + releaseProcessedFrameInternal( + frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY + ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY + : frameRenderTimeNs, + isLastFrame); + + maybeNotifyVideoSizeChanged(bufferPresentationTimeUs); } } @Override public void setPlaybackSpeed(float speed) { checkArgument(speed >= 0.0); - videoFrameReleaseControl.setPlaybackSpeed(speed); + this.playbackSpeed = speed; } - // VideoGraph.Listener methods - @Override public void onOutputSizeChanged(int width, int height) { VideoSize newVideoSize = new VideoSize(width, height); @@ -481,13 +477,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new IllegalStateException(); } - // Other methods - public void release() { videoFrameProcessor.release(); handler.removeCallbacksAndMessages(/* token= */ null); streamOffsets.clear(); processedFramesBufferTimestampsUs.clear(); + renderedFirstFrame = false; } /** Sets the {@linkplain Effect video effects} to apply immediately. */ @@ -546,17 +541,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; && currentSurfaceAndSize.second.equals(outputResolution)) { return; } - videoFrameReleaseControl.setOutputSurface(outputSurface); + renderedFirstFrame = + currentSurfaceAndSize == null || currentSurfaceAndSize.first.equals(outputSurface); currentSurfaceAndSize = Pair.create(outputSurface, outputResolution); videoFrameProcessor.setOutputSurfaceInfo( new SurfaceInfo( outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); } - /** Clears the output surface info. */ public void clearOutputSurfaceInfo() { videoFrameProcessor.setOutputSurfaceInfo(null); currentSurfaceAndSize = null; + renderedFirstFrame = false; } private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) { @@ -569,52 +565,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return updatedOffset; } - private void dropFrame(boolean isLastFrame) { - if (listenerExecutor != null) { - listenerExecutor.execute( - () -> { - if (listener != null) { - listener.onFrameDropped(this); - } - }); - } - releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); - } - - private void renderFrame( - long framePresentationTimeUs, - long bufferPresentationTimeUs, - @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction, - boolean isLastFrame) { - if (videoFrameMetadataListener != null) { - videoFrameMetadataListener.onVideoFrameAboutToBeRendered( - framePresentationTimeUs, - frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY - ? Clock.DEFAULT.nanoTime() - : videoFrameReleaseInfo.getReleaseTimeNs(), - checkNotNull(inputFormat), - /* mediaFormat= */ null); - } - if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && listenerExecutor != null) { - listenerExecutor.execute( - () -> { - if (listener != null) { - listener.onFirstFrameRendered(/* videoSink= */ this); - } - }); - } - releaseProcessedFrameInternal( - frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY - ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY - : videoFrameReleaseInfo.getReleaseTimeNs(), - isLastFrame); - - maybeNotifyVideoSizeChanged(bufferPresentationTimeUs); - } - private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) { videoFrameProcessor.renderOutputFrame(releaseTimeNs); processedFramesBufferTimestampsUs.remove(); + if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME) { + renderControl.onFrameDropped(); + } else { + renderControl.onFrameRendered(); + if (!renderedFirstFrame) { + if (listener != null) { + checkNotNull(listenerExecutor) + .execute(() -> checkNotNull(listener).onFirstFrameRendered(this)); + } + renderedFirstFrame = true; + } + } if (isLastFrame) { releasedLastFrame = true; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 75f42f91fc..323e1198b8 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -18,7 +18,7 @@ package androidx.media3.exoplayer.video; import static android.view.Display.DEFAULT_DISPLAY; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.msToUs; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_VIDEO_MAX_RESOLUTION_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO; @@ -38,6 +38,7 @@ import android.media.MediaFormat; import android.os.Bundle; import android.os.Handler; import android.os.Message; +import android.os.SystemClock; import android.util.Pair; import android.view.Display; import android.view.Surface; @@ -56,6 +57,7 @@ import androidx.media3.common.PlaybackException; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; +import androidx.media3.common.util.Clock; import androidx.media3.common.util.Log; import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.Size; @@ -86,8 +88,8 @@ import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.Executor; +import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * Decodes and renders video using {@link MediaCodec}. @@ -113,8 +115,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * */ @UnstableApi -public class MediaCodecVideoRenderer extends MediaCodecRenderer - implements VideoFrameReleaseControl.FrameTimingEvaluator { +public class MediaCodecVideoRenderer extends MediaCodecRenderer implements VideoSink.RenderControl { private static final String TAG = "MediaCodecVideoRenderer"; private static final String KEY_CROP_LEFT = "crop-left"; @@ -138,16 +139,19 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer /** The minimum input buffer size for HEVC. */ private static final int HEVC_MAX_INPUT_SIZE_THRESHOLD = 2 * 1024 * 1024; + /** The maximum earliest time, in microseconds, to release a frame on the surface. */ + private static final long MAX_EARLY_US_THRESHOLD = 50_000; + private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround; private static boolean deviceNeedsSetOutputSurfaceWorkaround; private final Context context; + private final VideoFrameReleaseHelper frameReleaseHelper; private final VideoSinkProvider videoSinkProvider; private final EventDispatcher eventDispatcher; + private final long allowedJoiningTimeMs; private final int maxDroppedFramesToNotify; private final boolean deviceNeedsNoPostProcessWorkaround; - private final VideoFrameReleaseControl videoFrameReleaseControl; - private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo; private @MonotonicNonNull CodecMaxValues codecMaxValues; private boolean codecNeedsSetOutputSurfaceWorkaround; @@ -156,10 +160,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @Nullable private PlaceholderSurface placeholderSurface; private boolean haveReportedFirstFrameRenderedForCurrentSurface; private @C.VideoScalingMode int scalingMode; + private @C.FirstFrameState int firstFrameState; + private long initialPositionUs; + private long joiningDeadlineMs; private long droppedFrameAccumulationStartTimeMs; private int droppedFrames; private int consecutiveDroppedFrameCount; private int buffersInCodecCount; + private long lastBufferPresentationTimeUs; + private long lastRenderRealtimeUs; private long totalVideoFrameProcessingOffsetUs; private int videoFrameProcessingOffsetCount; private long lastFrameReleaseTimeNs; @@ -384,42 +393,23 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer mediaCodecSelector, enableDecoderFallback, assumedMinimumCodecOperatingRate); + this.allowedJoiningTimeMs = allowedJoiningTimeMs; this.maxDroppedFramesToNotify = maxDroppedFramesToNotify; this.context = context.getApplicationContext(); - - videoFrameReleaseControl = new VideoFrameReleaseControl(this.context, allowedJoiningTimeMs); - videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); + frameReleaseHelper = new VideoFrameReleaseHelper(this.context); eventDispatcher = new EventDispatcher(eventHandler, eventListener); + @SuppressWarnings("nullness:assignment") + VideoSink.@Initialized RenderControl renderControl = this; videoSinkProvider = - new CompositingVideoSinkProvider( - context, videoFrameProcessorFactory, videoFrameReleaseControl); + new CompositingVideoSinkProvider(context, videoFrameProcessorFactory, renderControl); deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround(); + joiningDeadlineMs = C.TIME_UNSET; scalingMode = C.VIDEO_SCALING_MODE_DEFAULT; decodedVideoSize = VideoSize.UNKNOWN; tunnelingAudioSessionId = C.AUDIO_SESSION_ID_UNSET; - reportedVideoSize = null; + firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; } - // FrameTimingEvaluator methods - - @Override - public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { - return shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastReleaseUs); - } - - @Override - public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { - return shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastFrame); - } - - @Override - public boolean shouldDropFramesToKeyframe( - long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { - return shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastFrame); - } - - // Renderer methods - @Override public String getName() { return TAG; @@ -533,6 +523,53 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer format); } + // RenderControl implementation + + @Override + public long getFrameRenderTimeNs( + long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) { + long earlyUs = + calculateEarlyTimeUs( + positionUs, + elapsedRealtimeUs, + presentationTimeUs, + getState() == STATE_STARTED, + playbackSpeed, + getClock()); + if (isBufferLate(earlyUs)) { + return VideoSink.RenderControl.RENDER_TIME_DROP; + } + if (shouldForceRender(positionUs, earlyUs)) { + return VideoSink.RenderControl.RENDER_TIME_IMMEDIATELY; + } + + if (getState() != STATE_STARTED + || positionUs == initialPositionUs + || earlyUs > MAX_EARLY_US_THRESHOLD) { + return VideoSink.RenderControl.RENDER_TIME_TRY_AGAIN_LATER; + } + // Compute the buffer's desired release time in nanoseconds. + long unadjustedFrameReleaseTimeNs = getClock().nanoTime() + (earlyUs * 1000); + // Apply a timestamp adjustment, if there is one. + return frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); + } + + @Override + public void onNextFrame(long presentationTimeUs) { + frameReleaseHelper.onNextFrame(presentationTimeUs); + } + + @Override + public void onFrameRendered() { + lastRenderRealtimeUs = Util.msToUs(getClock().elapsedRealtime()); + } + + @Override + public void onFrameDropped() { + updateDroppedBufferCounters( + /* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1); + } + // Other methods /** @@ -597,13 +634,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } } - @Override - protected void onInit() { - super.onInit(); - videoFrameReleaseControl.setFrameTimingEvaluator(/* frameTimingEvaluator= */ this); - videoFrameReleaseControl.setClock(getClock()); - } - @Override protected void onEnabled(boolean joining, boolean mayRenderStartOfStream) throws ExoPlaybackException { @@ -615,12 +645,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer releaseCodec(); } eventDispatcher.enabled(decoderCounters); - videoFrameReleaseControl.onEnabled(mayRenderStartOfStream); + firstFrameState = + mayRenderStartOfStream + ? C.FIRST_FRAME_NOT_RENDERED + : C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; } @Override public void enableMayRenderStartOfStream() { - videoFrameReleaseControl.allowReleaseFirstFrameBeforeStarted(); + if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) { + firstFrameState = C.FIRST_FRAME_NOT_RENDERED; + } } @Override @@ -631,15 +666,21 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer videoSink.flush(); } super.onPositionReset(positionUs, joining); + if (videoSinkProvider.isInitialized()) { videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); } - videoFrameReleaseControl.reset(); - if (joining) { - videoFrameReleaseControl.join(); - } - maybeUpdateOnFrameRenderedListener(); + + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); + frameReleaseHelper.onPositionReset(); + lastBufferPresentationTimeUs = C.TIME_UNSET; + initialPositionUs = C.TIME_UNSET; consecutiveDroppedFrameCount = 0; + if (joining) { + setJoiningDeadlineMs(); + } else { + joiningDeadlineMs = C.TIME_UNSET; + } } @Override @@ -649,15 +690,26 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @Override public boolean isReady() { - boolean readyToReleaseFrames = super.isReady() && (videoSink == null || videoSink.isReady()); - if (readyToReleaseFrames - && ((placeholderSurface != null && displaySurface == placeholderSurface) + if (super.isReady() + && (videoSink == null || videoSink.isReady()) + && (firstFrameState == C.FIRST_FRAME_RENDERED + || (placeholderSurface != null && displaySurface == placeholderSurface) || getCodec() == null || tunneling)) { - // Not releasing frames. + // Ready. If we were joining then we've now joined, so clear the joining deadline. + joiningDeadlineMs = C.TIME_UNSET; return true; + } else if (joiningDeadlineMs == C.TIME_UNSET) { + // Not joining. + return false; + } else if (getClock().elapsedRealtime() < joiningDeadlineMs) { + // Joining and still within the joining deadline. + return true; + } else { + // The joining deadline has been exceeded. Give up and clear the deadline. + joiningDeadlineMs = C.TIME_UNSET; + return false; } - return videoFrameReleaseControl.isReady(readyToReleaseFrames); } @Override @@ -666,24 +718,25 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer droppedFrames = 0; long elapsedRealtimeMs = getClock().elapsedRealtime(); droppedFrameAccumulationStartTimeMs = elapsedRealtimeMs; + lastRenderRealtimeUs = msToUs(elapsedRealtimeMs); totalVideoFrameProcessingOffsetUs = 0; videoFrameProcessingOffsetCount = 0; - videoFrameReleaseControl.onStarted(); + frameReleaseHelper.onStarted(); } @Override protected void onStopped() { + joiningDeadlineMs = C.TIME_UNSET; maybeNotifyDroppedFrames(); maybeNotifyVideoFrameProcessingOffset(); - videoFrameReleaseControl.onStopped(); + frameReleaseHelper.onStopped(); super.onStopped(); } @Override protected void onDisabled() { reportedVideoSize = null; - videoFrameReleaseControl.onDisabled(); - maybeUpdateOnFrameRenderedListener(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED); haveReportedFirstFrameRenderedForCurrentSurface = false; tunnelingOnFrameRenderedListener = null; try { @@ -730,7 +783,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } break; case MSG_SET_CHANGE_FRAME_RATE_STRATEGY: - videoFrameReleaseControl.setChangeFrameRateStrategy((int) checkNotNull(message)); + frameReleaseHelper.setChangeFrameRateStrategy((int) checkNotNull(message)); break; case MSG_SET_VIDEO_FRAME_METADATA_LISTENER: frameMetadataListener = (VideoFrameMetadataListener) checkNotNull(message); @@ -790,7 +843,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer // We only need to update the codec if the display surface has changed. if (this.displaySurface != displaySurface) { this.displaySurface = displaySurface; - videoFrameReleaseControl.setOutputSurface(displaySurface); + frameReleaseHelper.onSurfaceChanged(displaySurface); haveReportedFirstFrameRenderedForCurrentSurface = false; @State int state = getState(); @@ -809,8 +862,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer if (displaySurface != null && displaySurface != placeholderSurface) { // If we know the video size, report it again immediately. maybeRenotifyVideoSizeChanged(); + // We haven't rendered to the new display surface yet. + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); if (state == STATE_STARTED) { - videoFrameReleaseControl.join(); + // Set joining deadline to report MediaCodecVideoRenderer is ready. + setJoiningDeadlineMs(); } // When effects previewing is enabled, set display surface and an unknown size. if (videoSinkProvider.isInitialized()) { @@ -819,11 +875,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } else { // The display surface has been removed. reportedVideoSize = null; + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); if (videoSinkProvider.isInitialized()) { videoSinkProvider.clearOutputSurfaceInfo(); } } - maybeUpdateOnFrameRenderedListener(); } else if (displaySurface != null && displaySurface != placeholderSurface) { // The display surface is set and unchanged. If we know the video size and/or have already // rendered to the display surface, report these again immediately. @@ -931,7 +987,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer public void setPlaybackSpeed(float currentPlaybackSpeed, float targetPlaybackSpeed) throws ExoPlaybackException { super.setPlaybackSpeed(currentPlaybackSpeed, targetPlaybackSpeed); - videoFrameReleaseControl.setPlaybackSpeed(currentPlaybackSpeed); + frameReleaseHelper.onPlaybackSpeed(currentPlaybackSpeed); if (videoSink != null) { videoSink.setPlaybackSpeed(currentPlaybackSpeed); } @@ -1048,14 +1104,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer new VideoSink.Listener() { @Override public void onFirstFrameRendered(VideoSink videoSink) { - checkStateNotNull(displaySurface); - notifyRenderedFirstFrame(); - } - - @Override - public void onFrameDropped(VideoSink videoSink) { - updateDroppedBufferCounters( - /* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1); + maybeNotifyRenderedFirstFrame(); } @Override @@ -1196,7 +1245,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } decodedVideoSize = new VideoSize(width, height, unappliedRotationDegrees, pixelWidthHeightRatio); - videoFrameReleaseControl.setFrameRate(format.frameRate); + frameReleaseHelper.onFormatChanged(format.frameRate); if (videoSink != null && mediaFormat != null) { onReadyToRegisterVideoSinkInputStream(); @@ -1270,34 +1319,40 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer throws ExoPlaybackException { checkNotNull(codec); // Can not render video without codec + if (initialPositionUs == C.TIME_UNSET) { + initialPositionUs = positionUs; + } + + if (bufferPresentationTimeUs != lastBufferPresentationTimeUs) { + if (videoSink == null) { + frameReleaseHelper.onNextFrame(bufferPresentationTimeUs); + } // else, update the frameReleaseHelper when releasing the processed frames. + this.lastBufferPresentationTimeUs = bufferPresentationTimeUs; + } + long outputStreamOffsetUs = getOutputStreamOffsetUs(); long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; - @VideoFrameReleaseControl.FrameReleaseAction - int frameReleaseAction = - videoFrameReleaseControl.getFrameReleaseAction( - bufferPresentationTimeUs, - positionUs, - elapsedRealtimeUs, - getOutputStreamStartPositionUs(), - isLastBuffer, - videoFrameReleaseInfo); - - // Skip decode-only buffers, e.g. after seeking, immediately. This check must be performed after - // getting the release action from the video frame release control although not necessary. - // That's because the release control estimates the content frame rate from frame timestamps - // and we want to have this information known as early as possible, especially during seeking. if (isDecodeOnlyBuffer && !isLastBuffer) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } - // We are not rendering on a surface, the renderer will wait until a surface is set. + boolean isStarted = getState() == STATE_STARTED; + long earlyUs = + calculateEarlyTimeUs( + positionUs, + elapsedRealtimeUs, + bufferPresentationTimeUs, + isStarted, + getPlaybackSpeed(), + getClock()); + if (displaySurface == placeholderSurface) { // Skip frames in sync with playback, so we'll be at the right frame if the mode changes. - if (videoFrameReleaseInfo.getEarlyUs() < 30_000) { + if (isBufferLate(earlyUs)) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); + updateVideoFrameProcessingOffsetCounters(earlyUs); return true; } return false; @@ -1313,86 +1368,137 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer return true; } - switch (frameReleaseAction) { - case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: - long releaseTimeNs = getClock().nanoTime(); - notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); - renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); - updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); - return true; - case VideoFrameReleaseControl.FRAME_RELEASE_SKIP: + boolean forceRenderOutputBuffer = shouldForceRender(positionUs, earlyUs); + if (forceRenderOutputBuffer) { + long releaseTimeNs = getClock().nanoTime(); + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); + renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); + updateVideoFrameProcessingOffsetCounters(earlyUs); + return true; + } + + if (!isStarted || positionUs == initialPositionUs) { + return false; + } + + // Compute the buffer's desired release time in nanoseconds. + long systemTimeNs = getClock().nanoTime(); + long unadjustedFrameReleaseTimeNs = systemTimeNs + (earlyUs * 1000); + // Apply a timestamp adjustment, if there is one. + long adjustedReleaseTimeNs = frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); + earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000; + boolean treatDroppedBuffersAsSkipped = joiningDeadlineMs != C.TIME_UNSET; + if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastBuffer) + && maybeDropBuffersToKeyframe(positionUs, treatDroppedBuffersAsSkipped)) { + return false; + } else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastBuffer)) { + if (treatDroppedBuffersAsSkipped) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); - return true; - case VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME: - if (!maybeDropBuffersToKeyframe(positionUs, /* treatDroppedBuffersAsSkipped= */ true)) { - skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); - return true; - } else { - return false; - } - case VideoFrameReleaseControl.FRAME_RELEASE_DROP: + } else { dropOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); - return true; - case VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME: - if (!maybeDropBuffersToKeyframe(positionUs, /* treatDroppedBuffersAsSkipped= */ false)) { - dropOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); - return true; + } + updateVideoFrameProcessingOffsetCounters(earlyUs); + return true; + } + + if (Util.SDK_INT >= 21) { + // Let the underlying framework time the release. + if (earlyUs < MAX_EARLY_US_THRESHOLD) { + if (shouldSkipBuffersWithIdenticalReleaseTime() + && adjustedReleaseTimeNs == lastFrameReleaseTimeNs) { + // This frame should be displayed on the same vsync with the previous released frame. We + // are likely rendering frames at a rate higher than the screen refresh rate. Skip + // this buffer so that it's returned to MediaCodec sooner otherwise MediaCodec may not + // be able to keep decoding with this rate [b/263454203]. + skipOutputBuffer(codec, bufferIndex, presentationTimeUs); } else { - return false; + notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format); + renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs); } - case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER: - return false; - case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED: - return maybeReleaseFrame(checkStateNotNull(codec), bufferIndex, presentationTimeUs, format); + updateVideoFrameProcessingOffsetCounters(earlyUs); + lastFrameReleaseTimeNs = adjustedReleaseTimeNs; + return true; + } + } else { + // We need to time the release ourselves. + if (earlyUs < 30000) { + if (earlyUs > 11000) { + // We're a little too early to render the frame. Sleep until the frame can be rendered. + // Note: The 11ms threshold was chosen fairly arbitrarily. + try { + // Subtracting 10000 rather than 11000 ensures the sleep time will be at least 1ms. + Thread.sleep((earlyUs - 10000) / 1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format); + renderOutputBuffer(codec, bufferIndex, presentationTimeUs); + updateVideoFrameProcessingOffsetCounters(earlyUs); + return true; + } + } + + // We're either not playing, or it's not time to render the frame yet. + return false; + } + + /** Returns whether a buffer or a processed frame should be force rendered. */ + private boolean shouldForceRender(long positionUs, long earlyUs) { + if (joiningDeadlineMs != C.TIME_UNSET) { + // No force rendering during joining. + return false; + } + boolean isStarted = getState() == STATE_STARTED; + switch (firstFrameState) { + case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED: + return isStarted; + case C.FIRST_FRAME_NOT_RENDERED: + return true; + case C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE: + return positionUs >= getOutputStreamStartPositionUs(); + case C.FIRST_FRAME_RENDERED: + long elapsedSinceLastRenderUs = msToUs(getClock().elapsedRealtime()) - lastRenderRealtimeUs; + return isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs); default: - throw new IllegalStateException(String.valueOf(frameReleaseAction)); + throw new IllegalStateException(); } } - private boolean maybeReleaseFrame( - MediaCodecAdapter codec, int bufferIndex, long presentationTimeUs, Format format) { - long releaseTimeNs = videoFrameReleaseInfo.getReleaseTimeNs(); - long earlyUs = videoFrameReleaseInfo.getEarlyUs(); - if (Util.SDK_INT >= 21) { - // Let the underlying framework time the release. - if (shouldSkipBuffersWithIdenticalReleaseTime() && releaseTimeNs == lastFrameReleaseTimeNs) { - // This frame should be displayed on the same vsync with the previous released frame. We - // are likely rendering frames at a rate higher than the screen refresh rate. Skip - // this buffer so that it's returned to MediaCodec sooner otherwise MediaCodec may not - // be able to keep decoding with this rate [b/263454203]. - skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - } else { - notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); - renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs); - } - updateVideoFrameProcessingOffsetCounters(earlyUs); - lastFrameReleaseTimeNs = releaseTimeNs; - return true; - } else if (earlyUs < 30000) { - // We need to time the release ourselves. - if (earlyUs > 11000) { - // We're a little too early to render the frame. Sleep until the frame can be rendered. - // Note: The 11ms threshold was chosen fairly arbitrarily. - try { - // Subtracting 10000 rather than 11000 ensures the sleep time will be at least 1ms. - Thread.sleep((earlyUs - 10000) / 1000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - } - notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); - renderOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(earlyUs); - return true; - } else { - // Too soon. - return false; + /** + * Calculates the time interval between the current player position and the buffer presentation + * time. + * + * @param positionUs The current media time in microseconds, measured at the start of the current + * iteration of the rendering loop. + * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the + * start of the current iteration of the rendering loop. + * @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds, + * with {@linkplain #getOutputStreamOffsetUs() stream offset added}. + * @param isStarted Whether the playback is in {@link #STATE_STARTED}. + * @param playbackSpeed The current playback speed. + * @param clock The {@link Clock} used by the renderer. + * @return The calculated early time, in microseconds. + */ + private static long calculateEarlyTimeUs( + long positionUs, + long elapsedRealtimeUs, + long bufferPresentationTimeUs, + boolean isStarted, + float playbackSpeed, + Clock clock) { + // Calculate how early we are. In other words, the realtime duration that needs to elapse whilst + // the renderer is started before the frame should be rendered. A negative value means that + // we're already late. + // Note: Use of double rather than float is intentional for accuracy in the calculations below. + long earlyUs = (long) ((bufferPresentationTimeUs - positionUs) / (double) playbackSpeed); + if (isStarted) { + // Account for the elapsed time since the start of this iteration of the rendering loop. + earlyUs -= Util.msToUs(clock.elapsedRealtime()) - elapsedRealtimeUs; } + + return earlyUs; } private void notifyFrameMetadataListener( @@ -1429,8 +1535,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @Override protected void onProcessedStreamChange() { super.onProcessedStreamChange(); - videoFrameReleaseControl.onProcessedStreamChange(); - maybeUpdateOnFrameRenderedListener(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE); if (videoSinkProvider.isInitialized()) { videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); } @@ -1447,8 +1552,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer */ protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { - return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFrame( - earlyUs, elapsedRealtimeUs, isLastBuffer); + return isBufferLate(earlyUs) && !isLastBuffer; } /** @@ -1463,8 +1567,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer */ protected boolean shouldDropBuffersToKeyframe( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { - return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFramesToKeyframe( - earlyUs, elapsedRealtimeUs, isLastBuffer); + return isBufferVeryLate(earlyUs) && !isLastBuffer; } /** @@ -1485,8 +1588,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer * @return Returns whether to force rendering an output buffer. */ protected boolean shouldForceRenderOutputBuffer(long earlyUs, long elapsedSinceLastRenderUs) { - return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldForceReleaseFrame( - earlyUs, elapsedSinceLastRenderUs); + // Force render late buffers every 100ms to avoid frozen video effect. + return isBufferLate(earlyUs) && elapsedSinceLastRenderUs > 100000; } /** @@ -1618,6 +1721,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; if (videoSink == null) { + lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime()); maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); } @@ -1641,6 +1745,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; if (videoSink == null) { + lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime()); maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); } @@ -1664,7 +1769,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } } - private void maybeUpdateOnFrameRenderedListener() { + private void setJoiningDeadlineMs() { + joiningDeadlineMs = + allowedJoiningTimeMs > 0 + ? (getClock().elapsedRealtime() + allowedJoiningTimeMs) + : C.TIME_UNSET; + } + + private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) { + this.firstFrameState = min(this.firstFrameState, firstFrameState); // The first frame notification is triggered by renderOutputBuffer or renderOutputBufferV21 for // non-tunneled playback, onQueueInputBuffer for tunneled playback prior to API level 23, and // OnFrameRenderedListenerV23.onFrameRenderedListener for tunneled playback on API level 23 and @@ -1679,17 +1792,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } private void maybeNotifyRenderedFirstFrame() { - if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && displaySurface != null) { - notifyRenderedFirstFrame(); + if (displaySurface != null && firstFrameState != C.FIRST_FRAME_RENDERED) { + firstFrameState = C.FIRST_FRAME_RENDERED; + eventDispatcher.renderedFirstFrame(displaySurface); + haveReportedFirstFrameRenderedForCurrentSurface = true; } } - @RequiresNonNull("displaySurface") - private void notifyRenderedFirstFrame() { - eventDispatcher.renderedFirstFrame(displaySurface); - haveReportedFirstFrameRenderedForCurrentSurface = true; - } - private void maybeRenotifyRenderedFirstFrame() { if (displaySurface != null && haveReportedFirstFrameRenderedForCurrentSurface) { eventDispatcher.renderedFirstFrame(displaySurface); @@ -1729,6 +1838,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } } + private static boolean isBufferLate(long earlyUs) { + // Class a buffer as late if it should have been presented more than 30 ms ago. + return earlyUs < -30000; + } + + private static boolean isBufferVeryLate(long earlyUs) { + // Class a buffer as very late if it should have been presented more than 500 ms ago. + return earlyUs < -500000; + } + @RequiresApi(29) private static void setHdr10PlusInfoV29(MediaCodecAdapter codec, byte[] hdr10PlusInfo) { Bundle codecParameters = new Bundle(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java deleted file mode 100644 index 4e99c405d2..0000000000 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java +++ /dev/null @@ -1,466 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.exoplayer.video; - -import static androidx.media3.common.util.Util.msToUs; -import static java.lang.Math.min; -import static java.lang.annotation.ElementType.TYPE_USE; - -import android.content.Context; -import android.os.SystemClock; -import android.view.Surface; -import androidx.annotation.IntDef; -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.util.Clock; -import androidx.media3.common.util.UnstableApi; -import androidx.media3.common.util.Util; -import androidx.media3.exoplayer.Renderer; -import java.lang.annotation.Documented; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** Controls the releasing of video frames. */ -/* package */ final class VideoFrameReleaseControl { - - /** - * The frame release action returned by {@link #getFrameReleaseAction(long, long, long, long, - * boolean, FrameReleaseInfo)}. - * - *

One of {@link #FRAME_RELEASE_IMMEDIATELY}, {@link #FRAME_RELEASE_SCHEDULED}, {@link - * #FRAME_RELEASE_DROP}, {@link #FRAME_RELEASE_DROP_TO_KEYFRAME}, {@link ##FRAME_RELEASE_SKIP} or - * {@link #FRAME_RELEASE_TRY_AGAIN_LATER}. - */ - @Documented - @Retention(RetentionPolicy.SOURCE) - @Target(TYPE_USE) - @UnstableApi - @IntDef({ - FRAME_RELEASE_IMMEDIATELY, - FRAME_RELEASE_SCHEDULED, - FRAME_RELEASE_DROP, - FRAME_RELEASE_DROP_TO_KEYFRAME, - FRAME_RELEASE_SKIP, - FRAME_RELEASE_SKIP_TO_KEYFRAME, - FRAME_RELEASE_TRY_AGAIN_LATER - }) - public @interface FrameReleaseAction {} - - /** Signals a frame should be released immediately. */ - public static final int FRAME_RELEASE_IMMEDIATELY = 0; - - /** - * Signals a frame should be scheduled for release. The release timestamp will be returned by - * {@link FrameReleaseInfo#getReleaseTimeNs()}. - */ - public static final int FRAME_RELEASE_SCHEDULED = 1; - - /** Signals a frame should be dropped. */ - public static final int FRAME_RELEASE_DROP = 2; - - /** Signals frames up to the next key-frame should be dropped. */ - public static final int FRAME_RELEASE_DROP_TO_KEYFRAME = 3; - - /** Signals that a frame should be skipped. */ - public static final int FRAME_RELEASE_SKIP = 4; - - /** Signals that frames up to the next key-frame should be skipped. */ - public static final int FRAME_RELEASE_SKIP_TO_KEYFRAME = 5; - - /** Signals that a frame should not be released and the renderer should try again later. */ - public static final int FRAME_RELEASE_TRY_AGAIN_LATER = 6; - - /** Per {@link FrameReleaseAction} metadata. */ - public static class FrameReleaseInfo { - private long earlyUs; - private long releaseTimeNs; - - /** Resets this instances state. */ - public FrameReleaseInfo() { - earlyUs = C.TIME_UNSET; - releaseTimeNs = C.TIME_UNSET; - } - - /** - * Returns this frame's early time compared to the playback position, before any release time - * adjustment to the screen vsync slots. - */ - public long getEarlyUs() { - return earlyUs; - } - - /** - * Returns the release time for the frame, in nanoseconds, or {@link C#TIME_UNSET} if the frame - * should not be released yet. - */ - public long getReleaseTimeNs() { - return releaseTimeNs; - } - - private void reset() { - earlyUs = C.TIME_UNSET; - releaseTimeNs = C.TIME_UNSET; - } - } - - /** Decides whether a frame should be forced to be released, or dropped. */ - public interface FrameTimingEvaluator { - /** - * Whether a frame should be forced for release. - * - * @param earlyUs The time until the buffer should be presented in microseconds. A negative - * value indicates that the buffer is late. - * @param elapsedSinceLastReleaseUs The elapsed time since the last frame was released, in - * microseconds. - * @return Whether the video frame should be force released. - */ - boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs); - - /** - * Returns whether the frame should be dropped. - * - * @param earlyUs The time until the buffer should be presented in microseconds. A negative - * value indicates that the buffer is late. - * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, - * measured at the start of the current iteration of the rendering loop. - * @param isLastFrame Whether the buffer is the last buffer in the current stream. - */ - boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame); - - /** - * Returns whether to drop all frames starting from this frame to the keyframe at or after the - * current playback position, if possible. - * - * @param earlyUs The time until the current buffer should be presented in microseconds. A - * negative value indicates that the buffer is late. - * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, - * measured at the start of the current iteration of the rendering loop. - * @param isLastFrame Whether the frame is the last frame in the current stream. - */ - boolean shouldDropFramesToKeyframe(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame); - - /** The default timing evaluator. */ - FrameTimingEvaluator DEFAULT = - new FrameTimingEvaluator() { - @Override - public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { - // Force render late buffers every 100ms to avoid frozen video effect. - return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && elapsedSinceLastReleaseUs > 100_000; - } - - @Override - public boolean shouldDropFrame( - long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { - return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastFrame; - } - - @Override - public boolean shouldDropFramesToKeyframe( - long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { - return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastFrame; - } - }; - } - - /** The earliest time threshold, in microseconds, after which a frame is considered late. */ - private static final long MIN_EARLY_US_LATE_THRESHOLD = -30_000; - - /** The earliest time threshold, in microseconds, after which a frame is considered very late. */ - private static final long MIN_EARLY_US_VERY_LATE_THRESHOLD = -500_000; - - /** The maximum earliest time, in microseconds, to release a frame on the surface. */ - private static final long MAX_EARLY_US_THRESHOLD = 50_000; - - private final VideoFrameReleaseHelper frameReleaseHelper; - private final long allowedJoiningTimeMs; - - private FrameTimingEvaluator frameTimingEvaluator; - private boolean started; - private @C.FirstFrameState int firstFrameState; - private long initialPositionUs; - private long lastReleaseRealtimeUs; - private long lastPresentationTimeUs; - private long joiningDeadlineMs; - private float playbackSpeed; - private Clock clock; - - /** - * Creates an instance. - * - * @param applicationContext The application context. - * @param allowedJoiningTimeMs The maximum duration in milliseconds for which the renderer can - * attempt to seamlessly join an ongoing playback. - */ - public VideoFrameReleaseControl(Context applicationContext, long allowedJoiningTimeMs) { - frameReleaseHelper = new VideoFrameReleaseHelper(applicationContext); - this.allowedJoiningTimeMs = allowedJoiningTimeMs; - this.frameTimingEvaluator = FrameTimingEvaluator.DEFAULT; - firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; - initialPositionUs = C.TIME_UNSET; - lastPresentationTimeUs = C.TIME_UNSET; - joiningDeadlineMs = C.TIME_UNSET; - playbackSpeed = 1f; - clock = Clock.DEFAULT; - } - - /** Sets the {@link FrameTimingEvaluator}. */ - public void setFrameTimingEvaluator(FrameTimingEvaluator frameTimingEvaluator) { - this.frameTimingEvaluator = frameTimingEvaluator; - } - - /** Called when the renderer is enabled. */ - public void onEnabled(boolean releaseFirstFrameBeforeStarted) { - firstFrameState = - releaseFirstFrameBeforeStarted - ? C.FIRST_FRAME_NOT_RENDERED - : C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; - } - - /** Called when the renderer is disabled. */ - public void onDisabled() { - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED); - } - - /** Called when the renderer is started. */ - public void onStarted() { - started = true; - lastReleaseRealtimeUs = msToUs(clock.elapsedRealtime()); - frameReleaseHelper.onStarted(); - } - - /** Called when the renderer is stopped. */ - public void onStopped() { - started = false; - joiningDeadlineMs = C.TIME_UNSET; - frameReleaseHelper.onStopped(); - } - - /** Called when the renderer processed a stream change. */ - public void onProcessedStreamChange() { - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE); - } - - /** Called when the display surface changed. */ - public void setOutputSurface(@Nullable Surface outputSurface) { - frameReleaseHelper.onSurfaceChanged(outputSurface); - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); - } - - /** Sets the frame rate. */ - public void setFrameRate(float frameRate) { - frameReleaseHelper.onFormatChanged(frameRate); - } - - /** - * Called when a frame have been released. - * - * @return Whether this is the first released frame. - */ - public boolean onFrameReleasedIsFirstFrame() { - boolean firstFrame = firstFrameState != C.FIRST_FRAME_RENDERED; - firstFrameState = C.FIRST_FRAME_RENDERED; - lastReleaseRealtimeUs = msToUs(clock.elapsedRealtime()); - return firstFrame; - } - - /** Sets the clock that will be used. */ - public void setClock(Clock clock) { - this.clock = clock; - } - - /** - * Allows the frame control to indicate the first frame can be released before this instance is - * started. - */ - public void allowReleaseFirstFrameBeforeStarted() { - if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) { - firstFrameState = C.FIRST_FRAME_NOT_RENDERED; - } - } - - /** - * Whether the release control is ready to start playback. - * - * @see Renderer#isReady() - * @param rendererReady Whether the renderer is ready. - * @return Whether the release control is ready. - */ - public boolean isReady(boolean rendererReady) { - if (rendererReady && firstFrameState == C.FIRST_FRAME_RENDERED) { - // Ready. If we were joining then we've now joined, so clear the joining deadline. - joiningDeadlineMs = C.TIME_UNSET; - return true; - } else if (joiningDeadlineMs == C.TIME_UNSET) { - // Not joining. - return false; - } else if (clock.elapsedRealtime() < joiningDeadlineMs) { - // Joining and still withing the deadline. - return true; - } else { - // The joining deadline has been exceeded. Give up and clear the deadline. - joiningDeadlineMs = C.TIME_UNSET; - return false; - } - } - - /** Joins the release control to a new stream. */ - public void join() { - joiningDeadlineMs = - allowedJoiningTimeMs > 0 ? (clock.elapsedRealtime() + allowedJoiningTimeMs) : C.TIME_UNSET; - } - - /** - * Returns a {@link FrameReleaseAction} for a video frame which instructs a renderer what to do - * with the frame. - * - * @param presentationTimeUs The presentation time of the video frame, in microseconds. - * @param positionUs The current playback position, in microseconds. - * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, - * taken approximately at the time the playback position was {@code positionUs}. - * @param outputStreamStartPositionUs The stream's start position, in microseconds. - * @param isLastFrame Whether the frame is known to contain the last frame of the current stream. - * @param frameReleaseInfo A {@link FrameReleaseInfo} that will be filled with detailed data only - * if the method returns {@link #FRAME_RELEASE_IMMEDIATELY} or {@link - * #FRAME_RELEASE_SCHEDULED}. - * @return A {@link FrameReleaseAction} that should instruct the renderer whether to release the - * frame or not. - */ - public @FrameReleaseAction int getFrameReleaseAction( - long presentationTimeUs, - long positionUs, - long elapsedRealtimeUs, - long outputStreamStartPositionUs, - boolean isLastFrame, - FrameReleaseInfo frameReleaseInfo) { - frameReleaseInfo.reset(); - - if (initialPositionUs == C.TIME_UNSET) { - initialPositionUs = positionUs; - } - if (lastPresentationTimeUs != presentationTimeUs) { - frameReleaseHelper.onNextFrame(presentationTimeUs); - lastPresentationTimeUs = presentationTimeUs; - } - - frameReleaseInfo.earlyUs = - calculateEarlyTimeUs(positionUs, elapsedRealtimeUs, presentationTimeUs); - - if (shouldForceRelease(positionUs, frameReleaseInfo.earlyUs, outputStreamStartPositionUs)) { - return FRAME_RELEASE_IMMEDIATELY; - } - if (!started - || positionUs == initialPositionUs - || frameReleaseInfo.earlyUs > MAX_EARLY_US_THRESHOLD) { - return FRAME_RELEASE_TRY_AGAIN_LATER; - } - - // Calculate release time and and adjust earlyUs to screen vsync. - long systemTimeNs = clock.nanoTime(); - frameReleaseInfo.releaseTimeNs = - frameReleaseHelper.adjustReleaseTime(systemTimeNs + (frameReleaseInfo.earlyUs * 1_000)); - frameReleaseInfo.earlyUs = (frameReleaseInfo.releaseTimeNs - systemTimeNs) / 1_000; - // While joining, late frames are skipped. - boolean treatDropAsSkip = joiningDeadlineMs != C.TIME_UNSET; - if (frameTimingEvaluator.shouldDropFramesToKeyframe( - frameReleaseInfo.earlyUs, elapsedRealtimeUs, isLastFrame)) { - return treatDropAsSkip ? FRAME_RELEASE_SKIP_TO_KEYFRAME : FRAME_RELEASE_DROP_TO_KEYFRAME; - } else if (frameTimingEvaluator.shouldDropFrame( - frameReleaseInfo.earlyUs, elapsedRealtimeUs, isLastFrame)) { - // While joining, dropped buffers are considered skipped. - return treatDropAsSkip ? FRAME_RELEASE_SKIP : FRAME_RELEASE_DROP; - } - return FRAME_RELEASE_SCHEDULED; - } - - /** Resets the release control. */ - public void reset() { - frameReleaseHelper.onPositionReset(); - lastPresentationTimeUs = C.TIME_UNSET; - initialPositionUs = C.TIME_UNSET; - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); - joiningDeadlineMs = C.TIME_UNSET; - } - - /** - * Change the {@link C.VideoChangeFrameRateStrategy}, used when calling {@link - * Surface#setFrameRate}. - */ - public void setChangeFrameRateStrategy( - @C.VideoChangeFrameRateStrategy int changeFrameRateStrategy) { - frameReleaseHelper.setChangeFrameRateStrategy(changeFrameRateStrategy); - } - - /** Sets the playback speed. Called when the renderer playback speed changes. */ - public void setPlaybackSpeed(float speed) { - this.playbackSpeed = speed; - frameReleaseHelper.onPlaybackSpeed(speed); - } - - private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) { - this.firstFrameState = min(this.firstFrameState, firstFrameState); - } - - /** - * Calculates the time interval between the current player position and the frame presentation - * time. - * - * @param positionUs The current media time in microseconds, measured at the start of the current - * iteration of the rendering loop. - * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the - * start of the current iteration of the rendering loop. - * @param framePresentationTimeUs The presentation time of the frame in microseconds. - * @return The calculated early time, in microseconds. - */ - private long calculateEarlyTimeUs( - long positionUs, long elapsedRealtimeUs, long framePresentationTimeUs) { - // Calculate how early we are. In other words, the realtime duration that needs to elapse whilst - // the renderer is started before the frame should be rendered. A negative value means that - // we're already late. - // Note: Use of double rather than float is intentional for accuracy in the calculations below. - long earlyUs = (long) ((framePresentationTimeUs - positionUs) / (double) playbackSpeed); - if (started) { - // Account for the elapsed time since the start of this iteration of the rendering loop. - earlyUs -= Util.msToUs(clock.elapsedRealtime()) - elapsedRealtimeUs; - } - - return earlyUs; - } - - /** Returns whether a frame should be force released. */ - private boolean shouldForceRelease( - long positionUs, long earlyUs, long outputStreamStartPositionUs) { - if (joiningDeadlineMs != C.TIME_UNSET) { - // No force releasing during joining. - return false; - } - switch (firstFrameState) { - case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED: - return started; - case C.FIRST_FRAME_NOT_RENDERED: - return true; - case C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE: - return positionUs >= outputStreamStartPositionUs; - case C.FIRST_FRAME_RENDERED: - long elapsedTimeSinceLastReleaseUs = - msToUs(clock.elapsedRealtime()) - lastReleaseRealtimeUs; - return started - && frameTimingEvaluator.shouldForceReleaseFrame(earlyUs, elapsedTimeSinceLastReleaseUs); - default: - throw new IllegalStateException(); - } - } -} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java index 0f441fbcb6..ce43566b22 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java @@ -55,9 +55,6 @@ import java.util.concurrent.Executor; /** Called when the sink renderers the first frame. */ void onFirstFrameRendered(VideoSink videoSink); - /** Called when the sink dropped a frame. */ - void onFrameDropped(VideoSink videoSink); - /** Called when the output video size changed. */ void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize); @@ -65,6 +62,49 @@ import java.util.concurrent.Executor; void onError(VideoSink videoSink, VideoSinkException videoSinkException); } + /** Controls the rendering of video frames. */ + interface RenderControl { + /** Signals a frame must be rendered immediately. */ + long RENDER_TIME_IMMEDIATELY = -1; + + /** Signals a frame must be dropped. */ + long RENDER_TIME_DROP = -2; + + /** Signals that a frame should not be rendered yet. */ + long RENDER_TIME_TRY_AGAIN_LATER = -3; + + /** + * Returns the render timestamp, in nanoseconds, associated with this video frames or one of the + * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not + * rendered yet. + * + * @param presentationTimeUs The presentation time of the video frame, in microseconds. + * @param positionUs The current playback position, in microseconds. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * taken approximately at the time the playback position was {@code positionUs}. + * @param playbackSpeed The current playback speed. + * @return The render timestamp, in nanoseconds, associated with this frame, or one of the + * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not + * rendered yet. + */ + long getFrameRenderTimeNs( + long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed); + + /** + * Informs the rendering control that a video frame will be rendered. Call this method before + * rendering a frame. + * + * @param presentationTimeUs The frame's presentation time, in microseconds. + */ + void onNextFrame(long presentationTimeUs); + + /** Informs the rendering control that a video frame was rendered. */ + void onFrameRendered(); + + /** Informs the rendering control that a video frame was dropped. */ + void onFrameDropped(); + } + /** * Specifies how the input frames are made available to the video sink. One of {@link * #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java index 9627d2030f..871425451c 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java @@ -132,13 +132,11 @@ public final class CompositingVideoSinkProviderTest { } private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() { - VideoFrameReleaseControl releaseControl = - new VideoFrameReleaseControl( - ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0); + VideoSink.RenderControl renderControl = new TestRenderControl(); return new CompositingVideoSinkProvider( ApplicationProvider.getApplicationContext(), new TestPreviewingVideoGraphFactory(), - releaseControl); + renderControl); } private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory { @@ -163,4 +161,22 @@ public final class CompositingVideoSinkProviderTest { return previewingVideoGraph; } } + + private static class TestRenderControl implements VideoSink.RenderControl { + + @Override + public long getFrameRenderTimeNs( + long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) { + return presentationTimeUs; + } + + @Override + public void onNextFrame(long presentationTimeUs) {} + + @Override + public void onFrameRendered() {} + + @Override + public void onFrameDropped() {} + } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java deleted file mode 100644 index cbfa2d7fda..0000000000 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java +++ /dev/null @@ -1,505 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.exoplayer.video; - -import static com.google.common.truth.Truth.assertThat; - -import androidx.media3.test.utils.FakeClock; -import androidx.test.core.app.ApplicationProvider; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** Unit tests for {@link VideoFrameReleaseControl}. */ -@RunWith(AndroidJUnit4.class) -public class VideoFrameReleaseControlTest { - @Test - public void isReady_onNewInstance_returnsFalse() { - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - - assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isFalse(); - assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse(); - } - - @Test - public void isReady_afterReleasingFrame_returnsTrue() { - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - - assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); - assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isTrue(); - } - - @Test - public void isReady_withinJoiningDeadline_returnsTrue() { - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = - createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100); - videoFrameReleaseControl.setClock(clock); - - videoFrameReleaseControl.join(); - - assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue(); - } - - @Test - public void isReady_joiningDeadlineExceeded_returnsFalse() { - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = - createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100); - videoFrameReleaseControl.setClock(clock); - - videoFrameReleaseControl.join(); - assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue(); - - clock.advanceTime(/* timeDiffMs= */ 101); - - assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse(); - } - - @Test - public void onFrameReleasedIsFirstFrame_resetsAfterOnEnabled() { - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - - assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); - } - - @Test - public void onFrameReleasedIsFirstFrame_resetsAfterOnProcessedStreamChange() { - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - - assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); - videoFrameReleaseControl.onProcessedStreamChange(); - - assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); - } - - @Test - public void onFrameReleasedIsFirstFrame_resetsAfterSetOutputSurface() { - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - - assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); - videoFrameReleaseControl.setOutputSurface(/* outputSurface= */ null); - - assertThat(videoFrameReleaseControl.onFrameReleasedIsFirstFrame()).isTrue(); - } - - @Test - public void isReady_afterReset_returnsFalse() { - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isTrue(); - videoFrameReleaseControl.reset(); - - assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ true)).isFalse(); - } - - @Test - public void getFrameReleaseAction_firstFrameAllowedBeforeStart_returnsReleaseImmediately() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - } - - @Test - public void getFrameReleaseAction_firstFrameNotAllowedBeforeStart_returnsTryAgainLater() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ false); - - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); - } - - @Test - public void - getFrameReleaseAction_firstFrameNotAllowedBeforeStartAndStarted_returnsReleaseImmediately() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ false); - - videoFrameReleaseControl.onStarted(); - - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - } - - @Test - public void getFrameReleaseAction_secondFrameAndNotStarted_returnsTryAgainLater() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - - // Second frame - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 10_000, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); - } - - @Test - public void getFrameReleaseAction_secondFrameAndStarted_returnsScheduled() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - videoFrameReleaseControl.onStarted(); - - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - - // Second frame - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 10_000, - /* positionUs= */ 1, - /* elapsedRealtimeUs= */ 1, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED); - } - - @Test - public void getFrameReleaseAction_secondFrameEarly_returnsTryAgainLater() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - videoFrameReleaseControl.onStarted(); - - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - clock.advanceTime(/* timeDiffMs= */ 10); - - // Second frame is 51 ms too soon. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 61_000, - /* positionUs= */ 10_000, - /* elapsedRealtimeUs= */ 10_000, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); - assertThat(frameReleaseInfo.getEarlyUs()).isEqualTo(51_000); - } - - @Test - public void getFrameReleaseAction_frameLate_returnsDrop() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - videoFrameReleaseControl.onStarted(); - - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - clock.advanceTime(/* timeDiffMs= */ 40); - - // Second frame is 31 ms late. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 9_000, - /* positionUs= */ 40_000, - /* elapsedRealtimeUs= */ 40_000, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP); - } - - @Test - public void getFrameReleaseAction_frameLateWhileJoining_returnsSkip() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = - createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 1234); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - videoFrameReleaseControl.onStarted(); - - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - clock.advanceTime(/* timeDiffMs= */ 40); - - // Start joining. - videoFrameReleaseControl.join(); - - // Second frame is 31 ms late. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 9_000, - /* positionUs= */ 40_000, - /* elapsedRealtimeUs= */ 40_000, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP); - } - - @Test - public void getFrameReleaseAction_frameVeryLate_returnsDropToKeyframe() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = - new VideoFrameReleaseControl( - ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.setFrameTimingEvaluator( - new TestFrameTimingEvaluator(/* shouldForceRelease= */ false)); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - videoFrameReleaseControl.onStarted(); - - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - clock.advanceTime(/* timeDiffMs= */ 1_000); - - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 1_000, - /* positionUs= */ 1_000_000, - /* elapsedRealtimeUs= */ 1_000_000, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME); - } - - @Test - public void getFrameReleaseAction_frameVeryLateAndJoining_returnsSkipToFrame() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = - createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 1234); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - videoFrameReleaseControl.onStarted(); - - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - clock.advanceTime(/* timeDiffMs= */ 1_000); - videoFrameReleaseControl.join(); - - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 1_000, - /* positionUs= */ 1_000_000, - /* elapsedRealtimeUs= */ 1_000_000, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME); - } - - @Test - public void - getFrameReleaseAction_frameVeryLateAndFrameTimeEvaluatorForcesFrameRelease_returnsReleaseImmediately() { - VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = - new VideoFrameReleaseControl.FrameReleaseInfo(); - FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); - VideoFrameReleaseControl videoFrameReleaseControl = - new VideoFrameReleaseControl( - ApplicationProvider.getApplicationContext(), /* allowedJoiningTimeMs= */ 0); - videoFrameReleaseControl.setFrameTimingEvaluator( - /* frameTimingEvaluator= */ new TestFrameTimingEvaluator(/* shouldForceRelease= */ true)); - videoFrameReleaseControl.setClock(clock); - videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); - - videoFrameReleaseControl.onStarted(); - - // First frame released. - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 0, - /* positionUs= */ 0, - /* elapsedRealtimeUs= */ 0, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - videoFrameReleaseControl.onFrameReleasedIsFirstFrame(); - clock.advanceTime(/* timeDiffMs= */ 1_000); - - assertThat( - videoFrameReleaseControl.getFrameReleaseAction( - /* presentationTimeUs= */ 1_000, - /* positionUs= */ 1_000_000, - /* elapsedRealtimeUs= */ 1_000_000, - /* outputStreamStartPositionUs= */ 0, - /* isLastFrame= */ false, - frameReleaseInfo)) - .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); - } - - private static VideoFrameReleaseControl createVideoFrameReleaseControl() { - return createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 0); - } - - private static VideoFrameReleaseControl createVideoFrameReleaseControl( - long allowedJoiningTimeMs) { - return new VideoFrameReleaseControl( - ApplicationProvider.getApplicationContext(), allowedJoiningTimeMs); - } - - private static class TestFrameTimingEvaluator - implements VideoFrameReleaseControl.FrameTimingEvaluator { - - private final boolean shouldForceRelease; - - public TestFrameTimingEvaluator(boolean shouldForceRelease) { - this.shouldForceRelease = shouldForceRelease; - } - - @Override - public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { - return shouldForceRelease; - } - - @Override - public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { - return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFrame( - earlyUs, elapsedRealtimeUs, isLastFrame); - } - - @Override - public boolean shouldDropFramesToKeyframe( - long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { - return VideoFrameReleaseControl.FrameTimingEvaluator.DEFAULT.shouldDropFramesToKeyframe( - earlyUs, elapsedRealtimeUs, isLastFrame); - } - } -}