From 0e169ab1bea3a4cd9ff2772d77618c66b5262f3c Mon Sep 17 00:00:00 2001 From: tonihei Date: Tue, 18 Mar 2025 03:44:46 -0700 Subject: [PATCH] Move decode-only and no surface logic inside VideoFrameReleaseControl This brings the parts related to video frame release decision making in a single place and simplifies the calling side in MediaCodecVideoRenderer. PiperOrigin-RevId: 737941729 --- .../exoplayer/video/DefaultVideoSink.java | 2 +- .../video/MediaCodecVideoRenderer.java | 28 +-- .../video/VideoFrameReleaseControl.java | 28 ++- .../video/VideoFrameRenderControl.java | 1 + .../video/VideoFrameReleaseControlTest.java | 211 +++++++++++++++++- .../video/VideoFrameRenderControlTest.java | 31 ++- .../ExperimentalFrameExtractor.java | 10 +- 7 files changed, 261 insertions(+), 50 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java index 897a77131f..391fc10bf4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java @@ -132,7 +132,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public boolean isReady(boolean rendererOtherwiseReady) { - return videoFrameReleaseControl.isReady(rendererOtherwiseReady); + return outputSurface == null || videoFrameReleaseControl.isReady(rendererOtherwiseReady); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 493aad7d66..5648b4bf82 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -1736,9 +1736,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer }); } - // The frame release action should be retrieved for all frames (even the ones that will be - // skipped), because the release control estimates the content frame rate from frame timestamps - // and we want to have this information known as early as possible, especially during seeking. @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction = videoFrameReleaseControl.getFrameReleaseAction( @@ -1746,31 +1743,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer positionUs, elapsedRealtimeUs, getOutputStreamStartPositionUs(), + isDecodeOnlyBuffer, isLastBuffer, videoFrameReleaseInfo); - - if (frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IGNORE) { - // The buffer is no longer valid and needs to be ignored. - return false; - } - - // Skip decode-only buffers, e.g. after seeking, immediately. - if (isDecodeOnlyBuffer && !isLastBuffer) { - skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - return true; - } - - // We are not rendering on a surface, the renderer will wait until a surface is set. - if (displaySurface == null) { - // Skip frames in sync with playback, so we'll be at the right frame if a surface is set. - if (getState() == STATE_STARTED && videoFrameReleaseInfo.getEarlyUs() < 30_000) { - skipOutputBuffer(codec, bufferIndex, presentationTimeUs); - updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); - return true; - } - return false; - } - switch (frameReleaseAction) { case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: long releaseTimeNs = getClock().nanoTime(); @@ -1787,6 +1762,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); return true; case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER: + case VideoFrameReleaseControl.FRAME_RELEASE_IGNORE: return false; case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED: releaseFrame(checkStateNotNull(codec), bufferIndex, presentationTimeUs, format); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java index 921018d7ab..e36f00ba3e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java @@ -71,7 +71,7 @@ public final class VideoFrameReleaseControl { /** * The frame release action returned by {@link #getFrameReleaseAction(long, long, long, long, - * boolean, FrameReleaseInfo)}. + * boolean, boolean, FrameReleaseInfo)}. * *

One of {@link #FRAME_RELEASE_IMMEDIATELY}, {@link #FRAME_RELEASE_SCHEDULED}, {@link * #FRAME_RELEASE_DROP}, {@link #FRAME_RELEASE_IGNORE}, {@link ##FRAME_RELEASE_SKIP} or {@link @@ -208,14 +208,15 @@ public final class VideoFrameReleaseControl { private boolean joiningRenderNextFrameImmediately; private float playbackSpeed; private Clock clock; + private boolean hasOutputSurface; /** * Creates an instance. * * @param applicationContext The application context. * @param frameTimingEvaluator The {@link FrameTimingEvaluator} that will assist in {@linkplain - * #getFrameReleaseAction(long, long, long, long, boolean, FrameReleaseInfo) frame release - * actions}. + * #getFrameReleaseAction(long, long, long, long, boolean, boolean, FrameReleaseInfo) frame + * release actions}. * @param allowedJoiningTimeMs The maximum duration in milliseconds for which the caller can * attempt to seamlessly join an ongoing playback. */ @@ -271,6 +272,7 @@ public final class VideoFrameReleaseControl { /** Called when the display surface changed. */ public void setOutputSurface(@Nullable Surface outputSurface) { + hasOutputSurface = outputSurface != null; frameReleaseHelper.onSurfaceChanged(outputSurface); lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); } @@ -355,6 +357,8 @@ public final class VideoFrameReleaseControl { * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, * taken approximately at the time the playback position was {@code positionUs}. * @param outputStreamStartPositionUs The stream's start position, in microseconds. + * @param isDecodeOnlyFrame Whether the frame is decode-only because its presentation time is + * before the intended start time. * @param isLastFrame Whether the frame is known to contain the last frame of the current stream. * @param frameReleaseInfo A {@link FrameReleaseInfo} that will be filled with detailed data only * if the method returns {@link #FRAME_RELEASE_IMMEDIATELY} or {@link @@ -367,6 +371,7 @@ public final class VideoFrameReleaseControl { long positionUs, long elapsedRealtimeUs, long outputStreamStartPositionUs, + boolean isDecodeOnlyFrame, boolean isLastFrame, FrameReleaseInfo frameReleaseInfo) throws ExoPlaybackException { @@ -383,6 +388,23 @@ public final class VideoFrameReleaseControl { frameReleaseInfo.earlyUs = calculateEarlyTimeUs(positionUs, elapsedRealtimeUs, presentationTimeUs); + if (isDecodeOnlyFrame && !isLastFrame) { + return FRAME_RELEASE_SKIP; + } + if (!hasOutputSurface) { + // Skip frames in sync with playback, so we'll be at the right frame if a surface is set. + if (frameTimingEvaluator.shouldIgnoreFrame( + frameReleaseInfo.earlyUs, + positionUs, + elapsedRealtimeUs, + isLastFrame, + /* treatDroppedBuffersAsSkipped= */ true)) { + return FRAME_RELEASE_IGNORE; + } + return started && frameReleaseInfo.earlyUs < 30_000 + ? FRAME_RELEASE_SKIP + : FRAME_RELEASE_TRY_AGAIN_LATER; + } if (shouldForceRelease(positionUs, frameReleaseInfo.earlyUs, outputStreamStartPositionUs)) { return FRAME_RELEASE_IMMEDIATELY; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java index 07ef1ff79b..f49286ae82 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java @@ -144,6 +144,7 @@ import androidx.media3.exoplayer.ExoPlaybackException; positionUs, elapsedRealtimeUs, outputStreamStartPositionUs, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, videoFrameReleaseInfo); switch (frameReleaseAction) { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java index 969f14f3c2..eddc29fae4 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java @@ -19,16 +19,33 @@ import static androidx.media3.exoplayer.video.VideoFrameReleaseControl.RELEASE_F import static androidx.media3.exoplayer.video.VideoFrameReleaseControl.RELEASE_FIRST_FRAME_WHEN_STARTED; import static com.google.common.truth.Truth.assertThat; +import android.graphics.SurfaceTexture; +import android.view.Surface; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.test.utils.FakeClock; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; /** Unit tests for {@link VideoFrameReleaseControl}. */ @RunWith(AndroidJUnit4.class) public class VideoFrameReleaseControlTest { + + private Surface surface; + + @Before + public void setUp() { + surface = new Surface(new SurfaceTexture(/* texName= */ 0)); + } + + @After + public void tearDown() { + surface.release(); + } + @Test public void isReady_onNewInstance_returnsFalse() { VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); @@ -145,6 +162,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -164,6 +182,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); @@ -188,6 +207,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -210,6 +230,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -222,6 +243,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); @@ -245,6 +267,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -257,6 +280,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 1, /* elapsedRealtimeUs= */ 1, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED); @@ -281,6 +305,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -294,6 +319,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 10_000, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); @@ -312,6 +338,7 @@ public class VideoFrameReleaseControlTest { /* shouldDropFrame= */ true, /* shouldIgnoreFrame= */ false), /* allowedJoiningTimeMs= */ 0); + videoFrameReleaseControl.setOutputSurface(surface); videoFrameReleaseControl.setClock(clock); videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); @@ -324,6 +351,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -337,6 +365,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP); @@ -357,6 +386,7 @@ public class VideoFrameReleaseControlTest { /* shouldDropFrame= */ true, /* shouldIgnoreFrame= */ false), /* allowedJoiningTimeMs= */ 1234); + videoFrameReleaseControl.setOutputSurface(surface); videoFrameReleaseControl.setClock(clock); videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); videoFrameReleaseControl.onStarted(); @@ -371,6 +401,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); @@ -381,6 +412,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 11_000, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP); @@ -401,6 +433,7 @@ public class VideoFrameReleaseControlTest { /* shouldDropFrame= */ true, /* shouldIgnoreFrame= */ false), /* allowedJoiningTimeMs= */ 1234); + videoFrameReleaseControl.setOutputSurface(surface); videoFrameReleaseControl.setClock(clock); videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); videoFrameReleaseControl.onStarted(); @@ -415,6 +448,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -426,6 +460,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 11_000, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP); @@ -444,6 +479,7 @@ public class VideoFrameReleaseControlTest { /* shouldDropFrame= */ false, /* shouldIgnoreFrame= */ true), /* allowedJoiningTimeMs= */ 0); + videoFrameReleaseControl.setOutputSurface(surface); videoFrameReleaseControl.setClock(clock); videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); @@ -456,6 +492,7 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 0, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); @@ -468,21 +505,181 @@ public class VideoFrameReleaseControlTest { /* positionUs= */ 1_000, /* elapsedRealtimeUs= */ 0, /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, /* isLastFrame= */ false, frameReleaseInfo)) .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IGNORE); } - private static VideoFrameReleaseControl createVideoFrameReleaseControl() { + @Test + public void getFrameReleaseAction_decodeOnlyFrame_returnsSkip() throws Exception { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ true, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP); + } + + @Test + public void getFrameReleaseAction_decodeOnlyAndLastFrame_returnsReleaseImmediately() + throws Exception { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ true, + /* isLastFrame= */ true, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.RELEASE_FIRST_FRAME_IMMEDIATELY); + } + + @Test + public void getFrameReleaseAction_decodeOnlyFrameWithoutSurface_returnsSkip() throws Exception { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setOutputSurface(/* outputSurface= */ null); + videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 0, + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ true, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP); + } + + @Test + public void getFrameReleaseAction_withoutSurfaceOnTime_returnsTryAgainLater() throws Exception { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setOutputSurface(/* outputSurface= */ null); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 100_000, + /* positionUs= */ 50_000, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); + } + + @Test + public void getFrameReleaseAction_withoutSurfaceShouldIgnore_returnsIgnore() throws Exception { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), + new TestFrameTimingEvaluator( + /* shouldForceRelease= */ false, + /* shouldDropFrame= */ false, + /* shouldIgnoreFrame= */ true), + /* allowedJoiningTimeMs= */ 0); + videoFrameReleaseControl.setOutputSurface(/* outputSurface= */ null); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 100_000, + /* positionUs= */ 50_000, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IGNORE); + } + + @Test + public void getFrameReleaseAction_withoutSurfaceFrameLateNotStarted_returnsTryAgainLater() + throws Exception { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setOutputSurface(/* outputSurface= */ null); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); + + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 100_000, + /* positionUs= */ 90_000, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER); + } + + @Test + public void getFrameReleaseAction_withoutSurfaceFrameLateAndStarted_returnsSkip() + throws Exception { + VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo = + new VideoFrameReleaseControl.FrameReleaseInfo(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setOutputSurface(/* outputSurface= */ null); + videoFrameReleaseControl.setClock(clock); + videoFrameReleaseControl.onStreamChanged(RELEASE_FIRST_FRAME_IMMEDIATELY); + + videoFrameReleaseControl.onStarted(); + assertThat( + videoFrameReleaseControl.getFrameReleaseAction( + /* presentationTimeUs= */ 100_000, + /* positionUs= */ 90_000, + /* elapsedRealtimeUs= */ 0, + /* outputStreamStartPositionUs= */ 0, + /* isDecodeOnlyFrame= */ false, + /* isLastFrame= */ false, + frameReleaseInfo)) + .isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP); + } + + private VideoFrameReleaseControl createVideoFrameReleaseControl() { return createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 0); } - private static VideoFrameReleaseControl createVideoFrameReleaseControl( - long allowedJoiningTimeMs) { - return new VideoFrameReleaseControl( - ApplicationProvider.getApplicationContext(), - new TestFrameTimingEvaluator(), - allowedJoiningTimeMs); + private VideoFrameReleaseControl createVideoFrameReleaseControl(long allowedJoiningTimeMs) { + VideoFrameReleaseControl videoFrameReleaseControl = + new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), + new TestFrameTimingEvaluator(), + allowedJoiningTimeMs); + videoFrameReleaseControl.setOutputSurface(surface); + return videoFrameReleaseControl; } private static class TestFrameTimingEvaluator diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java index 6fa7a8d9c8..36225dfae9 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java @@ -22,10 +22,14 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import android.graphics.SurfaceTexture; +import android.view.Surface; import androidx.media3.common.VideoSize; import androidx.media3.test.utils.FakeClock; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InOrder; @@ -38,6 +42,18 @@ public class VideoFrameRenderControlTest { private static final int VIDEO_WIDTH = 640; private static final int VIDEO_HEIGHT = 480; + private Surface surface; + + @Before + public void setUp() { + surface = new Surface(new SurfaceTexture(/* texName= */ 0)); + } + + @After + public void tearDown() { + surface.release(); + } + @Test public void releaseFirstFrame() throws Exception { VideoFrameRenderControl.FrameRenderer frameRenderer = @@ -279,7 +295,7 @@ public class VideoFrameRenderControlTest { assertThat(videoFrameRenderControl.isEnded()).isFalse(); } - private static VideoFrameReleaseControl createVideoFrameReleaseControl() { + private VideoFrameReleaseControl createVideoFrameReleaseControl() { return createVideoFrameReleaseControl( new TestFrameTimingEvaluator( /* shouldForceReleaseFrames= */ false, @@ -287,12 +303,15 @@ public class VideoFrameRenderControlTest { /* shouldIgnoreFrames= */ false)); } - private static VideoFrameReleaseControl createVideoFrameReleaseControl( + private VideoFrameReleaseControl createVideoFrameReleaseControl( VideoFrameReleaseControl.FrameTimingEvaluator frameTimingEvaluator) { - return new VideoFrameReleaseControl( - ApplicationProvider.getApplicationContext(), - frameTimingEvaluator, - /* allowedJoiningTimeMs= */ 0); + VideoFrameReleaseControl videoFrameReleaseControl = + new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), + frameTimingEvaluator, + /* allowedJoiningTimeMs= */ 0); + videoFrameReleaseControl.setOutputSurface(surface); + return videoFrameReleaseControl; } private static class TestFrameTimingEvaluator diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java index 9cc9c9e7e3..07cda40af8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java @@ -716,13 +716,9 @@ public final class ExperimentalFrameExtractor { public boolean isReady() { // When using FrameReadingGlShaderProgram, frames will not be rendered to the output surface, // and VideoFrameRenderControl.onFrameAvailableForRendering will not be called. The base class - // never becomes ready. - if (frameRenderedSinceLastPositionReset) { - // Treat this renderer as ready if a frame has been rendered into the effects pipeline. - // The renderer needs to become ready for ExoPlayer to enter STATE_READY. - return true; - } - return super.isReady(); + // never becomes ready. Treat this renderer as ready if a frame has been rendered into the + // effects pipeline. The renderer needs to become ready for ExoPlayer to enter STATE_READY. + return frameRenderedSinceLastPositionReset; } @Override