From d5d85558c1af98a0f662ae33df4b083b41da3dcf Mon Sep 17 00:00:00 2001 From: michaelkatz Date: Mon, 16 Dec 2024 08:02:58 -0800 Subject: [PATCH] Skip just-early video frames only if directed to release the frame `MediaCodecVideoRenderer` will skip frames if a surface has not been set and video frame presentation time is early but too close to the current playback position. In the case that the `VideoFrameReleaseControl` says to `FRAME_RELEASE_TRY_AGAIN_LATER`, these frames should not be skipped. PiperOrigin-RevId: 706711734 --- RELEASENOTES.md | 4 + .../video/MediaCodecVideoRenderer.java | 4 +- .../video/MediaCodecVideoRendererTest.java | 78 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 2d6a63b5ee..38a8dc8edb 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -56,6 +56,10 @@ * Rollback of using `MediaCodecAdapter` supplied pixel aspect ratio values when provided while processing `onOutputFormatChanged` ([#1371](https://github.com/androidx/media/pull/1371)). + * Fix `MediaCodecVideoRenderer` such that when without a `Surface`, the + renderer will skip just-early frames only if the + `VideoFrameReleaseControl.getFrameReleaseAction` is not + `FRAME_RELEASE_TRY_AGAIN_LATER`. * Text: * Stop eagerly loading all subtitle files configured with `MediaItem.Builder.setSubtitleConfigurations`, and instead only load one diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 2e8725d7f0..4eabf8842e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -1509,7 +1509,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer // We are not rendering on a surface, the renderer will wait until a surface is set. if (displaySurface == null) { // Skip frames in sync with playback, so we'll be at the right frame if the mode changes. - if (videoFrameReleaseInfo.getEarlyUs() < 30_000) { + if (videoFrameReleaseInfo.getEarlyUs() < 0 + || (videoFrameReleaseInfo.getEarlyUs() < 30_000 + && frameReleaseAction != VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER)) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); updateVideoFrameProcessingOffsetCounters(videoFrameReleaseInfo.getEarlyUs()); return true; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index f188f10014..6767db1139 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -299,6 +299,84 @@ public class MediaCodecVideoRendererTest { assertThat(decoderCounters.droppedToKeyframeCount).isEqualTo(1); } + @Test + public void render_earlyWithoutSurfaceAndStarted_skipsBuffer() throws Exception { + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 50_000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + // Set placeholder surface. + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, null); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ true, + /* mayRenderStartOfStream= */ false, + /* startPositionUs= */ 50_000, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + int posUs = 20_001; // Ensures buffer will be 29_999us early. + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + shadowOf(testMainLooper).idle(); + + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(1); + } + + @Test + public void render_earlyWithoutSurfaceAndNotStarted_doesNotSkipBuffer() throws Exception { + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 50_000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + // Set placeholder surface. + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, null); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ true, + /* mayRenderStartOfStream= */ false, + /* startPositionUs= */ 50_000, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.setCurrentStreamFinal(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + int posUs = 20_001; // Ensures buffer will be 29_999us early. + for (int i = 0; i < 3; i++) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 10_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(0); + } + @Test public void render_withBufferLimitEqualToNumberOfSamples_rendersLastFrameAfterEndOfStream() throws Exception {