From ff8dd0b4b9aa498a831c9721c530e77bf099a4ed Mon Sep 17 00:00:00 2001 From: claincly Date: Wed, 5 Oct 2022 14:45:27 +0000 Subject: [PATCH] Add FrameProcessor functionality to release a frame immediately Currently `FrameProcessor.releaseOutputFrame()` method supports Release at a specific system time Drops the frame This API is not that convenient to use when the caller wants to release a frame, now, regardless of the release time. A use case is to release (present) a frame when no frame is shown for a while, and it's thus better to just release the frame, now. Currently if MCVR wants a frame to be rendered now, MCVR calls release frame with a set offset like 10us: `releaseOutputFrame(System.nanoTime() + 10_000)`. The 10us offset is to prevent the frame processor dropping the frame, due to thread hopping delays. To make the API better usable, consider adding a mode for releasing the frame now, like (bold marks the new mode) - Use C.TIME_UNSET to drop - **Use -1 to release the frame immediately, or** - Use an actual release time. PiperOrigin-RevId: 479044215 --- ...EffectsFrameProcessorFrameReleaseTest.java | 23 +++++++++++++++++++ .../FinalMatrixTextureProcessorWrapper.java | 8 ++++++- .../media3/common/FrameProcessor.java | 11 +++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java b/google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java index c6c1f0b910..7ddc9b5213 100644 --- a/google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java +++ b/google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java @@ -135,6 +135,29 @@ public final class GlEffectsFrameProcessorFrameReleaseTest { assertThat(outputReleaseTimesNs).containsExactly(releaseTimesNs); } + @Test + public void controlledFrameRelease_withOneFrameRequestImmediateRelease_releasesFrame() + throws Exception { + long originalPresentationTimeUs = 1234; + long releaseTimesNs = FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY; + AtomicLong actualPresentationTimeUs = new AtomicLong(); + setupGlEffectsFrameProcessorWithBlankFrameProducer( + /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, + /* onFrameAvailableListener= */ presentationTimeUs -> { + actualPresentationTimeUs.set(presentationTimeUs); + checkNotNull(glEffectsFrameProcessor).releaseOutputFrame(releaseTimesNs); + }, + /* releaseFramesAutomatically= */ false); + + checkNotNull(produceBlankFramesTask).run(); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNull(); + assertThat(actualPresentationTimeUs.get()).isEqualTo(originalPresentationTimeUs); + // The actual release time is determined by the FrameProcessor when releasing the frame. + assertThat(outputReleaseTimesNs).hasSize(1); + } + @Test public void controlledFrameRelease_withLateFrame_dropsFrame() throws Exception { long originalPresentationTimeUs = 1234; diff --git a/google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java b/google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java index 8c03c78c61..f2a3e72ddd 100644 --- a/google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java +++ b/google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java @@ -170,12 +170,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public void releaseOutputFrame(long releaseTimeNs) { checkState(!releaseFramesAutomatically); + boolean dropLateFrame = true; + if (releaseTimeNs == FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY) { + dropLateFrame = false; + releaseTimeNs = System.nanoTime(); + } + Pair oldestAvailableFrame = availableFrames.remove(); renderFrameToSurfaces( /* inputTexture= */ oldestAvailableFrame.first, /* presentationTimeUs= */ oldestAvailableFrame.second, releaseTimeNs, - /* dropLateFrame= */ true); + dropLateFrame); } @Override diff --git a/library/common/src/main/java/androidx/media3/common/FrameProcessor.java b/library/common/src/main/java/androidx/media3/common/FrameProcessor.java index 64e940bdb7..c54b92af09 100644 --- a/library/common/src/main/java/androidx/media3/common/FrameProcessor.java +++ b/library/common/src/main/java/androidx/media3/common/FrameProcessor.java @@ -101,6 +101,12 @@ public interface FrameProcessor { void onFrameProcessingEnded(); } + /** + * Indicates the frame should be released immediately after {@link #releaseOutputFrame(long)} is + * invoked. + */ + long RELEASE_OUTPUT_FRAME_IMMEDIATELY = -1; + /** Returns the input {@link Surface}, where {@link FrameProcessor} consumes input frames from. */ Surface getInputSurface(); @@ -164,8 +170,9 @@ public interface FrameProcessor { * {@linkplain Listener#onOutputFrameAvailable(long) available}. * * @param releaseTimeNs The release time to use for the frame, in nanoseconds. Use {@link - * C#TIME_UNSET} to drop the frame. If {@code releaseTimeNs} is after {@link - * System#nanoTime()} at the time of the release, the frame is also dropped. + * C#TIME_UNSET} to drop the frame, or {@link #RELEASE_OUTPUT_FRAME_IMMEDIATELY} to release + * the frame immediately. If {@code releaseTimeNs} is after {@link System#nanoTime()} at the + * time of the release, the frame is also dropped. */ void releaseOutputFrame(long releaseTimeNs);