diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 408ce167cb..59b869dd21 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -172,8 +172,12 @@ public interface VideoFrameProcessor { * rendering. * * @param presentationTimeUs The presentation time of the frame, in microseconds. + * @param isRedrawnFrame Whether the frame is a frame that is {@linkplain #redraw redrawn}, + * redrawn frames are rendered directly thus {@link #renderOutputFrame} must not be called + * on such frames. */ - default void onOutputFrameAvailableForRendering(long presentationTimeUs) {} + default void onOutputFrameAvailableForRendering( + long presentationTimeUs, boolean isRedrawnFrame) {} /** * Called when an exception occurs during asynchronous video frame processing. @@ -354,15 +358,16 @@ public interface VideoFrameProcessor { /** * Renders the oldest unrendered output frame that has become {@linkplain - * Listener#onOutputFrameAvailableForRendering(long) available for rendering} at the given {@code - * renderTimeNs}. + * Listener#onOutputFrameAvailableForRendering(long, boolean) available for rendering} at the + * given {@code renderTimeNs}. * *

This will either render the output frame to the {@linkplain #setOutputSurfaceInfo output * surface}, or drop the frame, per {@code renderTimeNs}. * *

This method must only be called if {@code renderFramesAutomatically} was set to {@code * false} using the {@link Factory} and should be called exactly once for each frame that becomes - * {@linkplain Listener#onOutputFrameAvailableForRendering(long) available for rendering}. + * {@linkplain Listener#onOutputFrameAvailableForRendering(long, boolean) available for + * rendering}. * *

The {@code renderTimeNs} may be passed to {@link EGLExt#eglPresentationTimeANDROID} * depending on the implementation. @@ -371,8 +376,8 @@ public interface VideoFrameProcessor { * be before or after the current system time. Use {@link #DROP_OUTPUT_FRAME} to drop the * frame or {@link #RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME} to render the frame to the * {@linkplain #setOutputSurfaceInfo output surface} with the presentation timestamp seen in - * {@link Listener#onOutputFrameAvailableForRendering(long)}. If the frame should be rendered - * immediately, pass in {@link SystemClock#nanoTime()}. + * {@link Listener#onOutputFrameAvailableForRendering(long, boolean)}. If the frame should be + * rendered immediately, pass in {@link SystemClock#nanoTime()}. */ void renderOutputFrame(long renderTimeNs); diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 1e3502305c..b57852b159 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -93,8 +93,12 @@ public interface VideoGraph { * for rendering. * * @param framePresentationTimeUs The presentation time of the frame, in microseconds. + * @param isRedrawnFrame Whether the frame is a frame that is {@linkplain #redraw redrawn}, + * redrawn frames are rendered directly thus {@link #renderOutputFrame} must not be called + * on such frames. */ - default void onOutputFrameAvailableForRendering(long framePresentationTimeUs) {} + default void onOutputFrameAvailableForRendering( + long framePresentationTimeUs, boolean isRedrawnFrame) {} /** * Called after the {@link VideoGraph} has rendered its final output frame. @@ -224,8 +228,8 @@ public interface VideoGraph { * Renders the output frame from the {@code VideoGraph}. * *

This method must be called only for frames that have become {@linkplain - * Listener#onOutputFrameAvailableForRendering(long) available}, calling the method renders the - * frame that becomes available the earliest but not yet rendered. + * Listener#onOutputFrameAvailableForRendering available}, calling the method renders the frame + * that becomes available the earliest but not yet rendered. * * @see VideoFrameProcessor#renderOutputFrame(long) */ diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java index b8d9b46b0f..822cb8c6fc 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java @@ -225,7 +225,8 @@ public class DefaultVideoFrameProcessorTest { } @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long presentationTimeUs, boolean isRedrawnFrame) { outputFrameCount++; if (outputFrameCount == 30) { firstStreamLastFrameAvailableTimeMs.set(SystemClock.DEFAULT.elapsedRealtime()); @@ -312,7 +313,8 @@ public class DefaultVideoFrameProcessorTest { } @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long presentationTimeUs, boolean isRedrawnFrame) { outputFrameAvailableConditionVariable.open(); } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java index f9fba96348..bd1a7212ba 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java @@ -294,7 +294,8 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest { } @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long presentationTimeUs, boolean isRedrawnFrame) { onFrameAvailableListener.onFrameAvailableForRendering(presentationTimeUs); } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java index f54f2df1de..51c3af5d32 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java @@ -139,7 +139,8 @@ import java.util.concurrent.atomic.AtomicReference; } @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long presentationTimeUs, boolean isRedrawnFrame) { actualPresentationTimesUs.add(presentationTimeUs); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 62e3754e2e..b92e7ce1a9 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -220,10 +220,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { videoFrameProcessingTaskExecutor.verifyVideoFrameProcessingThread(); + if (!isWaitingForRedrawFrame()) { // Don't report output available when redrawing - the redrawn frames are released immediately. videoFrameProcessorListenerExecutor.execute( - () -> videoFrameProcessorListener.onOutputFrameAvailableForRendering(presentationTimeUs)); + () -> + videoFrameProcessorListener.onOutputFrameAvailableForRendering( + presentationTimeUs, /* isRedrawnFrame= */ false)); } if (textureOutputListener == null) { @@ -238,6 +241,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; if (isWaitingForRedrawFrame()) { if (presentationTimeUs == redrawFramePresentationTimeUs) { redrawFramePresentationTimeUs = C.TIME_UNSET; + videoFrameProcessorListenerExecutor.execute( + () -> + videoFrameProcessorListener.onOutputFrameAvailableForRendering( + presentationTimeUs, /* isRedrawnFrame= */ true)); renderFrame( glObjectsProvider, inputTexture, diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index 4cc6156cb1..30f01c11b3 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -229,14 +229,17 @@ public final class MultipleInputVideoGraph implements VideoGraph { } @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long presentationTimeUs, boolean isRedrawnFrame) { if (presentationTimeUs == 0) { hasProducedFrameWithTimestampZero = true; } lastRenderedPresentationTimeUs = presentationTimeUs; listenerExecutor.execute( - () -> listener.onOutputFrameAvailableForRendering(presentationTimeUs)); + () -> + listener.onOutputFrameAvailableForRendering( + presentationTimeUs, isRedrawnFrame)); } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 511662ceea..0f06cbd22b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -172,14 +172,17 @@ public class SingleInputVideoGraph implements VideoGraph { } @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long presentationTimeUs, boolean isRedrawnFrame) { // Frames are rendered automatically. if (presentationTimeUs == 0) { hasProducedFrameWithTimestampZero = true; } lastProcessedFramePresentationTimeUs = presentationTimeUs; listenerExecutor.execute( - () -> listener.onOutputFrameAvailableForRendering(presentationTimeUs)); + () -> + listener.onOutputFrameAvailableForRendering( + presentationTimeUs, isRedrawnFrame)); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index dc6dd0531f..f5c9f3e94f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -297,6 +297,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private Format videoGraphOutputFormat; private @MonotonicNonNull HandlerWrapper handler; private @MonotonicNonNull VideoGraph videoGraph; + private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener; private long outputStreamStartPositionUs; private @VideoSink.FirstFrameReleaseInstruction int nextFirstOutputFrameReleaseInstruction; @Nullable private Pair currentSurfaceAndSize; @@ -438,7 +439,8 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video } @Override - public void onOutputFrameAvailableForRendering(long framePresentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long framePresentationTimeUs, boolean isRedrawnFrame) { if (pendingFlushCount > 0) { // Ignore available frames while flushing return; @@ -447,9 +449,22 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video // Wake up the player when not playing to render the frame more promptly. wakeupListener.onWakeup(); } + + long bufferPresentationTimeUs = framePresentationTimeUs - bufferTimestampAdjustmentUs; + if (isRedrawnFrame) { + // Redrawn frames are rendered directly in the processing pipeline. + if (videoFrameMetadataListener != null) { + videoFrameMetadataListener.onVideoFrameAboutToBeRendered( + /* presentationTimeUs= */ bufferPresentationTimeUs, + /* releaseTimeNs= */ C.TIME_UNSET, + videoGraphOutputFormat, + /* mediaFormat= */ null); + } + return; + } + // The frame presentation time is relative to the start of the Composition and without the // renderer offset - long bufferPresentationTimeUs = framePresentationTimeUs - bufferTimestampAdjustmentUs; lastOutputBufferPresentationTimeUs = bufferPresentationTimeUs; Long newOutputStreamStartPositionUs = streamStartPositionsUs.pollFloor(bufferPresentationTimeUs); @@ -614,6 +629,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private void setVideoFrameMetadataListener( VideoFrameMetadataListener videoFrameMetadataListener) { + this.videoFrameMetadataListener = videoFrameMetadataListener; defaultVideoSink.setVideoFrameMetadataListener(videoFrameMetadataListener); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameMetadataListener.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameMetadataListener.java index 1a0d972c66..4e09418b4f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameMetadataListener.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameMetadataListener.java @@ -17,6 +17,7 @@ package androidx.media3.exoplayer.video; import android.media.MediaFormat; import androidx.annotation.Nullable; +import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.util.UnstableApi; @@ -28,7 +29,8 @@ public interface VideoFrameMetadataListener { * * @param presentationTimeUs The presentation time of the frame, in microseconds. * @param releaseTimeNs The system time at which the frame should be displayed, in nanoseconds. - * Can be compared to {@link System#nanoTime()}. + * Can be compared to {@link System#nanoTime()}. It will be {@link C#TIME_UNSET}, if the frame + * is rendered immediately automatically, this is typically the last frame that is rendered. * @param format The format associated with the frame. * @param mediaFormat The framework media format associated with the frame, or {@code null} if not * known or not applicable (e.g., because the frame was not output by a {@link diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index 3421b3b0c3..8bcae88d14 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -306,7 +306,8 @@ public final class VideoFrameProcessorTestRunner { } @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long presentationTimeUs, boolean isRedrawnFrame) { // Do nothing as frames are rendered automatically. onOutputFrameAvailableForRenderingListener.onFrameAvailableForRendering( presentationTimeUs); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java index 8ac77db0db..c916848556 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerSeekTest.java @@ -1072,8 +1072,9 @@ public class CompositionPlayerSeekTest { } @Override - public void onOutputFrameAvailableForRendering(long framePresentationTimeUs) { - listener.onOutputFrameAvailableForRendering(framePresentationTimeUs); + public void onOutputFrameAvailableForRendering( + long framePresentationTimeUs, boolean isRedrawnFrame) { + listener.onOutputFrameAvailableForRendering(framePresentationTimeUs, isRedrawnFrame); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index c9fcd2eea5..a7c1533572 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -570,7 +570,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public void onOutputFrameAvailableForRendering(long framePresentationTimeUs) { + public void onOutputFrameAvailableForRendering( + long framePresentationTimeUs, boolean isRedrawnFrame) { if (!renderFramesAutomatically) { synchronized (lock) { framesAvailableToRender += 1;