From 8260bb3d2efa4aacda6279427fc02f97f7eb648a Mon Sep 17 00:00:00 2001 From: kimvde Date: Tue, 22 Oct 2024 05:41:55 -0700 Subject: [PATCH] Refactor frame rate notification to release control Frame rate change is currently notified to the release control when the video frame processor input frame rate changes, but it should be when the video frame processor output frame rate changes. PiperOrigin-RevId: 688512300 --- .../media3/common/VideoFrameProcessor.java | 8 +++++++ .../androidx/media3/common/VideoGraph.java | 8 +++++++ .../effect/DefaultVideoFrameProcessor.java | 24 +++++++++++++------ .../effect/MultipleInputVideoGraph.java | 5 ++++ .../media3/effect/SingleInputVideoGraph.java | 5 ++++ .../video/PlaybackVideoGraphWrapper.java | 6 ++++- 6 files changed, 48 insertions(+), 8 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 7bff863d7b..7b44416b70 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -157,6 +157,14 @@ public interface VideoFrameProcessor { */ default void onOutputSizeChanged(int width, int height) {} + /** + * Called when the output frame rate changes. + * + * @param frameRate The output frame rate in frames per second, or {@link Format#NO_VALUE} if + * unknown. + */ + default void onOutputFrameRateChanged(float frameRate) {} + /** * Called when an output frame with the given {@code presentationTimeUs} becomes available for * rendering. diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 52d2a2c2b1..460bb0f4bf 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -35,6 +35,14 @@ public interface VideoGraph { */ default void onOutputSizeChanged(int width, int height) {} + /** + * Called when the output frame rate changes. + * + * @param frameRate The output frame rate in frames per second, or {@link Format#NO_VALUE} if + * unknown. + */ + default void onOutputFrameRateChanged(float frameRate) {} + /** * Called when an output frame with the given {@code framePresentationTimeUs} becomes available * for rendering. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 2dd3c0833f..9b88154a77 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -457,6 +457,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final List intermediateGlShaderPrograms; private final ConditionVariable inputStreamRegisteredCondition; + private @MonotonicNonNull InputStreamInfo currentInputStreamInfo; + /** * The input stream that is {@linkplain #registerInputStream registered}, but the pipeline has not * adapted to processing it. @@ -515,7 +517,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { if (pendingInputStreamInfo != null) { InputStreamInfo pendingInputStreamInfo = this.pendingInputStreamInfo; videoFrameProcessingTaskExecutor.submit( - () -> configureEffects(pendingInputStreamInfo, /* forceReconfigure= */ false)); + () -> configure(pendingInputStreamInfo, /* forceReconfigure= */ false)); this.pendingInputStreamInfo = null; } } @@ -659,7 +661,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { registeredFirstInputStream = true; inputStreamRegisteredCondition.close(); videoFrameProcessingTaskExecutor.submit( - () -> configureEffects(pendingInputStreamInfo, /* forceReconfigure= */ true)); + () -> configure(pendingInputStreamInfo, /* forceReconfigure= */ true)); } else { // Rejects further inputs after signaling EOS and before the next input stream is fully // configured. @@ -988,16 +990,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { } /** - * Configures the {@link GlShaderProgram} instances for {@code effects}. + * Configures for a new input stream. * - *

The pipeline will only re-configure if the {@link InputStreamInfo#effects new effects} - * doesn't match the {@link #activeEffects}, or when {@code forceReconfigure} is set to {@code - * true}. + *

The effect pipeline will only re-configure if the {@link InputStreamInfo#effects new + * effects} don't match the {@link #activeEffects}, or when {@code forceReconfigure} is set to + * {@code true}. */ - private void configureEffects(InputStreamInfo inputStreamInfo, boolean forceReconfigure) + private void configure(InputStreamInfo inputStreamInfo, boolean forceReconfigure) throws VideoFrameProcessingException { checkColors( /* inputColorInfo= */ checkNotNull(inputStreamInfo.format.colorInfo), outputColorInfo); + if (forceReconfigure || !activeEffects.equals(inputStreamInfo.effects)) { if (!intermediateGlShaderPrograms.isEmpty()) { for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) { @@ -1035,10 +1038,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { onInputSurfaceReadyListener = null; } } + listenerExecutor.execute( () -> listener.onInputStreamRegistered( inputStreamInfo.inputType, inputStreamInfo.format, inputStreamInfo.effects)); + if (currentInputStreamInfo == null + || inputStreamInfo.format.frameRate != currentInputStreamInfo.format.frameRate) { + listenerExecutor.execute( + () -> listener.onOutputFrameRateChanged(inputStreamInfo.format.frameRate)); + } + this.currentInputStreamInfo = inputStreamInfo; } /** Checks that color configuration is valid for {@link DefaultVideoFrameProcessor}. */ diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index a97087522c..f76620355d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -176,6 +176,11 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { listenerExecutor.execute(() -> listener.onOutputSizeChanged(width, height)); } + @Override + public void onOutputFrameRateChanged(float frameRate) { + listenerExecutor.execute(() -> listener.onOutputFrameRateChanged(frameRate)); + } + @Override public void onOutputFrameAvailableForRendering(long presentationTimeUs) { if (presentationTimeUs == 0) { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index f3a80cf25a..397e2248d4 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -114,6 +114,11 @@ public abstract class SingleInputVideoGraph implements VideoGraph { listenerExecutor.execute(() -> listener.onOutputSizeChanged(width, height)); } + @Override + public void onOutputFrameRateChanged(float frameRate) { + listenerExecutor.execute(() -> listener.onOutputFrameRateChanged(frameRate)); + } + @Override public void onOutputFrameAvailableForRendering(long presentationTimeUs) { if (isEnded) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index f99eb74a97..7d5bf120ca 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -339,6 +339,11 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video defaultVideoSink.onInputStreamChanged(INPUT_TYPE_SURFACE, format); } + @Override + public void onOutputFrameRateChanged(float frameRate) { + videoFrameReleaseControl.setFrameRate(frameRate); + } + @Override public void onOutputFrameAvailableForRendering(long framePresentationTimeUs) { if (pendingFlushCount > 0) { @@ -596,7 +601,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video default: throw new UnsupportedOperationException("Unsupported input type " + inputType); } - videoFrameReleaseControl.setFrameRate(format.frameRate); this.inputType = inputType; this.inputFormat = format;