From 225d3367133bda35fdb83837653b34822d262e9c Mon Sep 17 00:00:00 2001 From: kimvde Date: Tue, 23 Jul 2024 01:38:41 -0700 Subject: [PATCH] Remove VideoSinkProvider parameter from renderers PiperOrigin-RevId: 655073481 --- .../video/CompositingVideoSinkProvider.java | 46 ++++------- .../video/MediaCodecVideoRenderer.java | 76 ++++++++++++------- .../exoplayer/video/VideoSinkProvider.java | 6 -- .../media3/transformer/CompositionPlayer.java | 2 +- .../SequencePlayerRenderersWrapper.java | 32 ++++---- 5 files changed, 80 insertions(+), 82 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 9ca0c39bd8..5b58a45f89 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -267,11 +267,6 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi // VideoSinkProvider methods - @Override - public VideoFrameReleaseControl getVideoFrameReleaseControl() { - return videoFrameReleaseControl; - } - @Override public VideoSink getSink() { return videoSinkImpl; @@ -347,31 +342,6 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi } } - // Other public methods - - /** - * Incrementally renders available video frames. - * - * @param positionUs The current playback position, in microseconds. - * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, - * taken approximately at the time the playback position was {@code positionUs}. - */ - public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { - if (pendingFlushCount == 0) { - videoFrameRenderControl.render(positionUs, elapsedRealtimeUs); - } - } - - /** - * Returns the output surface that was {@linkplain #setOutputSurfaceInfo(Surface, Size) set}, or - * {@code null} if no surface is set or the surface is {@linkplain #clearOutputSurfaceInfo() - * cleared}. - */ - @Nullable - public Surface getOutputSurface() { - return currentSurfaceAndSize != null ? currentSurfaceAndSize.first : null; - } - // Internal methods private VideoFrameProcessor initialize(Format sourceFormat) throws VideoSink.VideoSinkException { @@ -432,6 +402,19 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi return pendingFlushCount == 0 && videoFrameRenderControl.hasReleasedFrame(presentationTimeUs); } + /** + * Incrementally renders available video frames. + * + * @param positionUs The current playback position, in microseconds. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * taken approximately at the time the playback position was {@code positionUs}. + */ + private void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + if (pendingFlushCount == 0) { + videoFrameRenderControl.render(positionUs, elapsedRealtimeUs); + } + } + private void flush() { if (!isInitialized()) { return; @@ -864,8 +847,7 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi return; } - ArrayList effects = new ArrayList<>(); - effects.addAll(videoEffects); + ArrayList effects = new ArrayList<>(videoEffects); Format inputFormat = checkNotNull(this.inputFormat); checkStateNotNull(videoFrameProcessor) .registerInputStream( diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index f1ba1aab1d..f7dcd650e7 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -150,7 +150,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer private static boolean deviceNeedsSetOutputSurfaceWorkaround; private final Context context; - @Nullable private final VideoSinkProvider videoSinkProvider; private final boolean ownsVideoSink; private final EventDispatcher eventDispatcher; private final int maxDroppedFramesToNotify; @@ -356,7 +355,37 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer eventListener, maxDroppedFramesToNotify, assumedMinimumCodecOperatingRate, - /* videoSinkProvider= */ null); + /* videoSink= */ (VideoSink) null); + } + + /** + * @deprecated Use {@link #MediaCodecVideoRenderer(Context, MediaCodecAdapter.Factory, + * MediaCodecSelector, long, boolean, Handler, VideoRendererEventListener, int, float, + * VideoSink)} instead. + */ + @Deprecated + public MediaCodecVideoRenderer( + Context context, + MediaCodecAdapter.Factory codecAdapterFactory, + MediaCodecSelector mediaCodecSelector, + long allowedJoiningTimeMs, + boolean enableDecoderFallback, + @Nullable Handler eventHandler, + @Nullable VideoRendererEventListener eventListener, + int maxDroppedFramesToNotify, + float assumedMinimumCodecOperatingRate, + @Nullable VideoSinkProvider videoSinkProvider) { + this( + context, + codecAdapterFactory, + mediaCodecSelector, + allowedJoiningTimeMs, + enableDecoderFallback, + eventHandler, + eventListener, + maxDroppedFramesToNotify, + assumedMinimumCodecOperatingRate, + videoSinkProvider == null ? null : videoSinkProvider.getSink()); } /** @@ -379,11 +408,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer * @param assumedMinimumCodecOperatingRate A codec operating rate that all codecs instantiated by * this renderer are assumed to meet implicitly (i.e. without the operating rate being set * explicitly using {@link MediaFormat#KEY_OPERATING_RATE}). - * @param videoSinkProvider The {@link VideoSinkProvider} that will be used for applying video - * effects also providing the {@linkplain VideoSinkProvider#getVideoFrameReleaseControl() - * VideoFrameReleaseControl} for releasing video frames. If {@code null}, the {@link - * CompositingVideoSinkProvider} with its default configuration will be used, and the renderer - * will drive releasing of video frames by itself. + * @param videoSink The {@link VideoSink} consuming the frames. If {@code null} and effects are + * {@linkplain #MSG_SET_VIDEO_EFFECTS set}, a {@link VideoSink} produced by a {@link + * CompositingVideoSinkProvider} with its default configuration will be used to apply effects + * and render the frames on the output. */ public MediaCodecVideoRenderer( Context context, @@ -395,7 +423,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @Nullable VideoRendererEventListener eventListener, int maxDroppedFramesToNotify, float assumedMinimumCodecOperatingRate, - @Nullable VideoSinkProvider videoSinkProvider) { + @Nullable VideoSink videoSink) { super( C.TRACK_TYPE_VIDEO, codecAdapterFactory, @@ -404,18 +432,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer assumedMinimumCodecOperatingRate); this.context = context.getApplicationContext(); this.maxDroppedFramesToNotify = maxDroppedFramesToNotify; - this.videoSinkProvider = videoSinkProvider; + this.videoSink = videoSink; eventDispatcher = new EventDispatcher(eventHandler, eventListener); - ownsVideoSink = videoSinkProvider == null; - if (videoSinkProvider == null) { - @SuppressWarnings("nullness:assignment") - VideoFrameReleaseControl.@Initialized FrameTimingEvaluator thisRef = this; - videoFrameReleaseControl = - new VideoFrameReleaseControl( - this.context, /* frameTimingEvaluator= */ thisRef, allowedJoiningTimeMs); - } else { - videoFrameReleaseControl = videoSinkProvider.getVideoFrameReleaseControl(); - } + ownsVideoSink = videoSink == null; + @SuppressWarnings("nullness:assignment") + VideoFrameReleaseControl.@Initialized FrameTimingEvaluator thisRef = this; + videoFrameReleaseControl = + new VideoFrameReleaseControl( + this.context, /* frameTimingEvaluator= */ thisRef, allowedJoiningTimeMs); videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround(); outputResolution = Size.UNKNOWN; @@ -651,14 +675,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer // The video sink can only be enabled the first time the renderer is enabled, or after it has // been reset. if (!hasSetVideoSink) { - if ((videoEffects != null || !ownsVideoSink) && videoSink == null) { - VideoSinkProvider videoSinkProvider = - this.videoSinkProvider != null - ? this.videoSinkProvider - : new CompositingVideoSinkProvider.Builder(this.context, videoFrameReleaseControl) - .setClock(getClock()) - .build(); - videoSink = videoSinkProvider.getSink(); + if (videoEffects != null && videoSink == null) { + videoSink = + new CompositingVideoSinkProvider.Builder(context, videoFrameReleaseControl) + .setClock(getClock()) + .build() + .getSink(); } hasSetVideoSink = true; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java index 95f0004f4d..e06ca453b1 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java @@ -24,12 +24,6 @@ import androidx.media3.common.util.UnstableApi; @UnstableApi public interface VideoSinkProvider { - /** - * Returns the {@link VideoFrameReleaseControl} that will be used for releasing of video frames - * during rendering. - */ - VideoFrameReleaseControl getVideoFrameReleaseControl(); - /** Returns a {@link VideoSink} to forward video frames for processing. */ VideoSink getSink(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java index 064e022562..6b8df86dca 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java @@ -647,7 +647,7 @@ public final class CompositionPlayer extends SimpleBasePlayer context, editedMediaItemSequence, previewAudioPipeline, - compositingVideoSinkProvider, + compositingVideoSinkProvider.getSink(), imageDecoderFactory) : SequencePlayerRenderersWrapper.createForAudio( context, editedMediaItemSequence, previewAudioPipeline); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java index a3ccfb94a0..2c3b86df11 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java @@ -46,7 +46,6 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; import androidx.media3.exoplayer.metadata.MetadataOutput; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.text.TextOutput; -import androidx.media3.exoplayer.video.CompositingVideoSinkProvider; import androidx.media3.exoplayer.video.MediaCodecVideoRenderer; import androidx.media3.exoplayer.video.VideoRendererEventListener; import androidx.media3.exoplayer.video.VideoSink; @@ -64,7 +63,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Context context; private final EditedMediaItemSequence sequence; private final PreviewAudioPipeline previewAudioPipeline; - @Nullable private final CompositingVideoSinkProvider compositingVideoSinkProvider; + @Nullable private final VideoSink videoSink; @Nullable private final ImageDecoder.Factory imageDecoderFactory; /** Creates a renderers wrapper for a player that will play video, image and audio. */ @@ -72,10 +71,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Context context, EditedMediaItemSequence sequence, PreviewAudioPipeline previewAudioPipeline, - CompositingVideoSinkProvider compositingVideoSinkProvider, + VideoSink videoSink, ImageDecoder.Factory imageDecoderFactory) { return new SequencePlayerRenderersWrapper( - context, sequence, previewAudioPipeline, compositingVideoSinkProvider, imageDecoderFactory); + context, sequence, previewAudioPipeline, videoSink, imageDecoderFactory); } /** Creates a renderers wrapper that for a player that will only play audio. */ @@ -87,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; context, sequence, previewAudioPipeline, - /* compositingVideoSinkProvider= */ null, + /* videoSink= */ null, /* imageDecoderFactory= */ null); } @@ -95,12 +94,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Context context, EditedMediaItemSequence sequence, PreviewAudioPipeline previewAudioPipeline, - @Nullable CompositingVideoSinkProvider compositingVideoSinkProvider, + @Nullable VideoSink videoSink, @Nullable ImageDecoder.Factory imageDecoderFactory) { this.context = context; this.sequence = sequence; this.previewAudioPipeline = previewAudioPipeline; - this.compositingVideoSinkProvider = compositingVideoSinkProvider; + this.videoSink = videoSink; this.imageDecoderFactory = imageDecoderFactory; } @@ -120,7 +119,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; audioRendererEventListener, previewAudioPipeline.createInput())); - if (compositingVideoSinkProvider != null) { + if (videoSink != null) { renderers.add( new SequenceVideoRenderer( checkStateNotNull(context), @@ -252,10 +251,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; videoRendererEventListener, MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, /* assumedMinimumCodecOperatingRate= */ DEFAULT_FRAME_RATE, - checkStateNotNull(sequencePlayerRenderersWrapper.compositingVideoSinkProvider)); + checkStateNotNull(sequencePlayerRenderersWrapper.videoSink)); this.sequencePlayerRenderersWrapper = sequencePlayerRenderersWrapper; - videoSink = - checkStateNotNull(sequencePlayerRenderersWrapper.compositingVideoSinkProvider).getSink(); + videoSink = checkStateNotNull(sequencePlayerRenderersWrapper.videoSink); experimentalEnableProcessedStreamChangedAtStart(); } @@ -294,7 +292,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final class SequenceImageRenderer extends ImageRenderer { private final SequencePlayerRenderersWrapper sequencePlayerRenderersWrapper; - private final CompositingVideoSinkProvider compositingVideoSinkProvider; private final VideoSink videoSink; private ImmutableList videoEffects; @@ -311,9 +308,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; super( checkStateNotNull(sequencePlayerRenderersWrapper.imageDecoderFactory), ImageOutput.NO_OP); this.sequencePlayerRenderersWrapper = sequencePlayerRenderersWrapper; - compositingVideoSinkProvider = - checkStateNotNull(sequencePlayerRenderersWrapper.compositingVideoSinkProvider); - videoSink = compositingVideoSinkProvider.getSink(); + videoSink = checkStateNotNull(sequencePlayerRenderersWrapper.videoSink); videoEffects = ImmutableList.of(); streamStartPositionUs = C.TIME_UNSET; streamOffsetUs = C.TIME_UNSET; @@ -421,7 +416,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw exoPlaybackException; } super.render(positionUs, elapsedRealtimeUs); - compositingVideoSinkProvider.render(positionUs, elapsedRealtimeUs); + try { + videoSink.render(positionUs, elapsedRealtimeUs); + } catch (VideoSink.VideoSinkException e) { + throw createRendererException( + e, e.format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED); + } } @Override