From 4d4f61b772b4a9f5c51a67f0ec1dd17fa88443fb Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Tue, 30 May 2023 14:21:35 +0000 Subject: [PATCH] Rollback of https://github.com/androidx/media/commit/438ae0ed6ac1059e8aaa9380f476c1403f24a986 *** Original commit *** ExoPlayer: Add setVideoFrameProcessorFactory(). This allows apps to use a custom VideoFrameProcessor implementation for video playback. This may be useful, for example, when outputting to a texture. *** PiperOrigin-RevId: 536391597 (cherry picked from commit 06908e1a865c4c5ae9a9f52986255756bb20cd07) --- RELEASENOTES.md | 2 - .../androidx/media3/exoplayer/ExoPlayer.java | 14 -- .../media3/exoplayer/ExoPlayerImpl.java | 10 -- .../androidx/media3/exoplayer/Renderer.java | 15 +-- .../media3/exoplayer/SimpleExoPlayer.java | 8 -- .../video/MediaCodecVideoRenderer.java | 127 ++++++++---------- .../media3/test/utils/StubExoPlayer.java | 7 - 7 files changed, 60 insertions(+), 123 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index e2b0e7138b..c3e750d259 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -14,8 +14,6 @@ This release includes the following changes since * ExoPlayer: * Add `FilteringMediaSource` that allows to filter available track types from a `MediaSource`. - * Add `ExoPlayer.setVideoFrameProcessorFactory()` for using `Effect` with - a custom `VideoFrameProcessor.Factory` during video playback. * Extractors: * Ogg: Fix bug when seeking in files with a long duration ([#391](https://github.com/androidx/media/issues/391)). diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java index b7cab5f4e8..4d3123287c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java @@ -44,7 +44,6 @@ import androidx.media3.common.Player; import androidx.media3.common.PriorityTaskManager; import androidx.media3.common.Timeline; import androidx.media3.common.Tracks; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; import androidx.media3.common.text.CueGroup; import androidx.media3.common.util.Clock; @@ -1563,19 +1562,6 @@ public interface ExoPlayer extends Player { @UnstableApi void setVideoEffects(List videoEffects); - /** - * Sets a {@link VideoFrameProcessor.Factory} to create the {@link VideoFrameProcessor} that - * applies video effects set in {@link #setVideoEffects}. - * - *

See {@link #setVideoEffects} for limitations. - * - * @param videoFrameProcessorFactory The {@link VideoFrameProcessor.Factory} to use to apply the - * video effects. - */ - @RequiresApi(18) - @UnstableApi - void setVideoFrameProcessorFactory(VideoFrameProcessor.Factory videoFrameProcessorFactory); - /** * Sets the {@link C.VideoScalingMode}. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java index 24bfabb985..a4c28f6922 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java @@ -32,7 +32,6 @@ import static androidx.media3.exoplayer.Renderer.MSG_SET_SCALING_MODE; import static androidx.media3.exoplayer.Renderer.MSG_SET_SKIP_SILENCE_ENABLED; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_EFFECTS; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_METADATA_LISTENER; -import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT_RESOLUTION; import static androidx.media3.exoplayer.Renderer.MSG_SET_VOLUME; @@ -78,7 +77,6 @@ import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.Tracks; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; import androidx.media3.common.text.Cue; import androidx.media3.common.text.CueGroup; @@ -1265,14 +1263,6 @@ import java.util.concurrent.TimeoutException; sendRendererMessage(TRACK_TYPE_VIDEO, MSG_SET_VIDEO_EFFECTS, videoEffects); } - @Override - public void setVideoFrameProcessorFactory( - VideoFrameProcessor.Factory videoFrameProcessorFactory) { - verifyApplicationThread(); - sendRendererMessage( - TRACK_TYPE_VIDEO, MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY, videoFrameProcessorFactory); - } - @Override public void setVideoScalingMode(@C.VideoScalingMode int videoScalingMode) { verifyApplicationThread(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java index 28e58f282b..739337df6d 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java @@ -27,7 +27,6 @@ import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.Player; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -91,9 +90,9 @@ public interface Renderer extends PlayerMessage.Target { * #MSG_SET_SCALING_MODE}, {@link #MSG_SET_CHANGE_FRAME_RATE_STRATEGY}, {@link * #MSG_SET_AUX_EFFECT_INFO}, {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER}, {@link * #MSG_SET_CAMERA_MOTION_LISTENER}, {@link #MSG_SET_SKIP_SILENCE_ENABLED}, {@link - * #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS}, - * {@link #MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY} or {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May - * also be an app-defined value (see {@link #MSG_CUSTOM_BASE}). + * #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS} or + * {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May also be an app-defined value (see {@link + * #MSG_CUSTOM_BASE}). */ @Documented @Retention(RetentionPolicy.SOURCE) @@ -113,7 +112,6 @@ public interface Renderer extends PlayerMessage.Target { MSG_SET_AUDIO_SESSION_ID, MSG_SET_WAKEUP_LISTENER, MSG_SET_VIDEO_EFFECTS, - MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY, MSG_SET_VIDEO_OUTPUT_RESOLUTION }) public @interface MessageType {} @@ -221,17 +219,12 @@ public interface Renderer extends PlayerMessage.Target { * {@link List} containing {@linkplain Effect video effects}. */ int MSG_SET_VIDEO_EFFECTS = 13; - /** - * The type of a message that can be passed to a video renderer. The message payload should be a - * {@link VideoFrameProcessor.Factory}. - */ - int MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY = 14; /** * The type of a message that can be passed to a video renderer to set the desired output * resolution. The message payload should be a {@link Size} of the desired output width and * height. Use this method only when playing with video {@linkplain Effect effects}. */ - int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 15; + int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 14; /** * Applications or extensions may define custom {@code MSG_*} constants that can be passed to * renderers. These custom constants must be greater than or equal to this value. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java index 045747f9b5..80a13ae1d3 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java @@ -43,7 +43,6 @@ import androidx.media3.common.PriorityTaskManager; import androidx.media3.common.Timeline; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.Tracks; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; import androidx.media3.common.text.CueGroup; import androidx.media3.common.util.Clock; @@ -672,13 +671,6 @@ public class SimpleExoPlayer extends BasePlayer player.setVideoEffects(videoEffects); } - @Override - public void setVideoFrameProcessorFactory( - VideoFrameProcessor.Factory videoFrameProcessorFactory) { - blockUntilConstructorFinished(); - player.setVideoFrameProcessorFactory(videoFrameProcessorFactory); - } - @Override public void setSkipSilenceEnabled(boolean skipSilenceEnabled) { blockUntilConstructorFinished(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 7f72e8cf11..3d848899ce 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -687,11 +687,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { List videoEffects = (List) checkNotNull(message); videoFrameProcessorManager.setVideoEffects(videoEffects); break; - case MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY: - VideoFrameProcessor.Factory videoFrameProcessorFactory = - (VideoFrameProcessor.Factory) checkNotNull(message); - videoFrameProcessorManager.setVideoFrameProcessorFactory(videoFrameProcessorFactory); - break; case MSG_SET_VIDEO_OUTPUT_RESOLUTION: Size outputResolution = (Size) checkNotNull(message); if (outputResolution.getWidth() != 0 @@ -1881,7 +1876,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private final ArrayDeque> pendingFrameFormats; private @MonotonicNonNull Handler handler; - private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory; @Nullable private VideoFrameProcessor videoFrameProcessor; @Nullable private CopyOnWriteArrayList videoEffects; @Nullable private Format inputFormat; @@ -1943,12 +1937,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { this.videoEffects.addAll(videoEffects); } - /** Sets the {@link VideoFrameProcessor.Factory}. */ - public void setVideoFrameProcessorFactory( - VideoFrameProcessor.Factory videoFrameProcessorFactory) { - this.videoFrameProcessorFactory = videoFrameProcessorFactory; - } - /** Returns whether video frame processing is enabled. */ public boolean isEnabled() { return videoFrameProcessor != null; @@ -2021,69 +2009,66 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { VideoFrameProcessorAccessor.createRotationEffect(inputFormat.rotationDegrees)); } - videoFrameProcessorFactory = - videoFrameProcessorFactory == null - ? VideoFrameProcessorAccessor.getFrameProcessorFactory() - : videoFrameProcessorFactory; videoFrameProcessor = - videoFrameProcessorFactory.create( - renderer.context, - checkNotNull(videoEffects), - DebugViewProvider.NONE, - inputAndOutputColorInfos.first, - inputAndOutputColorInfos.second, - /* renderFramesAutomatically= */ false, - /* listenerExecutor= */ handler::post, - new VideoFrameProcessor.Listener() { - @Override - public void onOutputSizeChanged(int width, int height) { - @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat; - checkStateNotNull(inputFormat); - // TODO(b/264889146): Handle Effect that changes output size based on pts. - processedFrameSize = - new VideoSize( - width, - height, - // VideoFrameProcessor is configured to produce rotation free - // frames. - /* unappliedRotationDegrees= */ 0, - // VideoFrameProcessor always outputs pixelWidthHeightRatio 1. - /* pixelWidthHeightRatio= */ 1.f); - pendingOutputSizeChange = true; - } + VideoFrameProcessorAccessor.getFrameProcessorFactory() + .create( + renderer.context, + checkNotNull(videoEffects), + DebugViewProvider.NONE, + inputAndOutputColorInfos.first, + inputAndOutputColorInfos.second, + /* renderFramesAutomatically= */ false, + /* listenerExecutor= */ handler::post, + new VideoFrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat; + checkStateNotNull(inputFormat); + // TODO(b/264889146): Handle Effect that changes output size based on pts. + processedFrameSize = + new VideoSize( + width, + height, + // VideoFrameProcessor is configured to produce rotation free + // frames. + /* unappliedRotationDegrees= */ 0, + // VideoFrameProcessor always outputs pixelWidthHeightRatio 1. + /* pixelWidthHeightRatio= */ 1.f); + pendingOutputSizeChange = true; + } - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { - if (registeredLastFrame) { - checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); - } - processedFramesTimestampsUs.add(presentationTimeUs); - // TODO(b/257464707) Support extensively modified media. - if (registeredLastFrame - && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { - processedLastFrame = true; - } - if (pendingOutputSizeChange) { - // Report the size change on releasing this frame. - pendingOutputSizeChange = false; - pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs; - } - } + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + if (registeredLastFrame) { + checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); + } + processedFramesTimestampsUs.add(presentationTimeUs); + // TODO(b/257464707) Support extensively modified media. + if (registeredLastFrame + && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { + processedLastFrame = true; + } + if (pendingOutputSizeChange) { + // Report the size change on releasing this frame. + pendingOutputSizeChange = false; + pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs; + } + } - @Override - public void onError(VideoFrameProcessingException exception) { - renderer.setPendingPlaybackException( - renderer.createRendererException( - exception, - inputFormat, - PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); - } + @Override + public void onError(VideoFrameProcessingException exception) { + renderer.setPendingPlaybackException( + renderer.createRendererException( + exception, + inputFormat, + PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); + } - @Override - public void onEnded() { - throw new IllegalStateException(); - } - }); + @Override + public void onEnded() { + throw new IllegalStateException(); + } + }); videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE); this.initialStreamOffsetUs = initialStreamOffsetUs; } catch (Exception e) { diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java index 35c0cedd5c..a49e03c2fb 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java @@ -24,7 +24,6 @@ import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.Player; import androidx.media3.common.PriorityTaskManager; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.Clock; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.DecoderCounters; @@ -250,12 +249,6 @@ public class StubExoPlayer extends StubPlayer implements ExoPlayer { throw new UnsupportedOperationException(); } - @Override - public void setVideoFrameProcessorFactory( - VideoFrameProcessor.Factory videoFrameProcessorFactory) { - throw new UnsupportedOperationException(); - } - @Override public void setVideoScalingMode(int videoScalingMode) { throw new UnsupportedOperationException();