loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rollback of 438ae0ed6a

Browse Source 
*** Original commit ***

ExoPlayer: Add setVideoFrameProcessorFactory().

This allows apps to use a custom VideoFrameProcessor implementation for video
playback. This may be useful, for example, when outputting to a texture.
***

PiperOrigin-RevId: 536391597
This commit is contained in:
huangdarwin 2023-05-30 14:21:35 +00:00 committed by Tofunmi Adigun-Hameed
parent ebbbcf9ac9
commit 06908e1a86
7 changed files with 60 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -49,8 +49,6 @@ This release includes the following changes since
* ExoPlayer: * ExoPlayer:
* Add `FilteringMediaSource` that allows to filter available track types * Add `FilteringMediaSource` that allows to filter available track types
from a `MediaSource`. from a `MediaSource`.
* Add `ExoPlayer.setVideoFrameProcessorFactory()` for using `Effect` with
a custom `VideoFrameProcessor.Factory` during video playback.
* Extractors: * Extractors:
* Ogg: Fix bug when seeking in files with a long duration * Ogg: Fix bug when seeking in files with a long duration
([#391](https://github.com/androidx/media/issues/391)). ([#391](https://github.com/androidx/media/issues/391)).

14
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java
View File

@ -45,7 +45,6 @@ import androidx.media3.common.PriorityTaskManager;
import androidx.media3.common.Timeline; import androidx.media3.common.Timeline;
import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TrackSelectionParameters;
import androidx.media3.common.Tracks; import androidx.media3.common.Tracks;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoSize; import androidx.media3.common.VideoSize;
import androidx.media3.common.text.CueGroup; import androidx.media3.common.text.CueGroup;
import androidx.media3.common.util.Clock; import androidx.media3.common.util.Clock;
@ -1563,19 +1562,6 @@ public interface ExoPlayer extends Player {
@UnstableApi @UnstableApi
void setVideoEffects(List<Effect> videoEffects); void setVideoEffects(List<Effect> videoEffects);
/**
* Sets a {@link VideoFrameProcessor.Factory} to create the {@link VideoFrameProcessor} that
* applies video effects set in {@link #setVideoEffects}.
*
* <p>See {@link #setVideoEffects} for limitations.
*
* @param videoFrameProcessorFactory The {@link VideoFrameProcessor.Factory} to use to apply the
* video effects.
*/
@RequiresApi(18)
@UnstableApi
void setVideoFrameProcessorFactory(VideoFrameProcessor.Factory videoFrameProcessorFactory);
/** /**
* Sets the {@link C.VideoScalingMode}. * Sets the {@link C.VideoScalingMode}.
* *

10
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java
View File

@ -32,7 +32,6 @@ import static androidx.media3.exoplayer.Renderer.MSG_SET_SCALING_MODE;
import static androidx.media3.exoplayer.Renderer.MSG_SET_SKIP_SILENCE_ENABLED; import static androidx.media3.exoplayer.Renderer.MSG_SET_SKIP_SILENCE_ENABLED;
import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_EFFECTS; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_EFFECTS;
import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_METADATA_LISTENER; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_METADATA_LISTENER;
import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY;
import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT;
import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT_RESOLUTION; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT_RESOLUTION;
import static androidx.media3.exoplayer.Renderer.MSG_SET_VOLUME; import static androidx.media3.exoplayer.Renderer.MSG_SET_VOLUME;
@ -78,7 +77,6 @@ import androidx.media3.common.Timeline;
import androidx.media3.common.TrackGroup; import androidx.media3.common.TrackGroup;
import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TrackSelectionParameters;
import androidx.media3.common.Tracks; import androidx.media3.common.Tracks;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoSize; import androidx.media3.common.VideoSize;
import androidx.media3.common.text.Cue; import androidx.media3.common.text.Cue;
import androidx.media3.common.text.CueGroup; import androidx.media3.common.text.CueGroup;
@ -1265,14 +1263,6 @@ import java.util.concurrent.TimeoutException;
sendRendererMessage(TRACK_TYPE_VIDEO, MSG_SET_VIDEO_EFFECTS, videoEffects); sendRendererMessage(TRACK_TYPE_VIDEO, MSG_SET_VIDEO_EFFECTS, videoEffects);
} }
@Override
public void setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
verifyApplicationThread();
sendRendererMessage(
TRACK_TYPE_VIDEO, MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY, videoFrameProcessorFactory);
}
@Override @Override
public void setVideoScalingMode(@C.VideoScalingMode int videoScalingMode) { public void setVideoScalingMode(@C.VideoScalingMode int videoScalingMode) {
verifyApplicationThread(); verifyApplicationThread();

15
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java
View File

@ -27,7 +27,6 @@ import androidx.media3.common.C;
import androidx.media3.common.Effect; import androidx.media3.common.Effect;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.Player; import androidx.media3.common.Player;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.Size; import androidx.media3.common.util.Size;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
@ -91,9 +90,9 @@ public interface Renderer extends PlayerMessage.Target {
* #MSG_SET_SCALING_MODE}, {@link #MSG_SET_CHANGE_FRAME_RATE_STRATEGY}, {@link * #MSG_SET_SCALING_MODE}, {@link #MSG_SET_CHANGE_FRAME_RATE_STRATEGY}, {@link
* #MSG_SET_AUX_EFFECT_INFO}, {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER}, {@link * #MSG_SET_AUX_EFFECT_INFO}, {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER}, {@link
* #MSG_SET_CAMERA_MOTION_LISTENER}, {@link #MSG_SET_SKIP_SILENCE_ENABLED}, {@link * #MSG_SET_CAMERA_MOTION_LISTENER}, {@link #MSG_SET_SKIP_SILENCE_ENABLED}, {@link
* #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS}, * #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS} or
* {@link #MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY} or {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May * {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May also be an app-defined value (see {@link
* also be an app-defined value (see {@link #MSG_CUSTOM_BASE}). * #MSG_CUSTOM_BASE}).
*/ */
@Documented @Documented
@Retention(RetentionPolicy.SOURCE) @Retention(RetentionPolicy.SOURCE)
@ -113,7 +112,6 @@ public interface Renderer extends PlayerMessage.Target {
MSG_SET_AUDIO_SESSION_ID, MSG_SET_AUDIO_SESSION_ID,
MSG_SET_WAKEUP_LISTENER, MSG_SET_WAKEUP_LISTENER,
MSG_SET_VIDEO_EFFECTS, MSG_SET_VIDEO_EFFECTS,
MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY,
MSG_SET_VIDEO_OUTPUT_RESOLUTION MSG_SET_VIDEO_OUTPUT_RESOLUTION
}) })
public @interface MessageType {} public @interface MessageType {}
@ -221,17 +219,12 @@ public interface Renderer extends PlayerMessage.Target {
* {@link List} containing {@linkplain Effect video effects}. * {@link List} containing {@linkplain Effect video effects}.
*/ */
int MSG_SET_VIDEO_EFFECTS = 13; int MSG_SET_VIDEO_EFFECTS = 13;
/**
* The type of a message that can be passed to a video renderer. The message payload should be a
* {@link VideoFrameProcessor.Factory}.
*/
int MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY = 14;
/** /**
* The type of a message that can be passed to a video renderer to set the desired output * The type of a message that can be passed to a video renderer to set the desired output
* resolution. The message payload should be a {@link Size} of the desired output width and * resolution. The message payload should be a {@link Size} of the desired output width and
* height. Use this method only when playing with video {@linkplain Effect effects}. * height. Use this method only when playing with video {@linkplain Effect effects}.
*/ */
int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 15; int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 14;
/** /**
* Applications or extensions may define custom {@code MSG_*} constants that can be passed to * Applications or extensions may define custom {@code MSG_*} constants that can be passed to
* renderers. These custom constants must be greater than or equal to this value. * renderers. These custom constants must be greater than or equal to this value.

8
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java
View File

@ -43,7 +43,6 @@ import androidx.media3.common.PriorityTaskManager;
import androidx.media3.common.Timeline; import androidx.media3.common.Timeline;
import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TrackSelectionParameters;
import androidx.media3.common.Tracks; import androidx.media3.common.Tracks;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoSize; import androidx.media3.common.VideoSize;
import androidx.media3.common.text.CueGroup; import androidx.media3.common.text.CueGroup;
import androidx.media3.common.util.Clock; import androidx.media3.common.util.Clock;
@ -672,13 +671,6 @@ public class SimpleExoPlayer extends BasePlayer
player.setVideoEffects(videoEffects); player.setVideoEffects(videoEffects);
} }
@Override
public void setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
blockUntilConstructorFinished();
player.setVideoFrameProcessorFactory(videoFrameProcessorFactory);
}
@Override @Override
public void setSkipSilenceEnabled(boolean skipSilenceEnabled) { public void setSkipSilenceEnabled(boolean skipSilenceEnabled) {
blockUntilConstructorFinished(); blockUntilConstructorFinished();

127
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -687,11 +687,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
List<Effect> videoEffects = (List<Effect>) checkNotNull(message); List<Effect> videoEffects = (List<Effect>) checkNotNull(message);
videoFrameProcessorManager.setVideoEffects(videoEffects); videoFrameProcessorManager.setVideoEffects(videoEffects);
break; break;
case MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY:
VideoFrameProcessor.Factory videoFrameProcessorFactory =
(VideoFrameProcessor.Factory) checkNotNull(message);
videoFrameProcessorManager.setVideoFrameProcessorFactory(videoFrameProcessorFactory);
break;
case MSG_SET_VIDEO_OUTPUT_RESOLUTION: case MSG_SET_VIDEO_OUTPUT_RESOLUTION:
Size outputResolution = (Size) checkNotNull(message); Size outputResolution = (Size) checkNotNull(message);
if (outputResolution.getWidth() != 0 if (outputResolution.getWidth() != 0
@ -1881,7 +1876,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private final ArrayDeque<Pair<Long, Format>> pendingFrameFormats; private final ArrayDeque<Pair<Long, Format>> pendingFrameFormats;
private @MonotonicNonNull Handler handler; private @MonotonicNonNull Handler handler;
private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory;
@Nullable private VideoFrameProcessor videoFrameProcessor; @Nullable private VideoFrameProcessor videoFrameProcessor;
@Nullable private CopyOnWriteArrayList<Effect> videoEffects; @Nullable private CopyOnWriteArrayList<Effect> videoEffects;
@Nullable private Format inputFormat; @Nullable private Format inputFormat;
@ -1943,12 +1937,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
this.videoEffects.addAll(videoEffects); this.videoEffects.addAll(videoEffects);
} }
/** Sets the {@link VideoFrameProcessor.Factory}. */
public void setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
}
/** Returns whether video frame processing is enabled. */ /** Returns whether video frame processing is enabled. */
public boolean isEnabled() { public boolean isEnabled() {
return videoFrameProcessor != null; return videoFrameProcessor != null;
@ -2021,69 +2009,66 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
VideoFrameProcessorAccessor.createRotationEffect(inputFormat.rotationDegrees)); VideoFrameProcessorAccessor.createRotationEffect(inputFormat.rotationDegrees));
} }
videoFrameProcessorFactory =
videoFrameProcessorFactory == null
? VideoFrameProcessorAccessor.getFrameProcessorFactory()
: videoFrameProcessorFactory;
videoFrameProcessor = videoFrameProcessor =
videoFrameProcessorFactory.create( VideoFrameProcessorAccessor.getFrameProcessorFactory()
renderer.context, .create(
checkNotNull(videoEffects), renderer.context,
DebugViewProvider.NONE, checkNotNull(videoEffects),
inputAndOutputColorInfos.first, DebugViewProvider.NONE,
inputAndOutputColorInfos.second, inputAndOutputColorInfos.first,
/* renderFramesAutomatically= */ false, inputAndOutputColorInfos.second,
/* listenerExecutor= */ handler::post, /* renderFramesAutomatically= */ false,
new VideoFrameProcessor.Listener() { /* listenerExecutor= */ handler::post,
@Override new VideoFrameProcessor.Listener() {
public void onOutputSizeChanged(int width, int height) { @Override
@Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat; public void onOutputSizeChanged(int width, int height) {
checkStateNotNull(inputFormat); @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat;
// TODO(b/264889146): Handle Effect that changes output size based on pts. checkStateNotNull(inputFormat);
processedFrameSize = // TODO(b/264889146): Handle Effect that changes output size based on pts.
new VideoSize( processedFrameSize =
width, new VideoSize(
height, width,
// VideoFrameProcessor is configured to produce rotation free height,
// frames. // VideoFrameProcessor is configured to produce rotation free
/* unappliedRotationDegrees= */ 0, // frames.
// VideoFrameProcessor always outputs pixelWidthHeightRatio 1. /* unappliedRotationDegrees= */ 0,
/* pixelWidthHeightRatio= */ 1.f); // VideoFrameProcessor always outputs pixelWidthHeightRatio 1.
pendingOutputSizeChange = true; /* pixelWidthHeightRatio= */ 1.f);
} pendingOutputSizeChange = true;
}
@Override @Override
public void onOutputFrameAvailableForRendering(long presentationTimeUs) { public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
if (registeredLastFrame) { if (registeredLastFrame) {
checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
} }
processedFramesTimestampsUs.add(presentationTimeUs); processedFramesTimestampsUs.add(presentationTimeUs);
// TODO(b/257464707) Support extensively modified media. // TODO(b/257464707) Support extensively modified media.
if (registeredLastFrame if (registeredLastFrame
&& presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
processedLastFrame = true; processedLastFrame = true;
} }
if (pendingOutputSizeChange) { if (pendingOutputSizeChange) {
// Report the size change on releasing this frame. // Report the size change on releasing this frame.
pendingOutputSizeChange = false; pendingOutputSizeChange = false;
pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs; pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs;
} }
} }
@Override @Override
public void onError(VideoFrameProcessingException exception) { public void onError(VideoFrameProcessingException exception) {
renderer.setPendingPlaybackException( renderer.setPendingPlaybackException(
renderer.createRendererException( renderer.createRendererException(
exception, exception,
inputFormat, inputFormat,
PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED));
} }
@Override @Override
public void onEnded() { public void onEnded() {
throw new IllegalStateException(); throw new IllegalStateException();
} }
}); });
videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE); videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE);
this.initialStreamOffsetUs = initialStreamOffsetUs; this.initialStreamOffsetUs = initialStreamOffsetUs;
} catch (Exception e) { } catch (Exception e) {

7
libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java
View File

@ -24,7 +24,6 @@ import androidx.media3.common.Effect;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.Player; import androidx.media3.common.Player;
import androidx.media3.common.PriorityTaskManager; import androidx.media3.common.PriorityTaskManager;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.Clock; import androidx.media3.common.util.Clock;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
import androidx.media3.exoplayer.DecoderCounters; import androidx.media3.exoplayer.DecoderCounters;
@ -250,12 +249,6 @@ public class StubExoPlayer extends StubPlayer implements ExoPlayer {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override
public void setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
throw new UnsupportedOperationException();
}
@Override @Override
public void setVideoScalingMode(int videoScalingMode) { public void setVideoScalingMode(int videoScalingMode) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();