mirror of
https://github.com/androidx/media.git
synced 2025-04-30 06:46:50 +08:00
DefaultVideoSink: implement handleInputFrame() and listeners
With this CL, PlaybackVideoGraphWrapper doesn't call the VideoFrameRenderControl directly anymore (which is one of the goals of DefaultVideoSink). PiperOrigin-RevId: 702673821
This commit is contained in:
kimvde
Copybara-Service
parent
c770a6ab6f
commit
06b94f5448
@ -23,12 +23,18 @@ import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.Effect;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.MimeTypes;
|
||||
import androidx.media3.common.VideoSize;
|
||||
import androidx.media3.common.util.Clock;
|
||||
import androidx.media3.common.util.Size;
|
||||
import androidx.media3.common.util.TimestampIterator;
|
||||
import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
import androidx.media3.exoplayer.Renderer;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.List;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.Executor;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
* The default {@link VideoSink} implementation. This implementation renders video frames to an
|
||||
@ -39,7 +45,7 @@ import java.util.concurrent.Executor;
|
||||
* <ul>
|
||||
* <li>Applying video effects
|
||||
* <li>Inputting bitmaps
|
||||
* <li>Setting WakeupListener
|
||||
* <li>Setting a WakeupListener
|
||||
* </ul>
|
||||
*
|
||||
* <p>The {@linkplain #getInputSurface() input} and {@linkplain #setOutputSurfaceInfo(Surface, Size)
|
||||
@ -48,19 +54,30 @@ import java.util.concurrent.Executor;
|
||||
/* package */ final class DefaultVideoSink implements VideoSink {
|
||||
|
||||
private final VideoFrameReleaseControl videoFrameReleaseControl;
|
||||
private final Clock clock;
|
||||
private final VideoFrameRenderControl videoFrameRenderControl;
|
||||
private final Queue<VideoFrameHandler> videoFrameHandlers;
|
||||
|
||||
@Nullable private Surface outputSurface;
|
||||
private Format inputFormat;
|
||||
private long streamStartPositionUs;
|
||||
private long bufferTimestampAdjustmentUs;
|
||||
private Listener listener;
|
||||
private Executor listenerExecutor;
|
||||
private VideoFrameMetadataListener videoFrameMetadataListener;
|
||||
|
||||
public DefaultVideoSink(
|
||||
VideoFrameReleaseControl videoFrameReleaseControl,
|
||||
VideoFrameRenderControl videoFrameRenderControl) {
|
||||
public DefaultVideoSink(VideoFrameReleaseControl videoFrameReleaseControl, Clock clock) {
|
||||
this.videoFrameReleaseControl = videoFrameReleaseControl;
|
||||
this.videoFrameRenderControl = videoFrameRenderControl;
|
||||
videoFrameReleaseControl.setClock(clock);
|
||||
this.clock = clock;
|
||||
videoFrameRenderControl =
|
||||
new VideoFrameRenderControl(new FrameRendererImpl(), videoFrameReleaseControl);
|
||||
videoFrameHandlers = new ArrayDeque<>();
|
||||
inputFormat = new Format.Builder().build();
|
||||
streamStartPositionUs = C.TIME_UNSET;
|
||||
listener = Listener.NO_OP;
|
||||
listenerExecutor = runnable -> {};
|
||||
videoFrameMetadataListener = (presentationTimeUs, releaseTimeNs, format, mediaFormat) -> {};
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -85,7 +102,8 @@ import java.util.concurrent.Executor;
|
||||
|
||||
@Override
|
||||
public void setListener(Listener listener, Executor executor) {
|
||||
throw new UnsupportedOperationException();
|
||||
this.listener = listener;
|
||||
this.listenerExecutor = executor;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -104,6 +122,7 @@ import java.util.concurrent.Executor;
|
||||
videoFrameReleaseControl.reset();
|
||||
}
|
||||
videoFrameRenderControl.flush();
|
||||
videoFrameHandlers.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -133,7 +152,7 @@ import java.util.concurrent.Executor;
|
||||
|
||||
@Override
|
||||
public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener) {
|
||||
throw new UnsupportedOperationException();
|
||||
this.videoFrameMetadataListener = videoFrameMetadataListener;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -168,6 +187,7 @@ import java.util.concurrent.Executor;
|
||||
videoFrameRenderControl.onStreamStartPositionChanged(streamStartPositionUs);
|
||||
this.streamStartPositionUs = streamStartPositionUs;
|
||||
}
|
||||
this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -206,7 +226,10 @@ import java.util.concurrent.Executor;
|
||||
@Override
|
||||
public boolean handleInputFrame(
|
||||
long framePresentationTimeUs, boolean isLastFrame, VideoFrameHandler videoFrameHandler) {
|
||||
throw new UnsupportedOperationException();
|
||||
videoFrameHandlers.add(videoFrameHandler);
|
||||
long bufferPresentationTimeUs = framePresentationTimeUs - bufferTimestampAdjustmentUs;
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(bufferPresentationTimeUs);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -245,4 +268,43 @@ import java.util.concurrent.Executor;
|
||||
|
||||
@Override
|
||||
public void release() {}
|
||||
|
||||
private final class FrameRendererImpl implements VideoFrameRenderControl.FrameRenderer {
|
||||
|
||||
private @MonotonicNonNull Format outputFormat;
|
||||
|
||||
@Override
|
||||
public void onVideoSizeChanged(VideoSize videoSize) {
|
||||
outputFormat =
|
||||
new Format.Builder()
|
||||
.setWidth(videoSize.width)
|
||||
.setHeight(videoSize.height)
|
||||
.setSampleMimeType(MimeTypes.VIDEO_RAW)
|
||||
.build();
|
||||
listenerExecutor.execute(() -> listener.onVideoSizeChanged(DefaultVideoSink.this, videoSize));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void renderFrame(
|
||||
long renderTimeNs, long bufferPresentationTimeUs, boolean isFirstFrame) {
|
||||
if (isFirstFrame && outputSurface != null) {
|
||||
listenerExecutor.execute(() -> listener.onFirstFrameRendered(DefaultVideoSink.this));
|
||||
}
|
||||
// TODO - b/292111083: outputFormat is initialized after the first frame is rendered because
|
||||
// onVideoSizeChanged is announced after the first frame is available for rendering.
|
||||
Format format = outputFormat == null ? new Format.Builder().build() : outputFormat;
|
||||
videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
|
||||
/* presentationTimeUs= */ bufferPresentationTimeUs,
|
||||
/* releaseTimeNs= */ clock.nanoTime(),
|
||||
format,
|
||||
/* mediaFormat= */ null);
|
||||
videoFrameHandlers.remove().render(renderTimeNs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropFrame() {
|
||||
listenerExecutor.execute(() -> listener.onFrameDropped(DefaultVideoSink.this));
|
||||
videoFrameHandlers.remove().skip();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
*/
|
||||
package androidx.media3.exoplayer.video;
|
||||
|
||||
import static androidx.media3.common.VideoFrameProcessor.DROP_OUTPUT_FRAME;
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
@ -36,7 +37,6 @@ import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.DebugViewProvider;
|
||||
import androidx.media3.common.Effect;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.MimeTypes;
|
||||
import androidx.media3.common.PreviewingVideoGraph;
|
||||
import androidx.media3.common.SurfaceInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
@ -235,15 +235,14 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
*/
|
||||
private final TimedValueQueue<Long> streamStartPositionsUs;
|
||||
|
||||
private final VideoFrameRenderControl videoFrameRenderControl;
|
||||
private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
|
||||
private final List<Effect> compositionEffects;
|
||||
private final VideoSink defaultVideoSink;
|
||||
private final VideoSink.VideoFrameHandler videoFrameHandler;
|
||||
private final Clock clock;
|
||||
private final CopyOnWriteArraySet<PlaybackVideoGraphWrapper.Listener> listeners;
|
||||
|
||||
private Format videoGraphOutputFormat;
|
||||
private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener;
|
||||
private @MonotonicNonNull HandlerWrapper handler;
|
||||
private @MonotonicNonNull PreviewingVideoGraph videoGraph;
|
||||
private long outputStreamStartPositionUs;
|
||||
@ -268,18 +267,26 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
context = builder.context;
|
||||
inputVideoSink = new InputVideoSink(context);
|
||||
streamStartPositionsUs = new TimedValueQueue<>();
|
||||
clock = builder.clock;
|
||||
VideoFrameReleaseControl videoFrameReleaseControl = builder.videoFrameReleaseControl;
|
||||
videoFrameReleaseControl.setClock(clock);
|
||||
videoFrameRenderControl =
|
||||
new VideoFrameRenderControl(new FrameRendererImpl(), videoFrameReleaseControl);
|
||||
previewingVideoGraphFactory = checkStateNotNull(builder.previewingVideoGraphFactory);
|
||||
compositionEffects = builder.compositionEffects;
|
||||
defaultVideoSink = new DefaultVideoSink(videoFrameReleaseControl, videoFrameRenderControl);
|
||||
clock = builder.clock;
|
||||
defaultVideoSink = new DefaultVideoSink(builder.videoFrameReleaseControl, clock);
|
||||
videoFrameHandler =
|
||||
new VideoSink.VideoFrameHandler() {
|
||||
@Override
|
||||
public void render(long renderTimestampNs) {
|
||||
checkStateNotNull(videoGraph).renderOutputFrame(renderTimestampNs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void skip() {
|
||||
checkStateNotNull(videoGraph).renderOutputFrame(DROP_OUTPUT_FRAME);
|
||||
}
|
||||
};
|
||||
listeners = new CopyOnWriteArraySet<>();
|
||||
state = STATE_CREATED;
|
||||
listeners.add(inputVideoSink);
|
||||
videoGraphOutputFormat = new Format.Builder().build();
|
||||
addListener(inputVideoSink);
|
||||
state = STATE_CREATED;
|
||||
finalBufferPresentationTimeUs = C.TIME_UNSET;
|
||||
}
|
||||
|
||||
@ -334,11 +341,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
if (state == STATE_RELEASED) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (handler != null) {
|
||||
handler.removeCallbacksAndMessages(/* token= */ null);
|
||||
}
|
||||
|
||||
if (videoGraph != null) {
|
||||
videoGraph.release();
|
||||
}
|
||||
@ -380,12 +385,14 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
if (newOutputStreamStartPositionUs != null
|
||||
&& newOutputStreamStartPositionUs != outputStreamStartPositionUs) {
|
||||
defaultVideoSink.setStreamTimestampInfo(
|
||||
newOutputStreamStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET);
|
||||
newOutputStreamStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET);
|
||||
outputStreamStartPositionUs = newOutputStreamStartPositionUs;
|
||||
}
|
||||
videoFrameRenderControl.onFrameAvailableForRendering(bufferPresentationTimeUs);
|
||||
if (finalBufferPresentationTimeUs != C.TIME_UNSET
|
||||
&& bufferPresentationTimeUs >= finalBufferPresentationTimeUs) {
|
||||
boolean isLastFrame =
|
||||
finalBufferPresentationTimeUs != C.TIME_UNSET
|
||||
&& bufferPresentationTimeUs >= finalBufferPresentationTimeUs;
|
||||
defaultVideoSink.handleInputFrame(framePresentationTimeUs, isLastFrame, videoFrameHandler);
|
||||
if (isLastFrame) {
|
||||
// TODO b/257464707 - Support extensively modified media.
|
||||
defaultVideoSink.signalEndOfCurrentInputStream();
|
||||
hasSignaledEndOfCurrentInputStream = true;
|
||||
@ -438,6 +445,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
} catch (VideoFrameProcessingException e) {
|
||||
throw new VideoSink.VideoSinkException(e, sourceFormat);
|
||||
}
|
||||
defaultVideoSink.setListener(new DefaultVideoSinkListener(), /* executor= */ handler::post);
|
||||
defaultVideoSink.initialize(sourceFormat);
|
||||
state = STATE_INITIALIZED;
|
||||
return videoGraph.getProcessor(/* inputIndex= */ 0);
|
||||
@ -496,7 +504,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
long lastStartPositionUs = checkNotNull(streamStartPositionsUs.pollFirst());
|
||||
// defaultVideoSink should use the latest startPositionUs if none is passed after flushing.
|
||||
defaultVideoSink.setStreamTimestampInfo(
|
||||
lastStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET);
|
||||
lastStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET);
|
||||
}
|
||||
finalBufferPresentationTimeUs = C.TIME_UNSET;
|
||||
hasSignaledEndOfCurrentInputStream = false;
|
||||
@ -507,7 +515,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
|
||||
private void setVideoFrameMetadataListener(
|
||||
VideoFrameMetadataListener videoFrameMetadataListener) {
|
||||
this.videoFrameMetadataListener = videoFrameMetadataListener;
|
||||
defaultVideoSink.setVideoFrameMetadataListener(videoFrameMetadataListener);
|
||||
}
|
||||
|
||||
private void setPlaybackSpeed(float speed) {
|
||||
@ -516,6 +524,8 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
|
||||
private void setBufferTimestampAdjustment(long bufferTimestampAdjustmentUs) {
|
||||
this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs;
|
||||
defaultVideoSink.setStreamTimestampInfo(
|
||||
outputStreamStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET);
|
||||
}
|
||||
|
||||
private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) {
|
||||
@ -854,51 +864,35 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
|
||||
}
|
||||
}
|
||||
|
||||
private final class FrameRendererImpl implements VideoFrameRenderControl.FrameRenderer {
|
||||
|
||||
private @MonotonicNonNull Format renderedFormat;
|
||||
private final class DefaultVideoSinkListener implements VideoSink.Listener {
|
||||
|
||||
@Override
|
||||
public void onVideoSizeChanged(VideoSize videoSize) {
|
||||
renderedFormat =
|
||||
new Format.Builder()
|
||||
.setWidth(videoSize.width)
|
||||
.setHeight(videoSize.height)
|
||||
.setSampleMimeType(MimeTypes.VIDEO_RAW)
|
||||
.build();
|
||||
public void onFirstFrameRendered(VideoSink videoSink) {
|
||||
for (PlaybackVideoGraphWrapper.Listener listener : listeners) {
|
||||
listener.onFirstFrameRendered(PlaybackVideoGraphWrapper.this);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFrameDropped(VideoSink videoSink) {
|
||||
for (PlaybackVideoGraphWrapper.Listener listener : listeners) {
|
||||
listener.onFrameDropped(PlaybackVideoGraphWrapper.this);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize) {
|
||||
for (PlaybackVideoGraphWrapper.Listener listener : listeners) {
|
||||
listener.onVideoSizeChanged(PlaybackVideoGraphWrapper.this, videoSize);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void renderFrame(
|
||||
long renderTimeNs, long bufferPresentationTimeUs, boolean isFirstFrame) {
|
||||
if (isFirstFrame && currentSurfaceAndSize != null) {
|
||||
for (PlaybackVideoGraphWrapper.Listener listener : listeners) {
|
||||
listener.onFirstFrameRendered(PlaybackVideoGraphWrapper.this);
|
||||
}
|
||||
}
|
||||
if (videoFrameMetadataListener != null) {
|
||||
// TODO b/292111083 - renderedFormat is initialized after the first frame is rendered
|
||||
// because onVideoSizeChanged is announced after the first frame is available for
|
||||
// rendering.
|
||||
Format format = renderedFormat == null ? new Format.Builder().build() : renderedFormat;
|
||||
videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
|
||||
/* presentationTimeUs= */ bufferPresentationTimeUs,
|
||||
clock.nanoTime(),
|
||||
format,
|
||||
/* mediaFormat= */ null);
|
||||
}
|
||||
checkStateNotNull(videoGraph).renderOutputFrame(renderTimeNs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropFrame() {
|
||||
public void onError(VideoSink videoSink, VideoSink.VideoSinkException videoSinkException) {
|
||||
for (PlaybackVideoGraphWrapper.Listener listener : listeners) {
|
||||
listener.onFrameDropped(PlaybackVideoGraphWrapper.this);
|
||||
listener.onError(
|
||||
PlaybackVideoGraphWrapper.this, VideoFrameProcessingException.from(videoSinkException));
|
||||
}
|
||||
checkStateNotNull(videoGraph).renderOutputFrame(VideoFrameProcessor.DROP_OUTPUT_FRAME);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user