From e3caed14410f7819626ef5a85681bb2f3a6e2b65 Mon Sep 17 00:00:00 2001 From: kimvde Date: Tue, 23 Apr 2024 00:34:08 -0700 Subject: [PATCH] Move VideoSinkProvider.initialize to VideoSink PiperOrigin-RevId: 627290721 --- .../video/CompositingVideoSinkProvider.java | 175 +++++++++--------- .../video/MediaCodecVideoRenderer.java | 140 +++++++------- .../media3/exoplayer/video/VideoSink.java | 37 +++- .../exoplayer/video/VideoSinkProvider.java | 25 +-- .../CompositingVideoSinkProviderTest.java | 43 ++--- 5 files changed, 217 insertions(+), 203 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 6c54050bb6..acd032b32c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -64,8 +64,8 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.Executor; -import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** Handles composition of video sinks. */ @@ -197,18 +197,17 @@ public final class CompositingVideoSinkProvider private static final Executor NO_OP_EXECUTOR = runnable -> {}; private final Context context; + private final VideoSinkImpl videoSinkImpl; private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; private final CopyOnWriteArraySet listeners; private Clock clock; - private List videoEffects; private @MonotonicNonNull VideoFrameReleaseControl videoFrameReleaseControl; private @MonotonicNonNull VideoFrameRenderControl videoFrameRenderControl; private @MonotonicNonNull Format outputFormat; private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener; private @MonotonicNonNull HandlerWrapper handler; private @MonotonicNonNull PreviewingVideoGraph videoGraph; - private @MonotonicNonNull VideoSinkImpl videoSinkImpl; @Nullable private Pair currentSurfaceAndSize; private int pendingFlushCount; private @State int state; @@ -216,12 +215,13 @@ public final class CompositingVideoSinkProvider private CompositingVideoSinkProvider(Builder builder) { context = builder.context; - videoEffects = ImmutableList.of(); + videoSinkImpl = new VideoSinkImpl(context); previewingVideoGraphFactory = checkStateNotNull(builder.previewingVideoGraphFactory); listeners = new CopyOnWriteArraySet<>(); clock = Clock.DEFAULT; state = STATE_CREATED; playbackSpeed = 1f; + addListener(videoSinkImpl); } /** @@ -267,78 +267,22 @@ public final class CompositingVideoSinkProvider @Override public void setVideoEffects(List videoEffects) { - this.videoEffects = videoEffects; - if (isInitialized()) { - checkStateNotNull(videoSinkImpl).setVideoEffects(videoEffects); - } + videoSinkImpl.setVideoEffects(videoEffects); } @Override public void setPendingVideoEffects(List videoEffects) { - this.videoEffects = videoEffects; - if (isInitialized()) { - checkStateNotNull(videoSinkImpl).setPendingVideoEffects(videoEffects); - } - } - - @Override - public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException { - checkState(state == STATE_CREATED); - checkState(videoFrameRenderControl != null && videoFrameReleaseControl != null); - - // Lazily initialize the handler here so it's initialized on the playback looper. - handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null); - - ColorInfo inputColorInfo = getAdjustedInputColorInfo(sourceFormat.colorInfo); - ColorInfo outputColorInfo = inputColorInfo; - if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG && Util.SDK_INT < 34) { - // PQ SurfaceView output is supported from API 33, but HLG output is supported from API 34. - // Therefore, convert HLG to PQ below API 34, so that HLG input can be displayed properly on - // API 33. - outputColorInfo = - inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build(); - } - try { - @SuppressWarnings("nullness:assignment") - VideoGraph.@Initialized Listener thisRef = this; - videoGraph = - previewingVideoGraphFactory.create( - context, - outputColorInfo, - DebugViewProvider.NONE, - /* listener= */ thisRef, - /* listenerExecutor= */ handler::post, - /* compositionEffects= */ ImmutableList.of(), - /* initialTimestampOffsetUs= */ 0); - if (currentSurfaceAndSize != null) { - Surface surface = currentSurfaceAndSize.first; - Size size = currentSurfaceAndSize.second; - maybeSetOutputSurfaceInfo(surface, size.getWidth(), size.getHeight()); - } - videoSinkImpl = new VideoSinkImpl(context, videoGraph); - } catch (VideoFrameProcessingException e) { - throw new VideoSink.VideoSinkException(e, sourceFormat); - } - if (!videoEffects.isEmpty()) { - videoSinkImpl.setVideoEffects(videoEffects); - } - addListener(videoSinkImpl); - state = STATE_INITIALIZED; - } - - @Override - public boolean isInitialized() { - return state == STATE_INITIALIZED; + videoSinkImpl.setPendingVideoEffects(videoEffects); } @Override public VideoSink getSink() { - return checkStateNotNull(videoSinkImpl); + return videoSinkImpl; } @Override public void setStreamOffsetUs(long streamOffsetUs) { - checkStateNotNull(videoSinkImpl).setStreamOffsetUs(streamOffsetUs); + videoSinkImpl.setStreamOffsetUs(streamOffsetUs); } @Override @@ -485,6 +429,50 @@ public final class CompositingVideoSinkProvider // Internal methods + private VideoFrameProcessor initialize(Format sourceFormat) throws VideoSink.VideoSinkException { + checkState(state == STATE_CREATED); + checkState(videoFrameRenderControl != null && videoFrameReleaseControl != null); + + // Lazily initialize the handler here so it's initialized on the playback looper. + handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null); + + ColorInfo inputColorInfo = getAdjustedInputColorInfo(sourceFormat.colorInfo); + ColorInfo outputColorInfo = inputColorInfo; + if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG && Util.SDK_INT < 34) { + // PQ SurfaceView output is supported from API 33, but HLG output is supported from API 34. + // Therefore, convert HLG to PQ below API 34, so that HLG input can be displayed properly on + // API 33. + outputColorInfo = + inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build(); + } + int videoGraphInputId; + try { + videoGraph = + previewingVideoGraphFactory.create( + context, + outputColorInfo, + DebugViewProvider.NONE, + /* listener= */ this, + /* listenerExecutor= */ handler::post, + /* compositionEffects= */ ImmutableList.of(), + /* initialTimestampOffsetUs= */ 0); + if (currentSurfaceAndSize != null) { + Surface surface = currentSurfaceAndSize.first; + Size size = currentSurfaceAndSize.second; + maybeSetOutputSurfaceInfo(surface, size.getWidth(), size.getHeight()); + } + videoGraphInputId = videoGraph.registerInput(); + } catch (VideoFrameProcessingException e) { + throw new VideoSink.VideoSinkException(e, sourceFormat); + } + state = STATE_INITIALIZED; + return videoGraph.getProcessor(videoGraphInputId); + } + + private boolean isInitialized() { + return state == STATE_INITIALIZED; + } + private void maybeSetOutputSurfaceInfo(@Nullable Surface surface, int width, int height) { if (videoGraph != null) { // Update the surface on the video graph and the video frame release control together. @@ -551,11 +539,11 @@ public final class CompositingVideoSinkProvider /** Receives input from an ExoPlayer renderer and forwards it to the video graph. */ private final class VideoSinkImpl implements VideoSink, CompositingVideoSinkProvider.Listener { private final Context context; - private final VideoFrameProcessor videoFrameProcessor; private final int videoFrameProcessorMaxPendingFrameCount; private final ArrayList videoEffects; @Nullable private Effect rotationEffect; + private @MonotonicNonNull VideoFrameProcessor videoFrameProcessor; @Nullable private Format inputFormat; private @InputType int inputType; private long inputStreamOffsetUs; @@ -575,8 +563,7 @@ public final class CompositingVideoSinkProvider private Executor listenerExecutor; /** Creates a new instance. */ - public VideoSinkImpl(Context context, PreviewingVideoGraph videoGraph) - throws VideoFrameProcessingException { + public VideoSinkImpl(Context context) { this.context = context; // TODO b/226330223 - Investigate increasing frame count when frame dropping is // allowed. @@ -584,8 +571,6 @@ public final class CompositingVideoSinkProvider // reduces decoder timeouts, and consider restoring. videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); - int videoGraphInputId = videoGraph.registerInput(); - videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId); videoEffects = new ArrayList<>(); finalBufferPresentationTimeUs = C.TIME_UNSET; @@ -596,9 +581,23 @@ public final class CompositingVideoSinkProvider // VideoSink impl + @Override + public void initialize(Format sourceFormat) throws VideoSinkException { + checkState(!isInitialized()); + videoFrameProcessor = CompositingVideoSinkProvider.this.initialize(sourceFormat); + } + + @Override + @EnsuresNonNullIf(result = true, expression = "videoFrameProcessor") + public boolean isInitialized() { + return videoFrameProcessor != null; + } + @Override public void flush() { - videoFrameProcessor.flush(); + if (isInitialized()) { + videoFrameProcessor.flush(); + } hasRegisteredFirstInputStream = false; finalBufferPresentationTimeUs = C.TIME_UNSET; lastBufferPresentationTimeUs = C.TIME_UNSET; @@ -611,17 +610,19 @@ public final class CompositingVideoSinkProvider @Override public boolean isReady() { - return CompositingVideoSinkProvider.this.isReady(); + return isInitialized() && CompositingVideoSinkProvider.this.isReady(); } @Override public boolean isEnded() { - return finalBufferPresentationTimeUs != C.TIME_UNSET + return isInitialized() + && finalBufferPresentationTimeUs != C.TIME_UNSET && CompositingVideoSinkProvider.this.hasReleasedFrame(finalBufferPresentationTimeUs); } @Override public void registerInputStream(@InputType int inputType, Format format) { + checkState(isInitialized()); switch (inputType) { case INPUT_TYPE_SURFACE: case INPUT_TYPE_BITMAP: @@ -674,11 +675,13 @@ public final class CompositingVideoSinkProvider @Override public Surface getInputSurface() { - return videoFrameProcessor.getInputSurface(); + checkState(isInitialized()); + return checkStateNotNull(videoFrameProcessor).getInputSurface(); } @Override public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame) { + checkState(isInitialized()); checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET); // An input stream is fully decoded, wait until all of its frames are released before queueing @@ -693,11 +696,11 @@ public final class CompositingVideoSinkProvider } } - if (videoFrameProcessor.getPendingInputFrameCount() + if (checkStateNotNull(videoFrameProcessor).getPendingInputFrameCount() >= videoFrameProcessorMaxPendingFrameCount) { return C.TIME_UNSET; } - if (!videoFrameProcessor.registerInputFrame()) { + if (!checkStateNotNull(videoFrameProcessor).registerInputFrame()) { return C.TIME_UNSET; } // The sink takes in frames with monotonically increasing, non-offset frame @@ -717,6 +720,8 @@ public final class CompositingVideoSinkProvider @Override public boolean queueBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator) { + checkState(isInitialized()); + if (!maybeRegisterPendingInputStream()) { return false; } @@ -744,6 +749,7 @@ public final class CompositingVideoSinkProvider @Override public void render(long positionUs, long elapsedRealtimeUs) throws VideoSinkException { + checkState(isInitialized()); try { CompositingVideoSinkProvider.this.render(positionUs, elapsedRealtimeUs); } catch (ExoPlaybackException e) { @@ -820,15 +826,16 @@ public final class CompositingVideoSinkProvider } effects.addAll(videoEffects); Format inputFormat = checkNotNull(this.inputFormat); - videoFrameProcessor.registerInputStream( - inputType, - effects, - new FrameInfo.Builder( - getAdjustedInputColorInfo(inputFormat.colorInfo), - inputFormat.width, - inputFormat.height) - .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) - .build()); + checkStateNotNull(videoFrameProcessor) + .registerInputStream( + inputType, + effects, + new FrameInfo.Builder( + getAdjustedInputColorInfo(inputFormat.colorInfo), + inputFormat.width, + inputFormat.height) + .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) + .build()); finalBufferPresentationTimeUs = C.TIME_UNSET; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 5e5ba09659..a779687946 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -22,6 +22,7 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_VIDEO_MAX_RESOLUTION_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.lang.Math.max; import static java.lang.Math.min; @@ -77,7 +78,6 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException; import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatcher; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; import java.util.List; import org.checkerframework.checker.initialization.qual.Initialized; @@ -367,12 +367,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer * @param assumedMinimumCodecOperatingRate A codec operating rate that all codecs instantiated by * this renderer are assumed to meet implicitly (i.e. without the operating rate being set * explicitly using {@link MediaFormat#KEY_OPERATING_RATE}). - * @param videoSinkProvider The {@link VideoSinkProvider} that will used be used for applying - * video effects also providing the {@linkplain - * VideoSinkProvider#getVideoFrameReleaseControl() VideoFrameReleaseControl} for releasing - * video frames. If {@code null}, the {@link CompositingVideoSinkProvider} with its default - * configuration will be used, and the renderer will drive releasing of video frames by - * itself. + * @param videoSinkProvider The {@link VideoSinkProvider} that will be used for applying video + * effects also providing the {@linkplain VideoSinkProvider#getVideoFrameReleaseControl() + * VideoFrameReleaseControl} for releasing video frames. If {@code null}, the {@link + * CompositingVideoSinkProvider} with its default configuration will be used, and the renderer + * will drive releasing of video frames by itself. */ public MediaCodecVideoRenderer( Context context, @@ -733,7 +732,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @Override protected void onRelease() { super.onRelease(); - if (ownsVideoSinkProvider && videoSinkProvider.isInitialized()) { + if (ownsVideoSinkProvider && videoSink != null) { videoSinkProvider.release(); } } @@ -777,7 +776,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer outputResolution = (Size) checkNotNull(message); // TODO: b/292111083 Set the surface on the videoSinkProvider before it's initialized // otherwise the first frames are missed until a new video output resolution arrives. - if (videoSinkProvider.isInitialized() + if (videoSink != null && checkNotNull(outputResolution).getWidth() != 0 && checkNotNull(outputResolution).getHeight() != 0 && displaySurface != null) { @@ -820,10 +819,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @State int state = getState(); @Nullable MediaCodecAdapter codec = getCodec(); - // The video sink provider is initialized just before the first codec is ever created, so - // the provider can be initialized only when the codec is non-null. Therefore, we don't have - // to check if the provider is initialized but the codec is not. - if (codec != null && !videoSinkProvider.isInitialized()) { + // The video sink is instantiated just before the first codec is ever created, so the sink can + // be non-null only when the codec is non-null. Therefore, we don't have to check if the sink + // is non-null but the codec is null. + if (codec != null && videoSink == null) { if (Util.SDK_INT >= 23 && displaySurface != null && !codecNeedsSetOutputSurfaceWorkaround) { setOutputSurfaceV23(codec, displaySurface); } else { @@ -842,13 +841,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true); } // When effects previewing is enabled, set display surface and an unknown size. - if (videoSinkProvider.isInitialized()) { + if (videoSink != null) { videoSinkProvider.setOutputSurfaceInfo(displaySurface, Size.UNKNOWN); } } else { // The display surface has been removed. reportedVideoSize = null; - if (videoSinkProvider.isInitialized()) { + if (videoSink != null) { videoSinkProvider.clearOutputSurfaceInfo(); } } @@ -1059,59 +1058,72 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer @CallSuper @Override protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException { - // We only enable effects preview on the first time a codec is initialized and if effects are + // If the video sink provider is created by the renderer, we only enable effects preview (by + // using the video sink) on the first time a codec is initialized and if effects are // already set. We do not enable effects mid-playback. For effects to be enabled after // playback has started, the renderer needs to be reset first. - if (hasEffects && !hasInitializedPlayback && !videoSinkProvider.isInitialized()) { + boolean enableEffectsForOwnSinkProvider = + ownsVideoSinkProvider && hasEffects && !hasInitializedPlayback; + // We always use the video sink if the video sink provider is passed to the renderer. + boolean useVideoSink = enableEffectsForOwnSinkProvider || !ownsVideoSinkProvider; + if (useVideoSink && videoSink == null) { try { - videoSinkProvider.initialize(format); - if (displaySurface != null && outputResolution != null) { - videoSinkProvider.setOutputSurfaceInfo(displaySurface, outputResolution); + videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); + videoSink = videoSinkProvider.getSink(); + if (!videoSink.isInitialized()) { + try { + videoSink.initialize(format); + } catch (VideoSink.VideoSinkException e) { + throw createRendererException( + e, format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED); + } } - } catch (VideoSink.VideoSinkException e) { - throw createRendererException( - e, format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED); + videoSink.setListener( + new VideoSink.Listener() { + @Override + public void onFirstFrameRendered(VideoSink videoSink) { + checkStateNotNull(displaySurface); + notifyRenderedFirstFrame(); + } + + @Override + public void onFrameDropped(VideoSink videoSink) { + updateDroppedBufferCounters( + /* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1); + } + + @Override + public void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize) { + // TODO: b/292111083 - Report video size change to app. Video size reporting is + // removed at the moment to ensure the first frame is rendered, and the video is + // rendered after switching on/off the screen. + } + + @Override + public void onError( + VideoSink videoSink, VideoSink.VideoSinkException videoSinkException) { + setPendingPlaybackException( + createRendererException( + videoSinkException, + videoSinkException.format, + PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); + } + }, + // Pass a direct executor since the callback handling involves posting on the app looper + // again, so there's no need to do two hops. + directExecutor()); + if (enableEffectsForOwnSinkProvider) { + if (displaySurface != null && outputResolution != null) { + videoSinkProvider.setOutputSurfaceInfo(displaySurface, outputResolution); + } + } + } catch (Exception e) { + // Set videoSink back to null so that, if the try block fails and the renderer retries the + // codec initialization, the try block is re-executed. + videoSink = null; + throw e; } } - - if (videoSink == null && videoSinkProvider.isInitialized()) { - videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); - videoSink = videoSinkProvider.getSink(); - videoSink.setListener( - new VideoSink.Listener() { - @Override - public void onFirstFrameRendered(VideoSink videoSink) { - checkStateNotNull(displaySurface); - notifyRenderedFirstFrame(); - } - - @Override - public void onFrameDropped(VideoSink videoSink) { - updateDroppedBufferCounters( - /* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1); - } - - @Override - public void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize) { - // TODO: b/292111083 - Report video size change to app. Video size reporting is - // removed at the moment to ensure the first frame is rendered, and the video is - // rendered after switching on/off the screen. - } - - @Override - public void onError( - VideoSink videoSink, VideoSink.VideoSinkException videoSinkException) { - setPendingPlaybackException( - createRendererException( - videoSinkException, - videoSinkException.format, - PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); - } - }, - // Pass a direct executor since the callback handling involves posting on the app looper - // again, so there's no need to do two hops. - MoreExecutors.directExecutor()); - } hasInitializedPlayback = true; } @@ -1335,7 +1347,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer // We are not rendering on a surface, the renderer will wait until a surface is set. // Opportunistically render to VideoFrameProcessor if effects are enabled. - if (displaySurface == placeholderSurface && !videoSinkProvider.isInitialized()) { + if (displaySurface == placeholderSurface && videoSink == null) { // Skip frames in sync with playback, so we'll be at the right frame if the mode changes. if (videoFrameReleaseInfo.getEarlyUs() < 30_000) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); @@ -1464,7 +1476,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer super.onProcessedStreamChange(); videoFrameReleaseControl.onProcessedStreamChange(); maybeSetupTunnelingForFirstFrame(); - if (videoSinkProvider.isInitialized()) { + if (videoSink != null) { videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java index 309cb3a346..26fb682916 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java @@ -108,6 +108,17 @@ public interface VideoSink { */ void setListener(Listener listener, Executor executor); + /** + * Initializes the video sink. + * + * @param sourceFormat The format of the compressed video. + * @throws VideoSink.VideoSinkException If initializing the sink failed. + */ + void initialize(Format sourceFormat) throws VideoSinkException; + + /** Returns whether the video sink is {@linkplain #initialize(Format) initialized}. */ + boolean isInitialized(); + /** * Flushes the video sink. * @@ -115,20 +126,28 @@ public interface VideoSink { */ void flush(); - /** Whether the video sink is able to immediately render media from the current position. */ + /** + * Returns whether the video sink is able to immediately render media from the current position. + */ boolean isReady(); /** - * Whether all queued video frames have been rendered, including the frame marked as last buffer. + * Returns whether all queued video frames have been rendered, including the frame marked as last + * buffer. */ boolean isEnded(); /** - * Whether frames could be dropped from the sink's {@linkplain #getInputSurface() input surface}. + * Returns whether frames could be dropped from the sink's {@linkplain #getInputSurface() input + * surface}. */ boolean isFrameDropAllowedOnInput(); - /** Returns the input {@link Surface} where the video sink consumes input frames from. */ + /** + * Returns the input {@link Surface} where the video sink consumes input frames from. + * + *

Must be called after the sink is {@linkplain #initialize(Format) initialized}. + */ Surface getInputSurface(); /** Sets the playback speed. */ @@ -137,6 +156,8 @@ public interface VideoSink { /** * Informs the video sink that a new input stream will be queued. * + *

Must be called after the sink is {@linkplain #initialize(Format) initialized}. + * * @param inputType The {@link InputType} of the stream. * @param format The {@link Format} of the stream. */ @@ -146,9 +167,11 @@ public interface VideoSink { * Informs the video sink that a frame will be queued to its {@linkplain #getInputSurface() input * surface}. * + *

Must be called after the sink is {@linkplain #initialize(Format) initialized}. + * * @param framePresentationTimeUs The frame's presentation time, in microseconds. * @param isLastFrame Whether this is the last frame of the video stream. - * @return a release timestamp, in nanoseconds, that should be associated when releasing this + * @return A release timestamp, in nanoseconds, that should be associated when releasing this * frame, or {@link C#TIME_UNSET} if the sink was not able to register the frame and the * caller must try again later. */ @@ -157,6 +180,8 @@ public interface VideoSink { /** * Provides an input {@link Bitmap} to the video sink. * + *

Must be called after the sink is {@linkplain #initialize(Format) initialized}. + * * @param inputBitmap The {@link Bitmap} queued to the video sink. * @param timestampIterator The times within the current stream that the bitmap should be shown * at. The timestamps should be monotonically increasing. @@ -168,6 +193,8 @@ public interface VideoSink { /** * Incrementally renders processed video frames. * + *

Must be called after the sink is {@linkplain #initialize(Format) initialized}. + * * @param positionUs The current playback position, in microseconds. * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, * taken approximately at the time the playback position was {@code positionUs}. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java index 45b02c1e3e..a597c06d04 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java @@ -33,7 +33,8 @@ public interface VideoSinkProvider { * Sets the {@link VideoFrameReleaseControl} that will be used for releasing of video frames * during rendering. * - *

Must be called before the sink provider is {@linkplain #initialize(Format) initialized}. + *

Must be called before the first {@linkplain #getSink() sink} is {@linkplain + * VideoSink#initialize(Format) initialized}. */ void setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl); @@ -49,7 +50,8 @@ public interface VideoSinkProvider { /** * Sets the {@link Clock} that the provider should use internally. * - *

Must be called before the sink provider is {@linkplain #initialize(Format) initialized}. + *

Must be called before the first {@linkplain #getSink() sink} is {@linkplain + * VideoSink#initialize(Format) initialized}. */ void setClock(Clock clock); @@ -62,29 +64,12 @@ public interface VideoSinkProvider { */ void setPendingVideoEffects(List videoEffects); - /** - * Initializes the provider for video frame processing. Can be called up to one time. - * - * @param sourceFormat The format of the compressed video. - * @throws VideoSink.VideoSinkException If enabling the provider failed. - */ - void initialize(Format sourceFormat) throws VideoSink.VideoSinkException; - - /** Returns whether this provider is initialized for frame processing. */ - boolean isInitialized(); - - /** - * Returns a {@link VideoSink} to forward video frames for processing. - * - *

Must be called after the sink provider is {@linkplain #initialize(Format) initialized}. - */ + /** Returns a {@link VideoSink} to forward video frames for processing. */ VideoSink getSink(); /** * Sets the offset, in microseconds, that is added to the video frames presentation timestamps * from the player. - * - *

Must be called after the sink provider is {@linkplain #initialize(Format) initialized}. */ void setStreamOffsetUs(long streamOffsetUs); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java index af40e47379..33e2b7ed21 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java @@ -51,50 +51,33 @@ public final class CompositingVideoSinkProviderTest { } @Test - public void initialize() throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - - provider.initialize(new Format.Builder().build()); - - assertThat(provider.isInitialized()).isTrue(); - } - - @Test - public void initialize_calledTwice_throws() throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.initialize(new Format.Builder().build()); + public void initializeSink_withoutReleaseControl_throws() { + CompositingVideoSinkProvider provider = + new CompositingVideoSinkProvider.Builder(ApplicationProvider.getApplicationContext()) + .setPreviewingVideoGraphFactory(new TestPreviewingVideoGraphFactory()) + .build(); + VideoSink sink = provider.getSink(); assertThrows( - IllegalStateException.class, () -> provider.initialize(new Format.Builder().build())); + IllegalStateException.class, + () -> sink.initialize(new Format.Builder().setWidth(640).setHeight(480).build())); } @Test - public void isInitialized_afterRelease_returnsFalse() throws VideoSink.VideoSinkException { + public void initializeSink_calledTwice_throws() throws VideoSink.VideoSinkException { CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.initialize(new Format.Builder().build()); + VideoSink sink = provider.getSink(); + sink.initialize(new Format.Builder().build()); - provider.release(); - - assertThat(provider.isInitialized()).isFalse(); - } - - @Test - public void initialize_afterRelease_throws() throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - Format format = new Format.Builder().build(); - - provider.initialize(format); - provider.release(); - - assertThrows(IllegalStateException.class, () -> provider.initialize(format)); + assertThrows(IllegalStateException.class, () -> sink.initialize(new Format.Builder().build())); } @Test public void setOutputStreamOffsetUs_frameReleaseTimesAreAdjusted() throws VideoSink.VideoSinkException { CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.initialize(new Format.Builder().build()); VideoSink videoSink = provider.getSink(); + videoSink.initialize(new Format.Builder().build()); videoSink.registerInputStream( VideoSink.INPUT_TYPE_SURFACE, new Format.Builder().setWidth(640).setHeight(480).build());