diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 3c7b606661..d774014ed8 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -50,7 +50,6 @@ import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.exoplayer.ExoPlaybackException; -import androidx.media3.exoplayer.video.VideoFrameReleaseControl.FrameTimingEvaluator; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; @@ -80,7 +79,6 @@ public final class CompositingVideoSinkProvider private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory; private PreviewingVideoGraph.@MonotonicNonNull Factory previewingVideoGraphFactory; - private @MonotonicNonNull VideoFrameReleaseControl videoFrameReleaseControl; private boolean built; /** Creates a builder with the supplied {@linkplain Context application context}. */ @@ -119,32 +117,6 @@ public final class CompositingVideoSinkProvider return this; } - /** - * Sets the {@link VideoFrameReleaseControl} that will be used. - * - *

By default, a {@link VideoFrameReleaseControl} will be used with a {@link - * FrameTimingEvaluator} implementation which: - * - *

- * - * @param videoFrameReleaseControl The {@link VideoFrameReleaseControl}. - * @return This builder, for convenience. - */ - public Builder setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl) { - this.videoFrameReleaseControl = videoFrameReleaseControl; - return this; - } - /** * Builds the {@link CompositingVideoSinkProvider}. * @@ -161,11 +133,6 @@ public final class CompositingVideoSinkProvider previewingVideoGraphFactory = new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory); } - if (videoFrameReleaseControl == null) { - videoFrameReleaseControl = - new VideoFrameReleaseControl( - context, new CompositionFrameTimingEvaluator(), /* allowedJoiningTimeMs= */ 0); - } CompositingVideoSinkProvider compositingVideoSinkProvider = new CompositingVideoSinkProvider(this); built = true; @@ -173,41 +140,6 @@ public final class CompositingVideoSinkProvider } } - /** The time threshold, in microseconds, after which a frame is considered late. */ - public static final long FRAME_LATE_THRESHOLD_US = -30_000; - - /** - * The maximum elapsed time threshold, in microseconds, since last releasing a frame after which a - * frame can be force released. - */ - public static final long FRAME_RELEASE_THRESHOLD_US = 100_000; - - /** A {@link FrameTimingEvaluator} for composition frames. */ - private static final class CompositionFrameTimingEvaluator implements FrameTimingEvaluator { - - @Override - public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { - return earlyUs < FRAME_LATE_THRESHOLD_US - && elapsedSinceLastReleaseUs > FRAME_RELEASE_THRESHOLD_US; - } - - @Override - public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { - return earlyUs < FRAME_LATE_THRESHOLD_US && !isLastFrame; - } - - @Override - public boolean shouldIgnoreFrame( - long earlyUs, - long positionUs, - long elapsedRealtimeUs, - boolean isLastFrame, - boolean treatDroppedBuffersAsSkipped) { - // TODO b/293873191 - Handle very late buffers and drop to key frame. - return false; - } - } - @Documented @Retention(RetentionPolicy.SOURCE) @Target(TYPE_USE) @@ -222,10 +154,10 @@ public final class CompositingVideoSinkProvider private final Context context; private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; - private final VideoFrameReleaseControl videoFrameReleaseControl; - private final VideoFrameRenderControl videoFrameRenderControl; private Clock clock; + private @MonotonicNonNull VideoFrameReleaseControl videoFrameReleaseControl; + private @MonotonicNonNull VideoFrameRenderControl videoFrameRenderControl; private @MonotonicNonNull Format outputFormat; private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener; private @MonotonicNonNull HandlerWrapper handler; @@ -241,11 +173,6 @@ public final class CompositingVideoSinkProvider private CompositingVideoSinkProvider(Builder builder) { this.context = builder.context; this.previewingVideoGraphFactory = checkStateNotNull(builder.previewingVideoGraphFactory); - videoFrameReleaseControl = checkStateNotNull(builder.videoFrameReleaseControl); - @SuppressWarnings("nullness:assignment") - VideoFrameRenderControl.@Initialized FrameRenderer thisRef = this; - videoFrameRenderControl = - new VideoFrameRenderControl(/* frameRenderer= */ thisRef, videoFrameReleaseControl); clock = Clock.DEFAULT; listener = VideoSink.Listener.NO_OP; listenerExecutor = NO_OP_EXECUTOR; @@ -258,6 +185,7 @@ public final class CompositingVideoSinkProvider public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException { checkState(state == STATE_CREATED); checkStateNotNull(videoEffects); + checkState(videoFrameRenderControl != null && videoFrameReleaseControl != null); // Lazily initialize the handler here so it's initialized on the playback looper. handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null); @@ -356,6 +284,14 @@ public final class CompositingVideoSinkProvider outputSurface, outputResolution.getWidth(), outputResolution.getHeight()); } + @Override + public void setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl) { + checkState(!isInitialized()); + this.videoFrameReleaseControl = videoFrameReleaseControl; + videoFrameRenderControl = + new VideoFrameRenderControl(/* frameRenderer= */ this, videoFrameReleaseControl); + } + @Override public void clearOutputSurfaceInfo() { maybeSetOutputSurfaceInfo( @@ -371,6 +307,7 @@ public final class CompositingVideoSinkProvider } @Override + @Nullable public VideoFrameReleaseControl getVideoFrameReleaseControl() { return videoFrameReleaseControl; } @@ -386,7 +323,7 @@ public final class CompositingVideoSinkProvider @Override public void onOutputSizeChanged(int width, int height) { // We forward output size changes to render control even if we are still flushing. - videoFrameRenderControl.onOutputSizeChanged(width, height); + checkStateNotNull(videoFrameRenderControl).onOutputSizeChanged(width, height); } @Override @@ -395,7 +332,8 @@ public final class CompositingVideoSinkProvider // Ignore available frames while the sink provider is flushing return; } - videoFrameRenderControl.onOutputFrameAvailableForRendering(presentationTimeUs); + checkStateNotNull(videoFrameRenderControl) + .onOutputFrameAvailableForRendering(presentationTimeUs); } @Override @@ -471,7 +409,7 @@ public final class CompositingVideoSinkProvider */ public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { if (pendingFlushCount == 0) { - videoFrameRenderControl.render(positionUs, elapsedRealtimeUs); + checkStateNotNull(videoFrameRenderControl).render(positionUs, elapsedRealtimeUs); } } @@ -502,23 +440,24 @@ public final class CompositingVideoSinkProvider // Update the surface on the video graph and the video frame release control together. SurfaceInfo surfaceInfo = surface != null ? new SurfaceInfo(surface, width, height) : null; videoGraph.setOutputSurfaceInfo(surfaceInfo); - videoFrameReleaseControl.setOutputSurface(surface); + checkNotNull(videoFrameReleaseControl).setOutputSurface(surface); } } private boolean isReady() { - return pendingFlushCount == 0 && videoFrameRenderControl.isReady(); + return pendingFlushCount == 0 && checkStateNotNull(videoFrameRenderControl).isReady(); } private boolean hasReleasedFrame(long presentationTimeUs) { - return pendingFlushCount == 0 && videoFrameRenderControl.hasReleasedFrame(presentationTimeUs); + return pendingFlushCount == 0 + && checkStateNotNull(videoFrameRenderControl).hasReleasedFrame(presentationTimeUs); } private void flush() { pendingFlushCount++; // Flush the render control now to ensure it has no data, eg calling isReady() must return false // and render() should not render any frames. - videoFrameRenderControl.flush(); + checkStateNotNull(videoFrameRenderControl).flush(); // Finish flushing after handling pending video graph callbacks to ensure video size changes // reach the video render control. checkStateNotNull(handler).post(this::flushInternal); @@ -533,15 +472,16 @@ public final class CompositingVideoSinkProvider throw new IllegalStateException(String.valueOf(pendingFlushCount)); } // Flush the render control again. - videoFrameRenderControl.flush(); + checkStateNotNull(videoFrameRenderControl).flush(); } private void setPlaybackSpeed(float speed) { - videoFrameRenderControl.setPlaybackSpeed(speed); + checkStateNotNull(videoFrameRenderControl).setPlaybackSpeed(speed); } private void onStreamOffsetChange(long bufferPresentationTimeUs, long streamOffsetUs) { - videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs); + checkStateNotNull(videoFrameRenderControl) + .onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs); } private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 4a5b576688..c02f3bb12b 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -393,17 +393,18 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer this.context = context.getApplicationContext(); eventDispatcher = new EventDispatcher(eventHandler, eventListener); if (videoSinkProvider == null) { + videoSinkProvider = new CompositingVideoSinkProvider.Builder(this.context).build(); + } + if (videoSinkProvider.getVideoFrameReleaseControl() == null) { @SuppressWarnings("nullness:assignment") VideoFrameReleaseControl.@Initialized FrameTimingEvaluator thisRef = this; - videoSinkProvider = - new CompositingVideoSinkProvider.Builder(this.context) - .setVideoFrameReleaseControl( - new VideoFrameReleaseControl( - this.context, /* frameTimingEvaluator= */ thisRef, allowedJoiningTimeMs)) - .build(); + videoSinkProvider.setVideoFrameReleaseControl( + new VideoFrameReleaseControl( + this.context, /* frameTimingEvaluator= */ thisRef, allowedJoiningTimeMs)); } this.videoSinkProvider = videoSinkProvider; - this.videoFrameReleaseControl = this.videoSinkProvider.getVideoFrameReleaseControl(); + this.videoFrameReleaseControl = + checkStateNotNull(this.videoSinkProvider.getVideoFrameReleaseControl()); videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround(); scalingMode = C.VIDEO_SCALING_MODE_DEFAULT; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java index e90eae9db2..63fc19a0e2 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java @@ -23,6 +23,7 @@ import androidx.media3.common.util.Clock; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import java.util.List; +import org.checkerframework.checker.nullness.qual.Nullable; /** A provider of {@link VideoSink VideoSinks}. */ @UnstableApi @@ -70,6 +71,15 @@ public interface VideoSinkProvider { */ void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution); + /** + * Sets the {@link VideoFrameReleaseControl} that will be used for releasing of video frames + * during rendering. + * + *

Must be called before, not after, the sink provider is {@linkplain #initialize(Format) + * initialized}. + */ + void setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl); + /** * Clears the set output surface info. * @@ -83,8 +93,11 @@ public interface VideoSinkProvider { /** * Returns the {@link VideoFrameReleaseControl} that will be used for releasing of video frames * during rendering. + * + *

If this value is {@code null}, it must be {@linkplain #setVideoFrameReleaseControl set} to a + * non-null value before rendering begins. */ - VideoFrameReleaseControl getVideoFrameReleaseControl(); + @Nullable VideoFrameReleaseControl getVideoFrameReleaseControl(); /** * Sets the {@link Clock} that the provider should use internally. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java index ea22a2bf73..d7182a7edf 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java @@ -18,7 +18,6 @@ package androidx.media3.exoplayer.video; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import android.content.Context; @@ -45,12 +44,7 @@ public final class CompositingVideoSinkProviderTest { @Test public void builder_calledMultipleTimes_throws() { CompositingVideoSinkProvider.Builder builder = - new CompositingVideoSinkProvider.Builder(ApplicationProvider.getApplicationContext()) - .setVideoFrameReleaseControl( - new VideoFrameReleaseControl( - ApplicationProvider.getApplicationContext(), - mock(VideoFrameReleaseControl.FrameTimingEvaluator.class), - /* allowedJoiningTimeMs= */ 0)); + new CompositingVideoSinkProvider.Builder(ApplicationProvider.getApplicationContext()); builder.build(); @@ -169,12 +163,13 @@ public final class CompositingVideoSinkProviderTest { return false; } }; - VideoFrameReleaseControl releaseControl = - new VideoFrameReleaseControl(context, frameTimingEvaluator, /* allowedJoiningTimeMs= */ 0); - return new CompositingVideoSinkProvider.Builder(context) - .setPreviewingVideoGraphFactory(new TestPreviewingVideoGraphFactory()) - .setVideoFrameReleaseControl(releaseControl) - .build(); + CompositingVideoSinkProvider compositingVideoSinkProvider = + new CompositingVideoSinkProvider.Builder(context) + .setPreviewingVideoGraphFactory(new TestPreviewingVideoGraphFactory()) + .build(); + compositingVideoSinkProvider.setVideoFrameReleaseControl( + new VideoFrameReleaseControl(context, frameTimingEvaluator, /* allowedJoiningTimeMs= */ 0)); + return compositingVideoSinkProvider; } private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory {