Previewing: set VideoFrameReleaseControl after CVSP is created.

This allows us to inject a videoFrameProcessorFactory into
MediaCodecVideoRenderer, without issues about creating the
VideoFrameReleaseControl in the MediaCodecVideoRenderer.

Unfortunately, this does result in more complex CVSP state, where
VideoFrameReleaseControl is no longer final, may be null, and may potentially
change. However, this tries to be careful with assertions to guarantee good
state, and is cleaner than modifying the long-standing MediaCodecVideoRenderer
interface.

Tested that this works on the ExoPlayer demo with setVideoEffects applied, and
using a playlist with SDR->HDR and HDR->SDR items.

PiperOrigin-RevId: 599823412

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java

@@ -50,7 +50,6 @@ import androidx.media3.common.util.TimestampIterator;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import androidx.media3.exoplayer.ExoPlaybackException;
-import androidx.media3.exoplayer.video.VideoFrameReleaseControl.FrameTimingEvaluator;
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
@@ -80,7 +79,6 @@ public final class CompositingVideoSinkProvider
 
     private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory;
     private PreviewingVideoGraph.@MonotonicNonNull Factory previewingVideoGraphFactory;
-    private @MonotonicNonNull VideoFrameReleaseControl videoFrameReleaseControl;
     private boolean built;
 
     /** Creates a builder with the supplied {@linkplain Context application context}. */
@@ -119,32 +117,6 @@ public final class CompositingVideoSinkProvider
       return this;
     }
 
-    /**
-     * Sets the {@link VideoFrameReleaseControl} that will be used.
-     *
-     * <p>By default, a {@link VideoFrameReleaseControl} will be used with a {@link
-     * FrameTimingEvaluator} implementation which:
-     *
-     * <ul>
-     *   <li>Signals to {@linkplain FrameTimingEvaluator#shouldForceReleaseFrame(long, long) force
-     *       release} a frame if the frame is late by more than {@link #FRAME_LATE_THRESHOLD_US} and
-     *       the elapsed time since the previous frame release is greater than {@link
-     *       #FRAME_RELEASE_THRESHOLD_US}.
-     *   <li>Signals to {@linkplain FrameTimingEvaluator#shouldDropFrame(long, long, boolean) drop a
-     *       frame} if the frame is late by more than {@link #FRAME_LATE_THRESHOLD_US} and the frame
-     *       is not marked as the last one.
-     *   <li>Signals to never {@linkplain FrameTimingEvaluator#shouldIgnoreFrame(long, long, long,
-     *       boolean, boolean) ignore} a frame.
-     * </ul>
-     *
-     * @param videoFrameReleaseControl The {@link VideoFrameReleaseControl}.
-     * @return This builder, for convenience.
-     */
-    public Builder setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl) {
-      this.videoFrameReleaseControl = videoFrameReleaseControl;
-      return this;
-    }
-
     /**
      * Builds the {@link CompositingVideoSinkProvider}.
      *
@@ -161,11 +133,6 @@ public final class CompositingVideoSinkProvider
         previewingVideoGraphFactory =
             new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory);
       }
-      if (videoFrameReleaseControl == null) {
-        videoFrameReleaseControl =
-            new VideoFrameReleaseControl(
-                context, new CompositionFrameTimingEvaluator(), /* allowedJoiningTimeMs= */ 0);
-      }
       CompositingVideoSinkProvider compositingVideoSinkProvider =
           new CompositingVideoSinkProvider(this);
       built = true;
@@ -173,41 +140,6 @@ public final class CompositingVideoSinkProvider
     }
   }
 
-  /** The time threshold, in microseconds, after which a frame is considered late. */
-  public static final long FRAME_LATE_THRESHOLD_US = -30_000;
-
-  /**
-   * The maximum elapsed time threshold, in microseconds, since last releasing a frame after which a
-   * frame can be force released.
-   */
-  public static final long FRAME_RELEASE_THRESHOLD_US = 100_000;
-
-  /** A {@link FrameTimingEvaluator} for composition frames. */
-  private static final class CompositionFrameTimingEvaluator implements FrameTimingEvaluator {
-
-    @Override
-    public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) {
-      return earlyUs < FRAME_LATE_THRESHOLD_US
-          && elapsedSinceLastReleaseUs > FRAME_RELEASE_THRESHOLD_US;
-    }
-
-    @Override
-    public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
-      return earlyUs < FRAME_LATE_THRESHOLD_US && !isLastFrame;
-    }
-
-    @Override
-    public boolean shouldIgnoreFrame(
-        long earlyUs,
-        long positionUs,
-        long elapsedRealtimeUs,
-        boolean isLastFrame,
-        boolean treatDroppedBuffersAsSkipped) {
-      // TODO b/293873191 - Handle very late buffers and drop to key frame.
-      return false;
-    }
-  }
-
   @Documented
   @Retention(RetentionPolicy.SOURCE)
   @Target(TYPE_USE)
@@ -222,10 +154,10 @@ public final class CompositingVideoSinkProvider
 
   private final Context context;
   private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
-  private final VideoFrameReleaseControl videoFrameReleaseControl;
-  private final VideoFrameRenderControl videoFrameRenderControl;
 
   private Clock clock;
+  private @MonotonicNonNull VideoFrameReleaseControl videoFrameReleaseControl;
+  private @MonotonicNonNull VideoFrameRenderControl videoFrameRenderControl;
   private @MonotonicNonNull Format outputFormat;
   private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener;
   private @MonotonicNonNull HandlerWrapper handler;
@@ -241,11 +173,6 @@ public final class CompositingVideoSinkProvider
   private CompositingVideoSinkProvider(Builder builder) {
     this.context = builder.context;
     this.previewingVideoGraphFactory = checkStateNotNull(builder.previewingVideoGraphFactory);
-    videoFrameReleaseControl = checkStateNotNull(builder.videoFrameReleaseControl);
-    @SuppressWarnings("nullness:assignment")
-    VideoFrameRenderControl.@Initialized FrameRenderer thisRef = this;
-    videoFrameRenderControl =
-        new VideoFrameRenderControl(/* frameRenderer= */ thisRef, videoFrameReleaseControl);
     clock = Clock.DEFAULT;
     listener = VideoSink.Listener.NO_OP;
     listenerExecutor = NO_OP_EXECUTOR;
@@ -258,6 +185,7 @@ public final class CompositingVideoSinkProvider
   public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException {
     checkState(state == STATE_CREATED);
     checkStateNotNull(videoEffects);
+    checkState(videoFrameRenderControl != null && videoFrameReleaseControl != null);
 
     // Lazily initialize the handler here so it's initialized on the playback looper.
     handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null);
@@ -356,6 +284,14 @@ public final class CompositingVideoSinkProvider
         outputSurface, outputResolution.getWidth(), outputResolution.getHeight());
   }
 
+  @Override
+  public void setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl) {
+    checkState(!isInitialized());
+    this.videoFrameReleaseControl = videoFrameReleaseControl;
+    videoFrameRenderControl =
+        new VideoFrameRenderControl(/* frameRenderer= */ this, videoFrameReleaseControl);
+  }
+
   @Override
   public void clearOutputSurfaceInfo() {
     maybeSetOutputSurfaceInfo(
@@ -371,6 +307,7 @@ public final class CompositingVideoSinkProvider
   }
 
   @Override
+  @Nullable
   public VideoFrameReleaseControl getVideoFrameReleaseControl() {
     return videoFrameReleaseControl;
   }
@@ -386,7 +323,7 @@ public final class CompositingVideoSinkProvider
   @Override
   public void onOutputSizeChanged(int width, int height) {
     // We forward output size changes to render control even if we are still flushing.
-    videoFrameRenderControl.onOutputSizeChanged(width, height);
+    checkStateNotNull(videoFrameRenderControl).onOutputSizeChanged(width, height);
   }
 
   @Override
@@ -395,7 +332,8 @@ public final class CompositingVideoSinkProvider
       // Ignore available frames while the sink provider is flushing
       return;
     }
-    videoFrameRenderControl.onOutputFrameAvailableForRendering(presentationTimeUs);
+    checkStateNotNull(videoFrameRenderControl)
+        .onOutputFrameAvailableForRendering(presentationTimeUs);
   }
 
   @Override
@@ -471,7 +409,7 @@ public final class CompositingVideoSinkProvider
    */
   public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
     if (pendingFlushCount == 0) {
-      videoFrameRenderControl.render(positionUs, elapsedRealtimeUs);
+      checkStateNotNull(videoFrameRenderControl).render(positionUs, elapsedRealtimeUs);
     }
   }
 
@@ -502,23 +440,24 @@ public final class CompositingVideoSinkProvider
       // Update the surface on the video graph and the video frame release control together.
       SurfaceInfo surfaceInfo = surface != null ? new SurfaceInfo(surface, width, height) : null;
       videoGraph.setOutputSurfaceInfo(surfaceInfo);
-      videoFrameReleaseControl.setOutputSurface(surface);
+      checkNotNull(videoFrameReleaseControl).setOutputSurface(surface);
     }
   }
 
   private boolean isReady() {
-    return pendingFlushCount == 0 && videoFrameRenderControl.isReady();
+    return pendingFlushCount == 0 && checkStateNotNull(videoFrameRenderControl).isReady();
   }
 
   private boolean hasReleasedFrame(long presentationTimeUs) {
-    return pendingFlushCount == 0 && videoFrameRenderControl.hasReleasedFrame(presentationTimeUs);
+    return pendingFlushCount == 0
+        && checkStateNotNull(videoFrameRenderControl).hasReleasedFrame(presentationTimeUs);
   }
 
   private void flush() {
     pendingFlushCount++;
     // Flush the render control now to ensure it has no data, eg calling isReady() must return false
     // and render() should not render any frames.
-    videoFrameRenderControl.flush();
+    checkStateNotNull(videoFrameRenderControl).flush();
     // Finish flushing after handling pending video graph callbacks to ensure video size changes
     // reach the video render control.
     checkStateNotNull(handler).post(this::flushInternal);
@@ -533,15 +472,16 @@ public final class CompositingVideoSinkProvider
       throw new IllegalStateException(String.valueOf(pendingFlushCount));
     }
     // Flush the render control again.
-    videoFrameRenderControl.flush();
+    checkStateNotNull(videoFrameRenderControl).flush();
   }
 
   private void setPlaybackSpeed(float speed) {
-    videoFrameRenderControl.setPlaybackSpeed(speed);
+    checkStateNotNull(videoFrameRenderControl).setPlaybackSpeed(speed);
   }
 
   private void onStreamOffsetChange(long bufferPresentationTimeUs, long streamOffsetUs) {
-    videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs);
+    checkStateNotNull(videoFrameRenderControl)
+        .onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs);
   }
 
   private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) {

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java

@@ -393,17 +393,18 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
     this.context = context.getApplicationContext();
     eventDispatcher = new EventDispatcher(eventHandler, eventListener);
     if (videoSinkProvider == null) {
+      videoSinkProvider = new CompositingVideoSinkProvider.Builder(this.context).build();
+    }
+    if (videoSinkProvider.getVideoFrameReleaseControl() == null) {
       @SuppressWarnings("nullness:assignment")
       VideoFrameReleaseControl.@Initialized FrameTimingEvaluator thisRef = this;
-      videoSinkProvider =
+      videoSinkProvider.setVideoFrameReleaseControl(
-          new CompositingVideoSinkProvider.Builder(this.context)
-              .setVideoFrameReleaseControl(
           new VideoFrameReleaseControl(
-                      this.context, /* frameTimingEvaluator= */ thisRef, allowedJoiningTimeMs))
+              this.context, /* frameTimingEvaluator= */ thisRef, allowedJoiningTimeMs));
-              .build();
     }
     this.videoSinkProvider = videoSinkProvider;
-    this.videoFrameReleaseControl = this.videoSinkProvider.getVideoFrameReleaseControl();
+    this.videoFrameReleaseControl =
+        checkStateNotNull(this.videoSinkProvider.getVideoFrameReleaseControl());
     videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
     deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround();
     scalingMode = C.VIDEO_SCALING_MODE_DEFAULT;

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java

@@ -23,6 +23,7 @@ import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.Size;
 import androidx.media3.common.util.UnstableApi;
 import java.util.List;
+import org.checkerframework.checker.nullness.qual.Nullable;
 
 /** A provider of {@link VideoSink VideoSinks}. */
 @UnstableApi
@@ -70,6 +71,15 @@ public interface VideoSinkProvider {
    */
   void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution);
 
+  /**
+   * Sets the {@link VideoFrameReleaseControl} that will be used for releasing of video frames
+   * during rendering.
+   *
+   * <p>Must be called before, not after, the sink provider is {@linkplain #initialize(Format)
+   * initialized}.
+   */
+  void setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl);
+
   /**
    * Clears the set output surface info.
    *
@@ -83,8 +93,11 @@ public interface VideoSinkProvider {
   /**
    * Returns the {@link VideoFrameReleaseControl} that will be used for releasing of video frames
    * during rendering.
+   *
+   * <p>If this value is {@code null}, it must be {@linkplain #setVideoFrameReleaseControl set} to a
+   * non-null value before rendering begins.
    */
-  VideoFrameReleaseControl getVideoFrameReleaseControl();
+  @Nullable VideoFrameReleaseControl getVideoFrameReleaseControl();
 
   /**
    * Sets the {@link Clock} that the provider should use internally.

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java

@@ -18,7 +18,6 @@ package androidx.media3.exoplayer.video;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
 import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import android.content.Context;
@@ -45,12 +44,7 @@ public final class CompositingVideoSinkProviderTest {
   @Test
   public void builder_calledMultipleTimes_throws() {
     CompositingVideoSinkProvider.Builder builder =
-        new CompositingVideoSinkProvider.Builder(ApplicationProvider.getApplicationContext())
+        new CompositingVideoSinkProvider.Builder(ApplicationProvider.getApplicationContext());
-            .setVideoFrameReleaseControl(
-                new VideoFrameReleaseControl(
-                    ApplicationProvider.getApplicationContext(),
-                    mock(VideoFrameReleaseControl.FrameTimingEvaluator.class),
-                    /* allowedJoiningTimeMs= */ 0));
 
     builder.build();
 
@@ -169,12 +163,13 @@ public final class CompositingVideoSinkProviderTest {
             return false;
           }
         };
-    VideoFrameReleaseControl releaseControl =
+    CompositingVideoSinkProvider compositingVideoSinkProvider =
-        new VideoFrameReleaseControl(context, frameTimingEvaluator, /* allowedJoiningTimeMs= */ 0);
+        new CompositingVideoSinkProvider.Builder(context)
-    return new CompositingVideoSinkProvider.Builder(context)
             .setPreviewingVideoGraphFactory(new TestPreviewingVideoGraphFactory())
-        .setVideoFrameReleaseControl(releaseControl)
             .build();
+    compositingVideoSinkProvider.setVideoFrameReleaseControl(
+        new VideoFrameReleaseControl(context, frameTimingEvaluator, /* allowedJoiningTimeMs= */ 0));
+    return compositingVideoSinkProvider;
   }
 
   private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory {