Rollback of f5a6ecdda1

PiperOrigin-RevId: 558779068

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java

@@ -0,0 +1,643 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.video;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.os.Handler;
+import android.util.Pair;
+import android.view.Surface;
+import androidx.annotation.Nullable;
+import androidx.media3.common.C;
+import androidx.media3.common.ColorInfo;
+import androidx.media3.common.DebugViewProvider;
+import androidx.media3.common.Effect;
+import androidx.media3.common.Format;
+import androidx.media3.common.FrameInfo;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.SurfaceInfo;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.VideoFrameProcessor;
+import androidx.media3.common.VideoSize;
+import androidx.media3.common.util.Size;
+import androidx.media3.common.util.TimedValueQueue;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.common.util.Util;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executor;
+import org.checkerframework.checker.initialization.qual.Initialized;
+import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
+/** Handles composition of video sinks. */
+@UnstableApi
+/* package */ final class CompositingVideoSinkProvider {
+
+  private final Context context;
+  private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
+  private final VideoSink.RenderControl renderControl;
+
+  @Nullable private VideoSinkImpl videoSinkImpl;
+  @Nullable private List<Effect> videoEffects;
+  @Nullable private VideoFrameMetadataListener videoFrameMetadataListener;
+  private boolean released;
+
+  /** Creates a new instance. */
+  public CompositingVideoSinkProvider(
+      Context context,
+      VideoFrameProcessor.Factory videoFrameProcessorFactory,
+      VideoSink.RenderControl renderControl) {
+    this.context = context;
+    this.videoFrameProcessorFactory = videoFrameProcessorFactory;
+    this.renderControl = renderControl;
+  }
+
+  /**
+   * Initializes the provider for video frame processing. Can be called up to one time and only
+   * after video effects are {@linkplain #setVideoEffects(List) set}.
+   *
+   * @param sourceFormat The format of the compressed video.
+   * @throws VideoSink.VideoSinkException If enabling the provider failed.
+   */
+  public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException {
+    checkState(!released && videoSinkImpl == null);
+    checkStateNotNull(videoEffects);
+
+    try {
+      videoSinkImpl =
+          new VideoSinkImpl(context, videoFrameProcessorFactory, renderControl, sourceFormat);
+      if (videoFrameMetadataListener != null) {
+        videoSinkImpl.setVideoFrameMetadataListener(videoFrameMetadataListener);
+      }
+    } catch (VideoFrameProcessingException e) {
+      throw new VideoSink.VideoSinkException(e, sourceFormat);
+    }
+  }
+
+  /** Returns whether this provider is initialized for frame processing. */
+  public boolean isInitialized() {
+    return videoSinkImpl != null;
+  }
+
+  /** Releases the sink provider. */
+  public void release() {
+    if (released) {
+      return;
+    }
+
+    if (videoSinkImpl != null) {
+      videoSinkImpl.release();
+      videoSinkImpl = null;
+    }
+    released = true;
+  }
+
+  /** Returns a {@link VideoSink} to forward video frames for processing. */
+  public VideoSink getSink() {
+    return checkStateNotNull(videoSinkImpl);
+  }
+
+  /** Sets video effects on this provider. */
+  public void setVideoEffects(List<Effect> videoEffects) {
+    this.videoEffects = videoEffects;
+    if (isInitialized()) {
+      checkStateNotNull(videoSinkImpl).setVideoEffects(videoEffects);
+    }
+  }
+
+  /**
+   * Sets the offset, in microseconds, that is added to the video frames presentation timestamps
+   * from the player.
+   *
+   * <p>Must be called after the sink provider is {@linkplain #initialize(Format) initialized}.
+   */
+  public void setStreamOffsetUs(long streamOffsetUs) {
+    checkStateNotNull(videoSinkImpl).setStreamOffsetUs(streamOffsetUs);
+  }
+
+  /**
+   * Sets the output surface info.
+   *
+   * <p>Must be called after the sink provider is {@linkplain #initialize(Format) initialized}.
+   */
+  public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
+    checkStateNotNull(videoSinkImpl).setOutputSurfaceInfo(outputSurface, outputResolution);
+  }
+
+  /**
+   * Clears the set output surface info.
+   *
+   * <p>Must be called after the sink provider is {@linkplain #initialize(Format) initialized}.
+   */
+  public void clearOutputSurfaceInfo() {
+    checkStateNotNull(videoSinkImpl).clearOutputSurfaceInfo();
+  }
+
+  /** Sets a {@link VideoFrameMetadataListener} which is used in the returned {@link VideoSink}. */
+  public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener) {
+    this.videoFrameMetadataListener = videoFrameMetadataListener;
+    if (isInitialized()) {
+      checkStateNotNull(videoSinkImpl).setVideoFrameMetadataListener(videoFrameMetadataListener);
+    }
+  }
+
+  private static final class VideoSinkImpl implements VideoSink, VideoFrameProcessor.Listener {
+
+    private final Context context;
+    private final RenderControl renderControl;
+    private final VideoFrameProcessor videoFrameProcessor;
+    // TODO b/293447478 - Use a queue for primitive longs to avoid the cost of boxing to Long.
+    private final ArrayDeque<Long> processedFramesBufferTimestampsUs;
+    private final TimedValueQueue<Long> streamOffsets;
+    private final TimedValueQueue<VideoSize> videoSizeChanges;
+    private final Handler handler;
+    private final int videoFrameProcessorMaxPendingFrameCount;
+    private final ArrayList<Effect> videoEffects;
+    @Nullable private final Effect rotationEffect;
+
+    private VideoSink.@MonotonicNonNull Listener listener;
+    private @MonotonicNonNull Executor listenerExecutor;
+    @Nullable private VideoFrameMetadataListener videoFrameMetadataListener;
+    @Nullable private Format inputFormat;
+    @Nullable private Pair<Surface, Size> currentSurfaceAndSize;
+
+    /**
+     * Whether the last frame of the current stream is decoded and registered to {@link
+     * VideoFrameProcessor}.
+     */
+    private boolean registeredLastFrame;
+
+    /**
+     * Whether the last frame of the current stream is processed by the {@link VideoFrameProcessor}.
+     */
+    private boolean processedLastFrame;
+
+    /** Whether the last frame of the current stream is released to the output {@link Surface}. */
+    private boolean releasedLastFrame;
+
+    private long lastCodecBufferPresentationTimestampUs;
+    private VideoSize processedFrameSize;
+    private VideoSize reportedVideoSize;
+    private boolean pendingVideoSizeChange;
+    private boolean renderedFirstFrame;
+    private long inputStreamOffsetUs;
+    private boolean pendingInputStreamOffsetChange;
+    private long outputStreamOffsetUs;
+    private float playbackSpeed;
+
+    // TODO b/292111083 - Remove the field and trigger the callback on every video size change.
+    private boolean onVideoSizeChangedCalled;
+
+    /** Creates a new instance. */
+    public VideoSinkImpl(
+        Context context,
+        VideoFrameProcessor.Factory videoFrameProcessorFactory,
+        RenderControl renderControl,
+        Format sourceFormat)
+        throws VideoFrameProcessingException {
+      this.context = context;
+      this.renderControl = renderControl;
+      processedFramesBufferTimestampsUs = new ArrayDeque<>();
+      streamOffsets = new TimedValueQueue<>();
+      videoSizeChanges = new TimedValueQueue<>();
+      // TODO b/226330223 - Investigate increasing frame count when frame dropping is
+      //  allowed.
+      // TODO b/278234847 - Evaluate whether limiting frame count when frame dropping is not allowed
+      //  reduces decoder timeouts, and consider restoring.
+      videoFrameProcessorMaxPendingFrameCount =
+          Util.getMaxPendingFramesCountForMediaCodecDecoders(context);
+      lastCodecBufferPresentationTimestampUs = C.TIME_UNSET;
+      processedFrameSize = VideoSize.UNKNOWN;
+      reportedVideoSize = VideoSize.UNKNOWN;
+      playbackSpeed = 1f;
+
+      // Playback thread handler.
+      handler = Util.createHandlerForCurrentLooper();
+
+      Pair<ColorInfo, ColorInfo> inputAndOutputColorInfos =
+          experimentalGetVideoFrameProcessorColorConfiguration(sourceFormat.colorInfo);
+
+      @SuppressWarnings("nullness:assignment")
+      @Initialized
+      VideoSinkImpl thisRef = this;
+      videoFrameProcessor =
+          videoFrameProcessorFactory.create(
+              context,
+              DebugViewProvider.NONE,
+              inputAndOutputColorInfos.first,
+              inputAndOutputColorInfos.second,
+              /* renderFramesAutomatically= */ false,
+              /* listenerExecutor= */ handler::post,
+              thisRef);
+      if (currentSurfaceAndSize != null) {
+        Size outputSurfaceSize = currentSurfaceAndSize.second;
+        videoFrameProcessor.setOutputSurfaceInfo(
+            new SurfaceInfo(
+                currentSurfaceAndSize.first,
+                outputSurfaceSize.getWidth(),
+                outputSurfaceSize.getHeight()));
+      }
+      videoEffects = new ArrayList<>();
+      // MediaCodec applies rotation after API 21
+      rotationEffect =
+          Util.SDK_INT < 21 && sourceFormat.rotationDegrees != 0
+              ? ScaleAndRotateAccessor.createRotationEffect(sourceFormat.rotationDegrees)
+              : null;
+    }
+
+    // VideoSink impl
+
+    @Override
+    public void flush() {
+      videoFrameProcessor.flush();
+      processedFramesBufferTimestampsUs.clear();
+      streamOffsets.clear();
+      handler.removeCallbacksAndMessages(/* token= */ null);
+      renderedFirstFrame = false;
+      if (registeredLastFrame) {
+        registeredLastFrame = false;
+        processedLastFrame = false;
+        releasedLastFrame = false;
+      }
+    }
+
+    @Override
+    public boolean isReady() {
+      return renderedFirstFrame;
+    }
+
+    @Override
+    public boolean isEnded() {
+      return releasedLastFrame;
+    }
+
+    @Override
+    public void registerInputStream(@InputType int inputType, Format format) {
+      if (inputType != INPUT_TYPE_SURFACE) {
+        throw new UnsupportedOperationException("Unsupported input type " + inputType);
+      }
+      this.inputFormat = format;
+      maybeRegisterInputStream();
+
+      if (registeredLastFrame) {
+        registeredLastFrame = false;
+        processedLastFrame = false;
+        releasedLastFrame = false;
+      }
+    }
+
+    @Override
+    public void setListener(Listener listener, Executor executor) {
+      if (Util.areEqual(this.listener, listener)) {
+        checkState(Util.areEqual(listenerExecutor, executor));
+        return;
+      }
+      this.listener = listener;
+      this.listenerExecutor = executor;
+    }
+
+    @Override
+    public boolean isFrameDropAllowedOnInput() {
+      return Util.isFrameDropAllowedOnSurfaceInput(context);
+    }
+
+    @Override
+    public Surface getInputSurface() {
+      return videoFrameProcessor.getInputSurface();
+    }
+
+    @Override
+    public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame) {
+      checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET);
+      if (videoFrameProcessor.getPendingInputFrameCount()
+          >= videoFrameProcessorMaxPendingFrameCount) {
+        return C.TIME_UNSET;
+      }
+      videoFrameProcessor.registerInputFrame();
+      // The sink takes in frames with monotonically increasing, non-offset frame
+      // timestamps. That is, with two ten-second long videos, the first frame of the second video
+      // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the
+      // timestamp of the said frame would be 0s, but the streamOffset is incremented 10s to include
+      // the duration of the first video. Thus this correction is need to correct for the different
+      // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor.
+      long bufferPresentationTimeUs = framePresentationTimeUs + inputStreamOffsetUs;
+      if (pendingInputStreamOffsetChange) {
+        streamOffsets.add(bufferPresentationTimeUs, inputStreamOffsetUs);
+        pendingInputStreamOffsetChange = false;
+      }
+      if (isLastFrame) {
+        registeredLastFrame = true;
+        lastCodecBufferPresentationTimestampUs = bufferPresentationTimeUs;
+      }
+      return bufferPresentationTimeUs * 1000;
+    }
+
+    @Override
+    public boolean queueBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void render(long positionUs, long elapsedRealtimeUs) {
+      while (!processedFramesBufferTimestampsUs.isEmpty()) {
+        long bufferPresentationTimeUs = checkNotNull(processedFramesBufferTimestampsUs.peek());
+        // check whether this buffer comes with a new stream offset.
+        if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) {
+          renderedFirstFrame = false;
+        }
+        long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
+        boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1;
+        long frameRenderTimeNs =
+            renderControl.getFrameRenderTimeNs(
+                bufferPresentationTimeUs, positionUs, elapsedRealtimeUs, playbackSpeed);
+        if (frameRenderTimeNs == RenderControl.RENDER_TIME_TRY_AGAIN_LATER) {
+          return;
+        } else if (framePresentationTimeUs == RenderControl.RENDER_TIME_DROP) {
+          // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
+          //  VideoFrameProcessor input frames in this case.
+          releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame);
+          continue;
+        }
+        renderControl.onNextFrame(bufferPresentationTimeUs);
+        if (videoFrameMetadataListener != null) {
+          videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
+              framePresentationTimeUs,
+              frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY
+                  ? System.nanoTime()
+                  : frameRenderTimeNs,
+              checkNotNull(inputFormat),
+              /* mediaFormat= */ null);
+        }
+        releaseProcessedFrameInternal(
+            frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY
+                ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
+                : frameRenderTimeNs,
+            isLastFrame);
+
+        maybeNotifyVideoSizeChanged(bufferPresentationTimeUs);
+      }
+    }
+
+    @Override
+    public void setPlaybackSpeed(float speed) {
+      checkArgument(speed >= 0.0);
+      this.playbackSpeed = speed;
+    }
+
+    // VideoFrameProcessor.Listener impl
+
+    @Override
+    public void onOutputSizeChanged(int width, int height) {
+      VideoSize newVideoSize = new VideoSize(width, height);
+      if (!processedFrameSize.equals(newVideoSize)) {
+        processedFrameSize = newVideoSize;
+        pendingVideoSizeChange = true;
+      }
+    }
+
+    @Override
+    public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
+      if (pendingVideoSizeChange) {
+        videoSizeChanges.add(presentationTimeUs, processedFrameSize);
+        pendingVideoSizeChange = false;
+      }
+      if (registeredLastFrame) {
+        checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
+      }
+      processedFramesBufferTimestampsUs.add(presentationTimeUs);
+      // TODO b/257464707 - Support extensively modified media.
+      if (registeredLastFrame && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
+        processedLastFrame = true;
+      }
+    }
+
+    @Override
+    public void onError(VideoFrameProcessingException exception) {
+      if (listener == null || listenerExecutor == null) {
+        return;
+      }
+      listenerExecutor.execute(
+          () -> {
+            if (listener != null) {
+              listener.onError(
+                  /* videoSink= */ this,
+                  new VideoSink.VideoSinkException(
+                      exception,
+                      new Format.Builder()
+                          .setSampleMimeType(MimeTypes.VIDEO_RAW)
+                          .setWidth(processedFrameSize.width)
+                          .setHeight(processedFrameSize.height)
+                          .build()));
+            }
+          });
+    }
+
+    @Override
+    public void onEnded() {
+      throw new IllegalStateException();
+    }
+
+    // Other methods
+
+    public void release() {
+      videoFrameProcessor.release();
+      handler.removeCallbacksAndMessages(/* token= */ null);
+      streamOffsets.clear();
+      processedFramesBufferTimestampsUs.clear();
+      renderedFirstFrame = false;
+    }
+
+    /** Sets the {@linkplain Effect video effects}. */
+    public void setVideoEffects(List<Effect> videoEffects) {
+      this.videoEffects.clear();
+      this.videoEffects.addAll(videoEffects);
+      maybeRegisterInputStream();
+    }
+
+    public void setStreamOffsetUs(long streamOffsetUs) {
+      pendingInputStreamOffsetChange = inputStreamOffsetUs != streamOffsetUs;
+      inputStreamOffsetUs = streamOffsetUs;
+    }
+
+    public void setVideoFrameMetadataListener(
+        VideoFrameMetadataListener videoFrameMetadataListener) {
+      this.videoFrameMetadataListener = videoFrameMetadataListener;
+    }
+
+    private void maybeRegisterInputStream() {
+      if (inputFormat == null) {
+        return;
+      }
+
+      ArrayList<Effect> effects = new ArrayList<>();
+      if (rotationEffect != null) {
+        effects.add(rotationEffect);
+      }
+      effects.addAll(videoEffects);
+      Format inputFormat = checkNotNull(this.inputFormat);
+      videoFrameProcessor.registerInputStream(
+          VideoFrameProcessor.INPUT_TYPE_SURFACE,
+          effects,
+          new FrameInfo.Builder(inputFormat.width, inputFormat.height)
+              .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio)
+              .build());
+    }
+
+    /**
+     * Returns a {@link Pair} of {@linkplain ColorInfo input color} and {@linkplain ColorInfo output
+     * color} to configure the {@code VideoFrameProcessor}.
+     */
+    private static Pair<ColorInfo, ColorInfo> experimentalGetVideoFrameProcessorColorConfiguration(
+        @Nullable ColorInfo inputColorInfo) {
+      // TODO b/279163661 - Remove this method after VideoFrameProcessor supports texture ID
+      //  input/output.
+      // explicit check for nullness
+      if (inputColorInfo == null || !ColorInfo.isTransferHdr(inputColorInfo)) {
+        return Pair.create(ColorInfo.SDR_BT709_LIMITED, ColorInfo.SDR_BT709_LIMITED);
+      }
+
+      if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) {
+        // SurfaceView only supports BT2020 PQ input, converting HLG to PQ.
+        return Pair.create(
+            inputColorInfo,
+            inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build());
+      }
+
+      return Pair.create(inputColorInfo, inputColorInfo);
+    }
+
+    /**
+     * Sets the output surface info.
+     *
+     * @param outputSurface The {@link Surface} to which {@link VideoFrameProcessor} outputs.
+     * @param outputResolution The {@link Size} of the output resolution.
+     */
+    public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
+      if (currentSurfaceAndSize != null
+          && currentSurfaceAndSize.first.equals(outputSurface)
+          && currentSurfaceAndSize.second.equals(outputResolution)) {
+        return;
+      }
+      renderedFirstFrame =
+          currentSurfaceAndSize == null || currentSurfaceAndSize.first.equals(outputSurface);
+      currentSurfaceAndSize = Pair.create(outputSurface, outputResolution);
+      videoFrameProcessor.setOutputSurfaceInfo(
+          new SurfaceInfo(
+              outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
+    }
+
+    /** Clears the set output surface info. */
+    public void clearOutputSurfaceInfo() {
+      videoFrameProcessor.setOutputSurfaceInfo(null);
+      currentSurfaceAndSize = null;
+      renderedFirstFrame = false;
+    }
+
+    private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) {
+      boolean updatedOffset = false;
+      @Nullable Long newOutputStreamOffsetUs = streamOffsets.pollFloor(bufferPresentationTimeUs);
+      if (newOutputStreamOffsetUs != null && newOutputStreamOffsetUs != outputStreamOffsetUs) {
+        outputStreamOffsetUs = newOutputStreamOffsetUs;
+        updatedOffset = true;
+      }
+      return updatedOffset;
+    }
+
+    private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) {
+      videoFrameProcessor.renderOutputFrame(releaseTimeNs);
+      processedFramesBufferTimestampsUs.remove();
+      if (releaseTimeNs != VideoFrameProcessor.DROP_OUTPUT_FRAME) {
+        renderControl.onFrameRendered();
+        if (!renderedFirstFrame) {
+          if (listener != null) {
+            checkNotNull(listenerExecutor)
+                .execute(() -> checkNotNull(listener).onFirstFrameRendered(this));
+          }
+          renderedFirstFrame = true;
+        }
+      }
+      if (isLastFrame) {
+        releasedLastFrame = true;
+      }
+    }
+
+    private void maybeNotifyVideoSizeChanged(long bufferPresentationTimeUs) {
+      if (onVideoSizeChangedCalled || listener == null) {
+        return;
+      }
+
+      @Nullable VideoSize videoSize = videoSizeChanges.pollFloor(bufferPresentationTimeUs);
+      if (videoSize == null) {
+        return;
+      }
+
+      if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) {
+        reportedVideoSize = videoSize;
+        checkNotNull(listenerExecutor)
+            .execute(() -> checkNotNull(listener).onVideoSizeChanged(this, videoSize));
+      }
+      onVideoSizeChangedCalled = true;
+    }
+
+    private static final class ScaleAndRotateAccessor {
+      private static @MonotonicNonNull Constructor<?>
+          scaleAndRotateTransformationBuilderConstructor;
+      private static @MonotonicNonNull Method setRotationMethod;
+      private static @MonotonicNonNull Method buildScaleAndRotateTransformationMethod;
+
+      public static Effect createRotationEffect(float rotationDegrees) {
+        try {
+          prepare();
+          Object builder = scaleAndRotateTransformationBuilderConstructor.newInstance();
+          setRotationMethod.invoke(builder, rotationDegrees);
+          return (Effect) checkNotNull(buildScaleAndRotateTransformationMethod.invoke(builder));
+        } catch (Exception e) {
+          throw new IllegalStateException(e);
+        }
+      }
+
+      @EnsuresNonNull({
+        "scaleAndRotateTransformationBuilderConstructor",
+        "setRotationMethod",
+        "buildScaleAndRotateTransformationMethod"
+      })
+      private static void prepare() throws NoSuchMethodException, ClassNotFoundException {
+        if (scaleAndRotateTransformationBuilderConstructor == null
+            || setRotationMethod == null
+            || buildScaleAndRotateTransformationMethod == null) {
+          // TODO: b/284964524 - Add LINT and proguard checks for media3.effect reflection.
+          Class<?> scaleAndRotateTransformationBuilderClass =
+              Class.forName("androidx.media3.effect.ScaleAndRotateTransformation$Builder");
+          scaleAndRotateTransformationBuilderConstructor =
+              scaleAndRotateTransformationBuilderClass.getConstructor();
+          setRotationMethod =
+              scaleAndRotateTransformationBuilderClass.getMethod("setRotationDegrees", float.class);
+          buildScaleAndRotateTransformationMethod =
+              scaleAndRotateTransformationBuilderClass.getMethod("build");
+        }
+      }
+    }
+  }
+}

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java

@@ -33,7 +33,7 @@ import java.util.concurrent.Executor;
 
 /** A sink that consumes decoded video frames. */
 @UnstableApi
-/*package */ interface VideoSink {
+/* package */ interface VideoSink {
 
   /** Thrown by {@link VideoSink} implementations. */
   final class VideoSinkException extends Exception {
@@ -61,6 +61,46 @@ import java.util.concurrent.Executor;
     void onError(VideoSink videoSink, VideoSinkException videoSinkException);
   }
 
+  /** Controls the rendering of video frames. */
+  interface RenderControl {
+    /** Signals a frame must be rendered immediately. */
+    long RENDER_TIME_IMMEDIATELY = -1;
+
+    /** Signals a frame must be dropped. */
+    long RENDER_TIME_DROP = -2;
+
+    /** Signals that a frame should not be rendered yet. */
+    long RENDER_TIME_TRY_AGAIN_LATER = -3;
+
+    /**
+     * Returns the render timestamp, in nanoseconds, associated with this video frames or one of the
+     * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not
+     * rendered yet.
+     *
+     * @param presentationTimeUs The presentation time of the video frame, in microseconds.
+     * @param positionUs The current playback position, in microseconds.
+     * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
+     *     taken approximately at the time the playback position was {@code positionUs}.
+     * @param playbackSpeed The current playback speed.
+     * @return The render timestamp, in nanoseconds, associated with this frame, or one of the
+     *     {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not
+     *     rendered yet.
+     */
+    long getFrameRenderTimeNs(
+        long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed);
+
+    /**
+     * Informs the rendering control that a video frame will be rendered. Call this method before
+     * rendering a frame.
+     *
+     * @param presentationTimeUs The frame's presentation time, in microseconds.
+     */
+    void onNextFrame(long presentationTimeUs);
+
+    /** Informs the rendering control that a video frame was rendered. */
+    void onFrameRendered();
+  }
+
   /**
    * Specifies how the input frames are made available to the video sink. One of {@link
    * #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}.

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.video;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import android.content.Context;
+import androidx.media3.common.ColorInfo;
+import androidx.media3.common.DebugViewProvider;
+import androidx.media3.common.Format;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.VideoFrameProcessor;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
+import java.util.concurrent.Executor;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+
+/** Unit test for {@link CompositingVideoSinkProvider}. */
+@RunWith(AndroidJUnit4.class)
+public final class CompositingVideoSinkProviderTest {
+
+  @Test
+  public void initialize() throws VideoSink.VideoSinkException {
+    CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
+    provider.setVideoEffects(ImmutableList.of());
+
+    provider.initialize(new Format.Builder().build());
+
+    assertThat(provider.isInitialized()).isTrue();
+  }
+
+  @Test
+  public void initialize_withoutEffects_throws() {
+    CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
+
+    assertThrows(
+        IllegalStateException.class,
+        () -> provider.initialize(new Format.Builder().setWidth(640).setHeight(480).build()));
+  }
+
+  @Test
+  public void initialize_calledTwice_throws() throws VideoSink.VideoSinkException {
+    CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
+    provider.setVideoEffects(ImmutableList.of());
+    provider.initialize(new Format.Builder().build());
+
+    assertThrows(
+        IllegalStateException.class, () -> provider.initialize(new Format.Builder().build()));
+  }
+
+  @Test
+  public void initialize_afterRelease_throws() throws VideoSink.VideoSinkException {
+    CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
+    provider.setVideoEffects(ImmutableList.of());
+    Format format = new Format.Builder().build();
+
+    provider.initialize(format);
+    provider.release();
+
+    assertThrows(IllegalStateException.class, () -> provider.initialize(format));
+  }
+
+  @Test
+  public void registerInputStream_withInputTypeBitmap_throws() throws VideoSink.VideoSinkException {
+    CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
+    provider.setVideoEffects(ImmutableList.of());
+    provider.initialize(new Format.Builder().build());
+    VideoSink videoSink = provider.getSink();
+
+    assertThrows(
+        UnsupportedOperationException.class,
+        () ->
+            videoSink.registerInputStream(
+                VideoSink.INPUT_TYPE_BITMAP, new Format.Builder().build()));
+  }
+
+  @Test
+  public void setOutputStreamOffsetUs_frameReleaseTimesAreAdjusted()
+      throws VideoSink.VideoSinkException {
+    CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
+    provider.setVideoEffects(ImmutableList.of());
+    provider.initialize(new Format.Builder().build());
+    VideoSink videoSink = provider.getSink();
+    videoSink.registerInputStream(
+        VideoSink.INPUT_TYPE_SURFACE, new Format.Builder().setWidth(640).setHeight(480).build());
+
+    assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false)).isEqualTo(0);
+    provider.setStreamOffsetUs(1_000);
+    assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false))
+        .isEqualTo(1_000_000);
+    provider.setStreamOffsetUs(2_000);
+    assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false))
+        .isEqualTo(2_000_000);
+  }
+
+  @Test
+  public void setListener_calledTwiceWithDifferentExecutor_throws()
+      throws VideoSink.VideoSinkException {
+    CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
+    provider.setVideoEffects(ImmutableList.of());
+    provider.initialize(new Format.Builder().build());
+    VideoSink videoSink = provider.getSink();
+    VideoSink.Listener listener = Mockito.mock(VideoSink.Listener.class);
+
+    videoSink.setListener(listener, /* executor= */ command -> {});
+
+    assertThrows(
+        IllegalStateException.class,
+        () -> videoSink.setListener(listener, /* executor= */ command -> {}));
+  }
+
+  private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() {
+    VideoFrameProcessor.Factory factory = new TestVideoFrameProcessorFactory();
+    VideoSink.RenderControl renderControl = new TestRenderControl();
+    return new CompositingVideoSinkProvider(
+        ApplicationProvider.getApplicationContext(), factory, renderControl);
+  }
+
+  private static class TestVideoFrameProcessorFactory implements VideoFrameProcessor.Factory {
+    // Using a mock but we don't assert mock interactions. If needed to assert interactions, we
+    // should a fake instead.
+    private final VideoFrameProcessor videoFrameProcessor = Mockito.mock(VideoFrameProcessor.class);
+
+    @Override
+    public VideoFrameProcessor create(
+        Context context,
+        DebugViewProvider debugViewProvider,
+        ColorInfo inputColorInfo,
+        ColorInfo outputColorInfo,
+        boolean renderFramesAutomatically,
+        Executor listenerExecutor,
+        VideoFrameProcessor.Listener listener)
+        throws VideoFrameProcessingException {
+      return videoFrameProcessor;
+    }
+  }
+
+  private static class TestRenderControl implements VideoSink.RenderControl {
+
+    @Override
+    public long getFrameRenderTimeNs(
+        long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) {
+      return presentationTimeUs;
+    }
+
+    @Override
+    public void onNextFrame(long presentationTimeUs) {}
+
+    @Override
+    public void onFrameRendered() {}
+  }
+}

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java

@@ -51,6 +51,7 @@ import androidx.media3.common.VideoSize;
 import androidx.media3.common.util.Clock;
 import androidx.media3.decoder.CryptoInfo;
 import androidx.media3.exoplayer.DecoderCounters;
+import androidx.media3.exoplayer.ExoPlaybackException;
 import androidx.media3.exoplayer.LoadingInfo;
 import androidx.media3.exoplayer.Renderer;
 import androidx.media3.exoplayer.RendererCapabilities;
@@ -81,6 +82,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -219,7 +221,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ true,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
 
     mediaCodecVideoRenderer.start();
     mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
@@ -527,7 +529,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ true,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
     mediaCodecVideoRenderer.setCurrentStreamFinal();
     mediaCodecVideoRenderer.start();
 
@@ -573,7 +575,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ false,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
     mediaCodecVideoRenderer.start();
     mediaCodecVideoRenderer.render(/* positionUs= */ 0, msToUs(SystemClock.elapsedRealtime()));
     ShadowSystemClock.advanceBy(10, TimeUnit.MILLISECONDS);
@@ -628,7 +630,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ true,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
 
     mediaCodecVideoRenderer.start();
     mediaCodecVideoRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
@@ -668,7 +670,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ true,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
     for (int i = 0; i < 10; i++) {
       mediaCodecVideoRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
     }
@@ -698,7 +700,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ false,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
     for (int i = 0; i < 10; i++) {
       mediaCodecVideoRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
     }
@@ -727,7 +729,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ false,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
     mediaCodecVideoRenderer.start();
     for (int i = 0; i < 10; i++) {
       mediaCodecVideoRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
@@ -801,7 +803,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ true,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
     mediaCodecVideoRenderer.start();
 
     boolean replacedStream = false;
@@ -863,7 +865,7 @@ public class MediaCodecVideoRendererTest {
         /* joining= */ false,
         /* mayRenderStartOfStream= */ true,
         /* startPositionUs= */ 0,
-        /* offsetUs */ 0);
+        /* offsetUs= */ 0);
 
     boolean replacedStream = false;
     for (int i = 0; i < 10; i++) {
@@ -1211,6 +1213,70 @@ public class MediaCodecVideoRendererTest {
         .isEqualTo(RendererCapabilities.DECODER_SUPPORT_PRIMARY);
   }
 
+  @Test
+  public void setVideoOutput_withNoEffects_updatesSurfaceOnMediaCodec()
+      throws ExoPlaybackException {
+    ArrayList<Surface> surfacesSet = new ArrayList<>();
+    MediaCodecAdapter.Factory codecAdapterFactory =
+        configuration ->
+            new ForwardingSynchronousMediaCodecAdapter(
+                new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration)) {
+              @Override
+              public void setOutputSurface(Surface surface) {
+                super.setOutputSurface(surface);
+                surfacesSet.add(surface);
+              }
+            };
+    MediaCodecVideoRenderer mediaCodecVideoRenderer =
+        new MediaCodecVideoRenderer(
+            ApplicationProvider.getApplicationContext(),
+            codecAdapterFactory,
+            mediaCodecSelector,
+            /* allowedJoiningTimeMs= */ 0,
+            /* enableDecoderFallback= */ false,
+            /* eventHandler= */ new Handler(testMainLooper),
+            /* eventListener= */ eventListener,
+            /* maxDroppedFramesToNotify= */ 1,
+            /* assumedMinimumCodecOperatingRate= */ 30) {
+          @Override
+          protected @Capabilities int supportsFormat(
+              MediaCodecSelector mediaCodecSelector, Format format) {
+            return RendererCapabilities.create(C.FORMAT_HANDLED);
+          }
+        };
+    mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
+    mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
+    FakeSampleStream fakeSampleStream =
+        new FakeSampleStream(
+            new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+            /* mediaSourceEventDispatcher= */ null,
+            DrmSessionManager.DRM_UNSUPPORTED,
+            new DrmSessionEventListener.EventDispatcher(),
+            /* initialFormat= */ VIDEO_H264,
+            /* fakeSampleStreamItems= */ ImmutableList.of(
+                oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
+                oneByteSample(/* timeUs= */ 50_000), // Late buffer.
+                oneByteSample(/* timeUs= */ 100_000), // Last buffer.
+                END_OF_STREAM_ITEM));
+    fakeSampleStream.writeData(/* startPositionUs= */ 0);
+    mediaCodecVideoRenderer.enable(
+        RendererConfiguration.DEFAULT,
+        new Format[] {VIDEO_H264},
+        fakeSampleStream,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ true,
+        /* startPositionUs= */ 0,
+        /* offsetUs= */ 0);
+    mediaCodecVideoRenderer.start();
+    mediaCodecVideoRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
+
+    Surface newSurface = new Surface(new SurfaceTexture(/* texName= */ 0));
+    mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, newSurface);
+
+    assertThat(surfacesSet).containsExactly(newSurface);
+  }
+
   private static CodecCapabilities createCodecCapabilities(int profile, int level) {
     CodecCapabilities capabilities = new CodecCapabilities();
     capabilities.profileLevels = new CodecProfileLevel[] {new CodecProfileLevel()};