From 836720ebfa3ca065c12e0e5be7184e07786ef63c Mon Sep 17 00:00:00 2001
From: huangdarwin <huangdarwin@google.com>
Date: Mon, 17 Jul 2023 14:50:14 +0100
Subject: [PATCH] Compositor: Use timestamps for compositing.

For now, just identical timestamps. A follow-up CL will allow for different
framerates. Prior tests already send identical timestamp frames, so no new
tests were added

PiperOrigin-RevId: 548676759
---
 .../media3/effect/VideoCompositor.java        | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java
index 230e62c319..f047e930c6 100644
--- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java
@@ -26,6 +26,7 @@ import android.opengl.GLES20;
 import androidx.annotation.GuardedBy;
 import androidx.annotation.IntRange;
 import androidx.annotation.Nullable;
+import androidx.media3.common.C;
 import androidx.media3.common.GlObjectsProvider;
 import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.VideoFrameProcessingException;
@@ -45,11 +46,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 /**
  * A basic VideoCompositor that takes in frames from exactly 2 input sources and combines it to one
  * output.
+ *
+ * <p>The first {@linkplain #registerInputSource registered source} will be the primary stream,
+ * which is used to determine the output frames' timestamps and dimensions.
  */
 @UnstableApi
 public final class VideoCompositor {
   // TODO: b/262694346 -  Flesh out this implementation by doing the following:
-  //  * Handle matched timestamps.
   //  * Handle mismatched timestamps
   //  * Before allowing customization of this class, add an interface, and rename this class to
   //    DefaultCompositor.
@@ -217,12 +220,22 @@ public final class VideoCompositor {
     if (outputTexturePool.freeTextureCount() == 0) {
       return false;
     }
-    // TODO: b/262694346 - Use timestamps to determine when to composite instead of number of
-    //  frames.
+    long compositeTimestampUs = C.TIME_UNSET;
     for (int inputId = 0; inputId < inputFrameInfos.size(); inputId++) {
-      if (checkNotNull(inputFrameInfos.get(inputId)).isEmpty()) {
+      Queue<InputFrameInfo> inputFrameInfoQueue = checkNotNull(inputFrameInfos.get(inputId));
+      if (inputFrameInfoQueue.isEmpty()) {
         return false;
       }
+
+      long inputTimestampUs = checkNotNull(inputFrameInfoQueue.peek()).presentationTimeUs;
+      if (inputId == 0) {
+        compositeTimestampUs = inputTimestampUs;
+      }
+      // TODO: b/262694346 - Allow for different frame-rates to be composited, by potentially
+      //  dropping some frames in non-primary streams.
+      if (inputTimestampUs != compositeTimestampUs) {
+        throw new IllegalStateException("Non-matched timestamps not yet supported.");
+      }
     }
     return true;
   }