Remove late frame dropping in FrameProcessor

Currently, a frame is dropped if it's requested release time is in the past.
This mode was added to support previewing. However, in normal ExoPlayer
playback, slightly late frames (<30ms late) are also rendered. On MediaCodec
side, this means calling `releaseOutputBuffer` with a release time in the
past.

PiperOrigin-RevId: 479615291

libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java

@@ -16,6 +16,7 @@
 package androidx.media3.common;
 
 import android.content.Context;
+import android.opengl.EGLExt;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.media3.common.util.UnstableApi;
@@ -174,10 +175,12 @@ public interface FrameProcessor {
    * false} using the {@link Factory} and should be called exactly once for each frame that becomes
    * {@linkplain Listener#onOutputFrameAvailable(long) available}.
    *
-   * @param releaseTimeNs The release time to use for the frame, in nanoseconds. Use {@link
+   * <p>The {@code releaseTimeNs} may be passed to {@link EGLExt#eglPresentationTimeANDROID}
-   *     #DROP_OUTPUT_FRAME} to drop the frame, or {@link #RELEASE_OUTPUT_FRAME_IMMEDIATELY} to
+   * depending on the implementation.
-   *     release the frame immediately. If {@code releaseTimeNs} is after {@link System#nanoTime()}
+   *
-   *     at the time of the release, the frame is also dropped.
+   * @param releaseTimeNs The release time to use for the frame, in nanoseconds. The release time
+   *     can be before of after the current system time. Use {@link #DROP_OUTPUT_FRAME} to drop the
+   *     frame, or {@link #RELEASE_OUTPUT_FRAME_IMMEDIATELY} to release the frame immediately.
    */
   void releaseOutputFrame(long releaseTimeNs);
 

libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java

@@ -158,7 +158,7 @@ public final class GlEffectsFrameProcessorFrameReleaseTest {
   }
 
   @Test
-  public void controlledFrameRelease_withLateFrame_dropsFrame() throws Exception {
+  public void controlledFrameRelease_withLateFrame_releasesFrame() throws Exception {
     long originalPresentationTimeUs = 1234;
     long releaseTimeBeforeCurrentTimeNs = System.nanoTime() - 345678;
     AtomicLong actualPresentationTimeUs = new AtomicLong();
@@ -175,7 +175,9 @@ public final class GlEffectsFrameProcessorFrameReleaseTest {
 
     assertThat(frameProcessingException.get()).isNull();
     assertThat(actualPresentationTimeUs.get()).isEqualTo(originalPresentationTimeUs);
-    assertThat(outputReleaseTimesNs).isEmpty();
+    assertThat(outputReleaseTimesNs).hasSize(1);
+    // The actual release time is determined by the FrameProcessor when releasing the frame.
+    assertThat(outputReleaseTimesNs.remove()).isAtLeast(releaseTimeBeforeCurrentTimeNs);
   }
 
   @Test

libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java

@@ -150,10 +150,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     frameProcessorListener.onOutputFrameAvailable(offsetPresentationTimeUs);
     if (releaseFramesAutomatically) {
       renderFrameToSurfaces(
-          inputTexture,
+          inputTexture, presentationTimeUs, /* releaseTimeNs= */ offsetPresentationTimeUs * 1000);
-          presentationTimeUs,
-          /* releaseTimeNs= */ offsetPresentationTimeUs * 1000,
-          /* dropLateFrame= */ false);
     } else {
       availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
     }
@@ -169,21 +166,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   @WorkerThread
   public void releaseOutputFrame(long releaseTimeNs) {
     checkState(!releaseFramesAutomatically);
-
-    boolean dropLateFrame = true;
-    if (releaseTimeNs == FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY) {
-      dropLateFrame = false;
-      releaseTimeNs = System.nanoTime();
-    } else if (releaseTimeNs == FrameProcessor.DROP_OUTPUT_FRAME) {
-      releaseTimeNs = C.TIME_UNSET;
-    }
-
     Pair<TextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
     renderFrameToSurfaces(
         /* inputTexture= */ oldestAvailableFrame.first,
         /* presentationTimeUs= */ oldestAvailableFrame.second,
-        releaseTimeNs,
+        releaseTimeNs);
-        dropLateFrame);
   }
 
   @Override
@@ -254,13 +241,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   private void renderFrameToSurfaces(
-      TextureInfo inputTexture,
+      TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) {
-      long presentationTimeUs,
-      long releaseTimeNs,
-      boolean dropLateFrame) {
     try {
-      maybeRenderFrameToOutputSurface(
+      maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
-          inputTexture, presentationTimeUs, releaseTimeNs, dropLateFrame);
     } catch (FrameProcessingException | GlUtil.GlException e) {
       frameProcessorListener.onFrameProcessingError(
           FrameProcessingException.from(e, presentationTimeUs));
@@ -270,10 +253,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   private synchronized void maybeRenderFrameToOutputSurface(
-      TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs, boolean dropLateFrame)
+      TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs)
       throws FrameProcessingException, GlUtil.GlException {
-    if (!ensureConfigured(inputTexture.width, inputTexture.height)) {
+    if (releaseTimeNs == FrameProcessor.DROP_OUTPUT_FRAME
-      return; // Drop frames when there is no output surface.
+        || !ensureConfigured(inputTexture.width, inputTexture.height)) {
+      return; // Drop frames when requested, or there is no output surface.
     }
 
     EGLSurface outputEglSurface = this.outputEglSurface;
@@ -289,10 +273,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     GlUtil.clearOutputFrame();
     matrixTextureProcessor.drawFrame(inputTexture.texId, presentationTimeUs);
 
-    if (dropLateFrame && System.nanoTime() > releaseTimeNs) {
+    EGLExt.eglPresentationTimeANDROID(
-      return;
+        eglDisplay,
-    }
+        outputEglSurface,
-    EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, releaseTimeNs);
+        releaseTimeNs == FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY
+            ? System.nanoTime()
+            : releaseTimeNs);
     EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
   }