Add FrameDropping GlEffect

PiperOrigin-RevId: 524030672

libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java

@@ -53,7 +53,7 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
   private final boolean useHdr;
 
   private GlObjectsProvider glObjectsProvider;
-  private InputListener inputListener;
+  protected InputListener inputListener;
   private OutputListener outputListener;
   private ErrorListener errorListener;
   private Executor errorListenerExecutor;

libraries/effect/src/main/java/androidx/media3/effect/FrameCacheGlShaderProgram.java

@@ -29,7 +29,7 @@ import java.io.IOException;
  *
  * <p>Implements {@link FrameCache}.
  */
-/* package */ final class FrameCacheGlShaderProgram extends BaseGlShaderProgram {
+/* package */ class FrameCacheGlShaderProgram extends BaseGlShaderProgram {
   private static final String VERTEX_SHADER_TRANSFORMATION_ES2_PATH =
       "shaders/vertex_shader_transformation_es2.glsl";
   private static final String FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH =

libraries/effect/src/main/java/androidx/media3/effect/FrameDropEffect.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.effect;
+
+import android.content.Context;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.UnstableApi;
+
+/** Drops frames to lower average frame rate to around {@code targetFrameRate}. */
+@UnstableApi
+public class FrameDropEffect implements GlEffect {
+
+  private final float targetFrameRate;
+
+  /**
+   * Creates an instance.
+   *
+   * @param targetFrameRate The number of frames per second the output video should roughly have.
+   */
+  public FrameDropEffect(float targetFrameRate) {
+    this.targetFrameRate = targetFrameRate;
+  }
+
+  @Override
+  public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr)
+      throws VideoFrameProcessingException {
+    return new FrameDroppingShaderProgram(context, useHdr, targetFrameRate);
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/FrameDroppingShaderProgram.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static java.lang.Math.abs;
+
+import android.content.Context;
+import androidx.annotation.Nullable;
+import androidx.media3.common.C;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.VideoFrameProcessingException;
+
+// TODO(b/227625363): Add tests for this file.
+/**
+ * Drops frames by only queuing input frames that are chosen by the frame dropping strategy.
+ *
+ * <p>The strategy used is to queue the current frame, x, with timestamp T_x if and only if one of
+ * the following is true:
+ *
+ * <ul>
+ *   <li>x is the first frame,
+ *   <li>(T_x - T_lastQueued) is closer to the target frame interval than (T_(x+1) - T_lastQueued)
+ * </ul>
+ *
+ * <p>Where T_lastQueued is the timestamp of the last queued frame and T_(x+1) is the timestamp of
+ * the next frame. The target frame interval is determined from {@code targetFps}.
+ */
+/* package */ final class FrameDroppingShaderProgram extends FrameCacheGlShaderProgram {
+  private final long targetFrameDeltaUs;
+
+  @Nullable private GlTextureInfo previousTexture;
+  private long previousPresentationTimeUs;
+  private long lastQueuedPresentationTimeUs;
+  private boolean isPreviousFrameFirstFrame;
+
+  /**
+   * Creates a new instance.
+   *
+   * @param context The {@link Context}.
+   * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
+   *     in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
+   * @param targetFps The number of frames per second the output video should roughly have.
+   */
+  public FrameDroppingShaderProgram(Context context, boolean useHdr, float targetFps)
+      throws VideoFrameProcessingException {
+    super(context, /* capacity= */ 1, useHdr);
+    this.targetFrameDeltaUs = (long) (C.MICROS_PER_SECOND / targetFps);
+    lastQueuedPresentationTimeUs = C.TIME_UNSET;
+    previousPresentationTimeUs = C.TIME_UNSET;
+  }
+
+  @Override
+  public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
+    if (previousTexture == null) {
+      super.queueInputFrame(inputTexture, presentationTimeUs);
+      lastQueuedPresentationTimeUs = presentationTimeUs;
+      isPreviousFrameFirstFrame = true;
+    } else if (shouldQueuePreviousFrame(presentationTimeUs)) {
+      super.queueInputFrame(checkNotNull(previousTexture), previousPresentationTimeUs);
+      lastQueuedPresentationTimeUs = previousPresentationTimeUs;
+    } else {
+      inputListener.onInputFrameProcessed(checkNotNull(previousTexture));
+      inputListener.onReadyToAcceptInputFrame();
+    }
+    previousTexture = inputTexture;
+    previousPresentationTimeUs = presentationTimeUs;
+  }
+
+  @Override
+  public void flush() {
+    super.flush();
+    lastQueuedPresentationTimeUs = C.TIME_UNSET;
+    previousPresentationTimeUs = C.TIME_UNSET;
+    previousTexture = null;
+  }
+
+  private boolean shouldQueuePreviousFrame(long currentPresentationTimeUs) {
+    if (isPreviousFrameFirstFrame) {
+      isPreviousFrameFirstFrame = false;
+      return false;
+    }
+
+    long previousFrameTimeDeltaUs = previousPresentationTimeUs - lastQueuedPresentationTimeUs;
+    long currentFrameTimeDeltaUs = currentPresentationTimeUs - lastQueuedPresentationTimeUs;
+
+    return abs(previousFrameTimeDeltaUs - targetFrameDeltaUs)
+        < abs(currentFrameTimeDeltaUs - targetFrameDeltaUs);
+  }
+}