Add GlEffect that asynchronously adjusts input presentation times

Single asset export only effect.

PiperOrigin-RevId: 616114153

libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java

@@ -73,7 +73,8 @@ import java.util.concurrent.atomic.AtomicReference;
       maybeSaveTestBitmap(
           testId, String.valueOf(presentationTimeUs), actualBitmap, /* path= */ null);
       float averagePixelAbsoluteDifference =
-          getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+          getBitmapAveragePixelAbsoluteDifferenceArgb8888(
+              expectedBitmap, actualBitmap, testId + "_" + i);
       assertThat(averagePixelAbsoluteDifference)
           .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
     }

libraries/effect/src/androidTest/java/androidx/media3/effect/TimestampAdjustmentTest.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames;
+import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.graphics.Color;
+import android.text.Spannable;
+import android.text.SpannableString;
+import android.text.style.AbsoluteSizeSpan;
+import android.text.style.ForegroundColorSpan;
+import android.text.style.TypefaceSpan;
+import androidx.media3.common.util.Consumer;
+import androidx.media3.test.utils.TextureBitmapReader;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
+import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.RequiresNonNull;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+/** Tests for {@link TimestampAdjustment}. */
+@RunWith(AndroidJUnit4.class)
+public class TimestampAdjustmentTest {
+  @Rule public final TestName testName = new TestName();
+
+  private static final String ASSET_PATH = "test-generated-goldens/TimestampAdjustmentTest";
+
+  private @MonotonicNonNull TextureBitmapReader textureBitmapReader;
+  private @MonotonicNonNull String testId;
+
+  @EnsuresNonNull({"textureBitmapReader", "testId"})
+  @Before
+  public void setUp() {
+    textureBitmapReader = new TextureBitmapReader();
+    testId = testName.getMethodName();
+  }
+
+  @Test
+  @RequiresNonNull({"textureBitmapReader", "testId"})
+  public void timestampAdjustmentTest_outputsFramesAtTheCorrectPresentationTimesUs()
+      throws Exception {
+    ImmutableList<Long> frameTimesUs = ImmutableList.of(0L, 32_000L, 71_000L);
+    TimestampAdjustment timestampAdjustment =
+        new TimestampAdjustment(
+            (inputTimeUs, callback) -> {
+              try {
+                Thread.sleep(/* millis= */ 50);
+              } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new IllegalStateException(e);
+              }
+              callback.accept(inputTimeUs / 2);
+            });
+
+    ImmutableList<Long> actualPresentationTimesUs =
+        generateAndProcessBlackTimeStampedFrames(frameTimesUs, timestampAdjustment);
+
+    assertThat(actualPresentationTimesUs).containsExactly(0L, 16_000L, 35_500L).inOrder();
+    getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH);
+  }
+
+  private ImmutableList<Long> generateAndProcessBlackTimeStampedFrames(
+      ImmutableList<Long> frameTimesUs, GlEffect effect) throws Exception {
+    int blankFrameWidth = 100;
+    int blankFrameHeight = 50;
+    Consumer<SpannableString> textSpanConsumer =
+        (text) -> {
+          text.setSpan(
+              new ForegroundColorSpan(Color.BLACK),
+              /* start= */ 0,
+              text.length(),
+              Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+          text.setSpan(
+              new AbsoluteSizeSpan(/* size= */ 24),
+              /* start= */ 0,
+              text.length(),
+              Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+          text.setSpan(
+              new TypefaceSpan(/* family= */ "sans-serif"),
+              /* start= */ 0,
+              text.length(),
+              Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+        };
+    return generateAndProcessFrames(
+        blankFrameWidth,
+        blankFrameHeight,
+        frameTimesUs,
+        effect,
+        checkNotNull(textureBitmapReader),
+        textSpanConsumer);
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustment.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import android.content.Context;
+import androidx.media3.common.util.UnstableApi;
+import java.util.function.LongConsumer;
+
+/**
+ * Changes the frame timestamps using the {@link TimestampMap}.
+ *
+ * <p>This effect doesn't drop any frames.
+ *
+ * <p>This effect is not supported for effects previewing.
+ */
+@UnstableApi
+public final class TimestampAdjustment implements GlEffect {
+
+  /**
+   * Maps input timestamps to output timestamps asynchronously.
+   *
+   * <p>Implementation can choose to calculate the timestamp and invoke the consumer on another
+   * thread asynchronously.
+   */
+  public interface TimestampMap {
+
+    /**
+     * Calculates the output timestamp that corresponds to the input timestamp.
+     *
+     * <p>The implementation should invoke the {@code outputTimeConsumer} with the output timestamp,
+     * on any thread.
+     */
+    void calculateOutputTimeUs(long inputTimeUs, LongConsumer outputTimeConsumer);
+  }
+
+  private final TimestampMap timestampMap;
+
+  /** Creates an instance. */
+  public TimestampAdjustment(TimestampMap timestampMap) {
+    this.timestampMap = timestampMap;
+  }
+
+  @Override
+  public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
+    return new TimestampAdjustmentShaderProgram(timestampMap);
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustmentShaderProgram.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
+
+import androidx.annotation.Nullable;
+import androidx.media3.common.GlObjectsProvider;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.effect.TimestampAdjustment.TimestampMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** Changes the frame timestamps using the {@link TimestampMap}. */
+@UnstableApi
+public class TimestampAdjustmentShaderProgram implements GlShaderProgram {
+
+  private final TimestampMap timestampMap;
+  private final AtomicInteger pendingCallbacksCount;
+  private final AtomicBoolean pendingEndOfStream;
+
+  @Nullable private GlTextureInfo inputTexture;
+  private InputListener inputListener;
+  private OutputListener outputListener;
+
+  public TimestampAdjustmentShaderProgram(TimestampMap timestampMap) {
+    inputListener = new InputListener() {};
+    outputListener = new OutputListener() {};
+
+    this.timestampMap = timestampMap;
+    pendingCallbacksCount = new AtomicInteger();
+    pendingEndOfStream = new AtomicBoolean();
+  }
+
+  @Override
+  public void setInputListener(InputListener inputListener) {
+    this.inputListener = inputListener;
+    if (inputTexture == null) {
+      inputListener.onReadyToAcceptInputFrame();
+    }
+  }
+
+  @Override
+  public void setOutputListener(OutputListener outputListener) {
+    this.outputListener = outputListener;
+  }
+
+  @Override
+  public void setErrorListener(Executor executor, ErrorListener errorListener) {
+    // No checked exceptions thrown.
+  }
+
+  @Override
+  public void queueInputFrame(
+      GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
+    this.inputTexture = inputTexture;
+    timestampMap.calculateOutputTimeUs(
+        presentationTimeUs, /* outputTimeConsumer= */ this::onOutputTimeAvailable);
+    pendingCallbacksCount.incrementAndGet();
+  }
+
+  @Override
+  public void signalEndOfCurrentInputStream() {
+    if (pendingCallbacksCount.get() == 0) {
+      outputListener.onCurrentOutputStreamEnded();
+    } else {
+      pendingEndOfStream.set(true);
+    }
+  }
+
+  @Override
+  public void releaseOutputFrame(GlTextureInfo outputTexture) {
+    checkState(outputTexture.texId == checkNotNull(inputTexture).texId);
+    inputListener.onInputFrameProcessed(outputTexture);
+    inputListener.onReadyToAcceptInputFrame();
+  }
+
+  @Override
+  public void flush() {
+    // TODO - b/320242819: Investigate support for previewing.
+    throw new UnsupportedOperationException("This effect is not supported for previewing.");
+  }
+
+  @Override
+  public void release() throws VideoFrameProcessingException {
+    inputTexture = null;
+  }
+
+  private void onOutputTimeAvailable(long outputTimeUs) {
+    outputListener.onOutputFrameAvailable(checkNotNull(inputTexture), outputTimeUs);
+    if (pendingEndOfStream.get()) {
+      outputListener.onCurrentOutputStreamEnded();
+      pendingEndOfStream.set(false);
+    }
+    pendingCallbacksCount.decrementAndGet();
+  }
+}