Add a thumbnail strip effect that tiles frames horizontally.

The size of the thumbnail strip and the timestamp of the video frames to use must be specified by the user of the effect.

PiperOrigin-RevId: 552809210

libraries/effect/src/androidTest/java/androidx/media3/effect/ThumbnailStripEffectPixelTest.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapFromFocusedGlFramebuffer;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapWithSolidColor;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.createGlTextureFromBitmap;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
+import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.Color;
+import android.opengl.EGLContext;
+import android.opengl.EGLDisplay;
+import android.opengl.EGLSurface;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.Util;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Pixel tests for {@link ThumbnailStripEffect}. */
+@RunWith(AndroidJUnit4.class)
+public final class ThumbnailStripEffectPixelTest {
+  private static final String ORIGINAL_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
+  private static final String TWO_THUMBNAILS_STRIP_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/linear_colors/two_thumbnails_strip.png";
+
+  private final Context context = getApplicationContext();
+
+  private @MonotonicNonNull EGLDisplay eglDisplay;
+  private @MonotonicNonNull EGLContext eglContext;
+  private @MonotonicNonNull EGLSurface placeholderEglSurface;
+  private @MonotonicNonNull ThumbnailStripShaderProgram thumbnailStripShaderProgram;
+  private int inputTexId;
+  private int inputWidth;
+  private int inputHeight;
+
+  @Before
+  public void setUp() throws Exception {
+    eglDisplay = GlUtil.getDefaultEglDisplay();
+    eglContext = GlUtil.createEglContext(eglDisplay);
+    placeholderEglSurface = GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay);
+
+    Bitmap inputBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
+    inputWidth = inputBitmap.getWidth();
+    inputHeight = inputBitmap.getHeight();
+    inputTexId = createGlTextureFromBitmap(inputBitmap);
+
+    int outputTexId =
+        GlUtil.createTexture(inputWidth, inputHeight, /* useHighPrecisionColorComponents= */ false);
+    int frameBuffer = GlUtil.createFboForTexture(outputTexId);
+    GlUtil.focusFramebuffer(
+        checkNotNull(eglDisplay),
+        checkNotNull(eglContext),
+        checkNotNull(placeholderEglSurface),
+        frameBuffer,
+        inputWidth,
+        inputHeight);
+  }
+
+  @After
+  public void tearDown() throws GlUtil.GlException, VideoFrameProcessingException {
+    if (thumbnailStripShaderProgram != null) {
+      thumbnailStripShaderProgram.release();
+    }
+    GlUtil.destroyEglContext(eglDisplay, eglContext);
+  }
+
+  @Test
+  public void drawFrame_withOneTimestampAndOriginalSize_producesOriginalFrame() throws Exception {
+    String testId = "drawFrame_withOneTimestampAndOriginalSize";
+    ThumbnailStripEffect thumbnailStripEffect = new ThumbnailStripEffect(inputWidth, inputHeight);
+    thumbnailStripEffect.setTimestampsMs(ImmutableList.of(0L));
+    thumbnailStripShaderProgram =
+        thumbnailStripEffect.toGlShaderProgram(context, /* useHdr= */ false);
+    Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
+
+    thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0L);
+    Bitmap actualBitmap = createArgb8888BitmapFromFocusedGlFramebuffer(inputWidth, inputHeight);
+
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_zeroTimestamps_producesEmptyFrame() throws Exception {
+    String testId = "drawFrame_zeroTimestamps";
+    ThumbnailStripEffect thumbnailStripEffect = new ThumbnailStripEffect(inputWidth, inputHeight);
+    thumbnailStripEffect.setTimestampsMs(ImmutableList.of());
+    thumbnailStripShaderProgram =
+        thumbnailStripEffect.toGlShaderProgram(context, /* useHdr= */ false);
+    Bitmap expectedBitmap =
+        createArgb8888BitmapWithSolidColor(inputWidth, inputHeight, Color.TRANSPARENT);
+
+    thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0L);
+    Bitmap actualBitmap = createArgb8888BitmapFromFocusedGlFramebuffer(inputWidth, inputHeight);
+
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_lateTimestamp_producesEmptyFrame() throws Exception {
+    String testId = "drawFrame_lateTimestamp";
+    ThumbnailStripEffect thumbnailStripEffect = new ThumbnailStripEffect(inputWidth, inputHeight);
+    thumbnailStripEffect.setTimestampsMs(ImmutableList.of(1L));
+    thumbnailStripShaderProgram =
+        thumbnailStripEffect.toGlShaderProgram(context, /* useHdr= */ false);
+    Bitmap expectedBitmap =
+        createArgb8888BitmapWithSolidColor(inputWidth, inputHeight, Color.TRANSPARENT);
+
+    thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0L);
+    Bitmap actualBitmap = createArgb8888BitmapFromFocusedGlFramebuffer(inputWidth, inputHeight);
+
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_twoTimestamps_producesStrip() throws Exception {
+    String testId = "drawFrame_twoTimestamps";
+    ThumbnailStripEffect thumbnailStripEffect = new ThumbnailStripEffect(inputWidth, inputHeight);
+    thumbnailStripEffect.setTimestampsMs(ImmutableList.of(0L, 1L));
+    thumbnailStripShaderProgram =
+        thumbnailStripEffect.toGlShaderProgram(context, /* useHdr= */ false);
+    Bitmap expectedBitmap = readBitmap(TWO_THUMBNAILS_STRIP_PNG_ASSET_PATH);
+
+    thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0L);
+    thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ Util.msToUs(1L));
+    Bitmap actualBitmap = createArgb8888BitmapFromFocusedGlFramebuffer(inputWidth, inputHeight);
+
+    maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+}

libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl

@@ -0,0 +1,26 @@
+#version 100
+// Copyright 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ES 2 fragment shader that samples from a (non-external) texture with
+// uTexSampler.
+
+precision mediump float;
+uniform sampler2D uTexSampler;
+varying vec2 vTexSamplingCoord;
+
+void main() {
+    vec3 src = texture2D(uTexSampler, vTexSamplingCoord).xyz;
+    gl_FragColor = vec4(src, 1.0);
+}

libraries/effect/src/main/assets/shaders/vertex_shader_thumbnail_strip_es2.glsl

@@ -0,0 +1,35 @@
+#version 100
+// Copyright 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ES2 vertex shader that tiles frames horizontally.
+
+attribute vec4 aFramePosition;
+uniform int uIndex;
+uniform int uCount;
+varying vec2 vTexSamplingCoord;
+
+void main() {
+    // Translate the coordinates from -1,+1 to 0,+2.
+    float x = aFramePosition.x + 1.0;
+    // Offset the frame by its index times its width (2).
+    x += float(uIndex) * 2.0;
+    // Shrink the frame to fit the thumbnail strip.
+    x /= float(uCount);
+    // Translate the coordinates back to -1,+1.
+    x -= 1.0;
+
+    gl_Position = vec4(x, aFramePosition.yzw);
+    vTexSamplingCoord = aFramePosition.xy * 0.5 + 0.5;
+}

libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java

@@ -114,6 +114,14 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
     this.errorListener = errorListener;
   }
 
+  /**
+   * Returns {@code true} if the texture buffer should be cleared before calling {@link #drawFrame}
+   * or {@code false} if it should retain the content of the last drawn frame.
+   */
+  public boolean shouldClearTextureBuffer() {
+    return true;
+  }
+
   @Override
   public void queueInputFrame(
       GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
@@ -128,7 +136,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
       // Copy frame to fbo.
       GlUtil.focusFramebufferUsingCurrentContext(
           outputTexture.fboId, outputTexture.width, outputTexture.height);
-      GlUtil.clearFocusedBuffers();
+      if (shouldClearTextureBuffer()) {
+        GlUtil.clearFocusedBuffers();
+      }
       drawFrame(inputTexture.texId, presentationTimeUs);
       inputListener.onInputFrameProcessed(inputTexture);
       outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);

libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripEffect.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import android.content.Context;
+import androidx.media3.common.C;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.UnstableApi;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Generate a thumbnail strip (i.e. tile frames horizontally) containing frames at given {@link
+ * #setTimestampsMs timestamps}.
+ */
+@UnstableApi
+/* package */ final class ThumbnailStripEffect implements GlEffect {
+
+  /* package */ final int stripWidth;
+  /* package */ final int stripHeight;
+  private final List<Long> timestampsMs;
+  private int currentThumbnailIndex;
+
+  /**
+   * Creates a new instance with the given size. No thumbnails are drawn by default, call {@link
+   * #setTimestampsMs} to change how many to draw and their timestamp.
+   *
+   * @param stripWidth The width of the thumbnail strip.
+   * @param stripHeight The height of the thumbnail strip.
+   */
+  public ThumbnailStripEffect(int stripWidth, int stripHeight) {
+    this.stripWidth = stripWidth;
+    this.stripHeight = stripHeight;
+    timestampsMs = new ArrayList<>();
+  }
+
+  @Override
+  public ThumbnailStripShaderProgram toGlShaderProgram(Context context, boolean useHdr)
+      throws VideoFrameProcessingException {
+    return new ThumbnailStripShaderProgram(context, useHdr, this);
+  }
+
+  /**
+   * Sets the timestamps of the frames to draw, in milliseconds.
+   *
+   * <p>The timestamp represents the minimum presentation time of the next frame added to the strip.
+   * For example, if the timestamp is 10, a frame with a time of 100 will be drawn but one with a
+   * time of 9 will be ignored.
+   */
+  public void setTimestampsMs(List<Long> timestampsMs) {
+    this.timestampsMs.clear();
+    this.timestampsMs.addAll(timestampsMs);
+    currentThumbnailIndex = 0;
+  }
+
+  /** Returns whether all the thumbnails have already been drawn. */
+  public boolean isDone() {
+    return currentThumbnailIndex >= timestampsMs.size();
+  }
+
+  /** Returns the index of the next thumbnail to draw. */
+  public int getNextThumbnailIndex() {
+    return currentThumbnailIndex;
+  }
+
+  /** Returns the timestamp in milliseconds of the next thumbnail to draw. */
+  public long getNextTimestampMs() {
+    return isDone() ? C.TIME_END_OF_SOURCE : timestampsMs.get(currentThumbnailIndex);
+  }
+
+  /** Returns the total number of thumbnails to be drawn in the strip. */
+  public int getNumberOfThumbnails() {
+    return timestampsMs.size();
+  }
+
+  /* package */ void onThumbnailDrawn() {
+    currentThumbnailIndex++;
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripShaderProgram.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import android.content.Context;
+import android.opengl.GLES20;
+import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.GlProgram;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.Size;
+import androidx.media3.common.util.Util;
+import java.io.IOException;
+
+/**
+ * Draws the target input frame at a given horizontal position of the output texture to generate an
+ * horizontal tiling effect.
+ */
+/* package */ final class ThumbnailStripShaderProgram extends SingleFrameGlShaderProgram {
+  private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_thumbnail_strip_es2.glsl";
+  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl";
+
+  private final GlProgram glProgram;
+  private final ThumbnailStripEffect thumbnailStripEffect;
+
+  public ThumbnailStripShaderProgram(
+      Context context, boolean useHdr, ThumbnailStripEffect thumbnailStripEffect)
+      throws VideoFrameProcessingException {
+    super(useHdr);
+    this.thumbnailStripEffect = thumbnailStripEffect;
+
+    try {
+      this.glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
+    } catch (IOException | GlUtil.GlException e) {
+      throw VideoFrameProcessingException.from(e);
+    }
+
+    glProgram.setBufferAttribute(
+        "aFramePosition",
+        GlUtil.getNormalizedCoordinateBounds(),
+        GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
+  }
+
+  @Override
+  public boolean shouldClearTextureBuffer() {
+    // The output texture buffer is never cleared in order to keep the previously drawn frames and
+    // generate an horizontal tiling effect.
+    return false;
+  }
+
+  @Override
+  public Size configure(int inputWidth, int inputHeight) {
+    return new Size(thumbnailStripEffect.stripWidth, thumbnailStripEffect.stripHeight);
+  }
+
+  @Override
+  public void drawFrame(int inputTexId, long presentationTimeUs)
+      throws VideoFrameProcessingException {
+    long targetPresentationTimeUs = Util.msToUs(thumbnailStripEffect.getNextTimestampMs());
+    // Ignore the frame if there are no more thumbnails to draw or if it's earlier than the target.
+    if (thumbnailStripEffect.isDone() || presentationTimeUs < targetPresentationTimeUs) {
+      return;
+    }
+    try {
+      glProgram.use();
+      glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
+      glProgram.setIntUniform("uIndex", thumbnailStripEffect.getNextThumbnailIndex());
+      glProgram.setIntUniform("uCount", thumbnailStripEffect.getNumberOfThumbnails());
+      glProgram.bindAttributesAndUniforms();
+      // The four-vertex triangle strip forms a quad.
+      GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
+      thumbnailStripEffect.onThumbnailDrawn();
+    } catch (GlUtil.GlException e) {
+      throw new VideoFrameProcessingException(e, presentationTimeUs);
+    }
+  }
+}