diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/ThumbnailStripEffectPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/ThumbnailStripEffectPixelTest.java new file mode 100644 index 0000000000..6328a8ff17 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/ThumbnailStripEffectPixelTest.java @@ -0,0 +1,169 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapFromFocusedGlFramebuffer; +import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapWithSolidColor; +import static androidx.media3.test.utils.BitmapPixelTestUtil.createGlTextureFromBitmap; +import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Color; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLSurface; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Util; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Pixel tests for {@link ThumbnailStripEffect}. */ +@RunWith(AndroidJUnit4.class) +public final class ThumbnailStripEffectPixelTest { + private static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/linear_colors/original.png"; + private static final String TWO_THUMBNAILS_STRIP_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/linear_colors/two_thumbnails_strip.png"; + + private final Context context = getApplicationContext(); + + private @MonotonicNonNull EGLDisplay eglDisplay; + private @MonotonicNonNull EGLContext eglContext; + private @MonotonicNonNull EGLSurface placeholderEglSurface; + private @MonotonicNonNull ThumbnailStripShaderProgram thumbnailStripShaderProgram; + private int inputTexId; + private int inputWidth; + private int inputHeight; + + @Before + public void setUp() throws Exception { + eglDisplay = GlUtil.getDefaultEglDisplay(); + eglContext = GlUtil.createEglContext(eglDisplay); + placeholderEglSurface = GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay); + + Bitmap inputBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); + inputWidth = inputBitmap.getWidth(); + inputHeight = inputBitmap.getHeight(); + inputTexId = createGlTextureFromBitmap(inputBitmap); + + int outputTexId = + GlUtil.createTexture(inputWidth, inputHeight, /* useHighPrecisionColorComponents= */ false); + int frameBuffer = GlUtil.createFboForTexture(outputTexId); + GlUtil.focusFramebuffer( + checkNotNull(eglDisplay), + checkNotNull(eglContext), + checkNotNull(placeholderEglSurface), + frameBuffer, + inputWidth, + inputHeight); + } + + @After + public void tearDown() throws GlUtil.GlException, VideoFrameProcessingException { + if (thumbnailStripShaderProgram != null) { + thumbnailStripShaderProgram.release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + } + + @Test + public void drawFrame_withOneTimestampAndOriginalSize_producesOriginalFrame() throws Exception { + String testId = "drawFrame_withOneTimestampAndOriginalSize"; + ThumbnailStripEffect thumbnailStripEffect = new ThumbnailStripEffect(inputWidth, inputHeight); + thumbnailStripEffect.setTimestampsMs(ImmutableList.of(0L)); + thumbnailStripShaderProgram = + thumbnailStripEffect.toGlShaderProgram(context, /* useHdr= */ false); + Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); + + thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0L); + Bitmap actualBitmap = createArgb8888BitmapFromFocusedGlFramebuffer(inputWidth, inputHeight); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_zeroTimestamps_producesEmptyFrame() throws Exception { + String testId = "drawFrame_zeroTimestamps"; + ThumbnailStripEffect thumbnailStripEffect = new ThumbnailStripEffect(inputWidth, inputHeight); + thumbnailStripEffect.setTimestampsMs(ImmutableList.of()); + thumbnailStripShaderProgram = + thumbnailStripEffect.toGlShaderProgram(context, /* useHdr= */ false); + Bitmap expectedBitmap = + createArgb8888BitmapWithSolidColor(inputWidth, inputHeight, Color.TRANSPARENT); + + thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0L); + Bitmap actualBitmap = createArgb8888BitmapFromFocusedGlFramebuffer(inputWidth, inputHeight); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_lateTimestamp_producesEmptyFrame() throws Exception { + String testId = "drawFrame_lateTimestamp"; + ThumbnailStripEffect thumbnailStripEffect = new ThumbnailStripEffect(inputWidth, inputHeight); + thumbnailStripEffect.setTimestampsMs(ImmutableList.of(1L)); + thumbnailStripShaderProgram = + thumbnailStripEffect.toGlShaderProgram(context, /* useHdr= */ false); + Bitmap expectedBitmap = + createArgb8888BitmapWithSolidColor(inputWidth, inputHeight, Color.TRANSPARENT); + + thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0L); + Bitmap actualBitmap = createArgb8888BitmapFromFocusedGlFramebuffer(inputWidth, inputHeight); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_twoTimestamps_producesStrip() throws Exception { + String testId = "drawFrame_twoTimestamps"; + ThumbnailStripEffect thumbnailStripEffect = new ThumbnailStripEffect(inputWidth, inputHeight); + thumbnailStripEffect.setTimestampsMs(ImmutableList.of(0L, 1L)); + thumbnailStripShaderProgram = + thumbnailStripEffect.toGlShaderProgram(context, /* useHdr= */ false); + Bitmap expectedBitmap = readBitmap(TWO_THUMBNAILS_STRIP_PNG_ASSET_PATH); + + thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ 0L); + thumbnailStripShaderProgram.drawFrame(inputTexId, /* presentationTimeUs= */ Util.msToUs(1L)); + Bitmap actualBitmap = createArgb8888BitmapFromFocusedGlFramebuffer(inputWidth, inputHeight); + + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } +} diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl new file mode 100644 index 0000000000..74ce8153cf --- /dev/null +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl @@ -0,0 +1,26 @@ +#version 100 +// Copyright 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 fragment shader that samples from a (non-external) texture with +// uTexSampler. + +precision mediump float; +uniform sampler2D uTexSampler; +varying vec2 vTexSamplingCoord; + +void main() { + vec3 src = texture2D(uTexSampler, vTexSamplingCoord).xyz; + gl_FragColor = vec4(src, 1.0); +} diff --git a/libraries/effect/src/main/assets/shaders/vertex_shader_thumbnail_strip_es2.glsl b/libraries/effect/src/main/assets/shaders/vertex_shader_thumbnail_strip_es2.glsl new file mode 100644 index 0000000000..2d1837ab0a --- /dev/null +++ b/libraries/effect/src/main/assets/shaders/vertex_shader_thumbnail_strip_es2.glsl @@ -0,0 +1,35 @@ +#version 100 +// Copyright 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES2 vertex shader that tiles frames horizontally. + +attribute vec4 aFramePosition; +uniform int uIndex; +uniform int uCount; +varying vec2 vTexSamplingCoord; + +void main() { + // Translate the coordinates from -1,+1 to 0,+2. + float x = aFramePosition.x + 1.0; + // Offset the frame by its index times its width (2). + x += float(uIndex) * 2.0; + // Shrink the frame to fit the thumbnail strip. + x /= float(uCount); + // Translate the coordinates back to -1,+1. + x -= 1.0; + + gl_Position = vec4(x, aFramePosition.yzw); + vTexSamplingCoord = aFramePosition.xy * 0.5 + 0.5; +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java index 972510619c..edc000fd9a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java @@ -114,6 +114,14 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram { this.errorListener = errorListener; } + /** + * Returns {@code true} if the texture buffer should be cleared before calling {@link #drawFrame} + * or {@code false} if it should retain the content of the last drawn frame. + */ + public boolean shouldClearTextureBuffer() { + return true; + } + @Override public void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { @@ -128,7 +136,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram { // Copy frame to fbo. GlUtil.focusFramebufferUsingCurrentContext( outputTexture.fboId, outputTexture.width, outputTexture.height); - GlUtil.clearFocusedBuffers(); + if (shouldClearTextureBuffer()) { + GlUtil.clearFocusedBuffers(); + } drawFrame(inputTexture.texId, presentationTimeUs); inputListener.onInputFrameProcessed(inputTexture); outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripEffect.java new file mode 100644 index 0000000000..476f5388b8 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripEffect.java @@ -0,0 +1,92 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.content.Context; +import androidx.media3.common.C; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.UnstableApi; +import java.util.ArrayList; +import java.util.List; + +/** + * Generate a thumbnail strip (i.e. tile frames horizontally) containing frames at given {@link + * #setTimestampsMs timestamps}. + */ +@UnstableApi +/* package */ final class ThumbnailStripEffect implements GlEffect { + + /* package */ final int stripWidth; + /* package */ final int stripHeight; + private final List timestampsMs; + private int currentThumbnailIndex; + + /** + * Creates a new instance with the given size. No thumbnails are drawn by default, call {@link + * #setTimestampsMs} to change how many to draw and their timestamp. + * + * @param stripWidth The width of the thumbnail strip. + * @param stripHeight The height of the thumbnail strip. + */ + public ThumbnailStripEffect(int stripWidth, int stripHeight) { + this.stripWidth = stripWidth; + this.stripHeight = stripHeight; + timestampsMs = new ArrayList<>(); + } + + @Override + public ThumbnailStripShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return new ThumbnailStripShaderProgram(context, useHdr, this); + } + + /** + * Sets the timestamps of the frames to draw, in milliseconds. + * + *

The timestamp represents the minimum presentation time of the next frame added to the strip. + * For example, if the timestamp is 10, a frame with a time of 100 will be drawn but one with a + * time of 9 will be ignored. + */ + public void setTimestampsMs(List timestampsMs) { + this.timestampsMs.clear(); + this.timestampsMs.addAll(timestampsMs); + currentThumbnailIndex = 0; + } + + /** Returns whether all the thumbnails have already been drawn. */ + public boolean isDone() { + return currentThumbnailIndex >= timestampsMs.size(); + } + + /** Returns the index of the next thumbnail to draw. */ + public int getNextThumbnailIndex() { + return currentThumbnailIndex; + } + + /** Returns the timestamp in milliseconds of the next thumbnail to draw. */ + public long getNextTimestampMs() { + return isDone() ? C.TIME_END_OF_SOURCE : timestampsMs.get(currentThumbnailIndex); + } + + /** Returns the total number of thumbnails to be drawn in the strip. */ + public int getNumberOfThumbnails() { + return timestampsMs.size(); + } + + /* package */ void onThumbnailDrawn() { + currentThumbnailIndex++; + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripShaderProgram.java new file mode 100644 index 0000000000..624669ea69 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripShaderProgram.java @@ -0,0 +1,89 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.content.Context; +import android.opengl.GLES20; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Size; +import androidx.media3.common.util.Util; +import java.io.IOException; + +/** + * Draws the target input frame at a given horizontal position of the output texture to generate an + * horizontal tiling effect. + */ +/* package */ final class ThumbnailStripShaderProgram extends SingleFrameGlShaderProgram { + private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_thumbnail_strip_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; + + private final GlProgram glProgram; + private final ThumbnailStripEffect thumbnailStripEffect; + + public ThumbnailStripShaderProgram( + Context context, boolean useHdr, ThumbnailStripEffect thumbnailStripEffect) + throws VideoFrameProcessingException { + super(useHdr); + this.thumbnailStripEffect = thumbnailStripEffect; + + try { + this.glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + } catch (IOException | GlUtil.GlException e) { + throw VideoFrameProcessingException.from(e); + } + + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + } + + @Override + public boolean shouldClearTextureBuffer() { + // The output texture buffer is never cleared in order to keep the previously drawn frames and + // generate an horizontal tiling effect. + return false; + } + + @Override + public Size configure(int inputWidth, int inputHeight) { + return new Size(thumbnailStripEffect.stripWidth, thumbnailStripEffect.stripHeight); + } + + @Override + public void drawFrame(int inputTexId, long presentationTimeUs) + throws VideoFrameProcessingException { + long targetPresentationTimeUs = Util.msToUs(thumbnailStripEffect.getNextTimestampMs()); + // Ignore the frame if there are no more thumbnails to draw or if it's earlier than the target. + if (thumbnailStripEffect.isDone() || presentationTimeUs < targetPresentationTimeUs) { + return; + } + try { + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + glProgram.setIntUniform("uIndex", thumbnailStripEffect.getNextThumbnailIndex()); + glProgram.setIntUniform("uCount", thumbnailStripEffect.getNumberOfThumbnails()); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + thumbnailStripEffect.onThumbnailDrawn(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e, presentationTimeUs); + } + } +} diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/linear_colors/two_thumbnails_strip.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/linear_colors/two_thumbnails_strip.png new file mode 100644 index 0000000000..4fa29b5464 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/linear_colors/two_thumbnails_strip.png differ