diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java index 9360693fec..8082b33855 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; import static androidx.media3.common.PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static com.google.common.truth.Truth.assertThat; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.util.concurrent.TimeUnit.SECONDS; @@ -23,6 +24,7 @@ import static org.junit.Assert.assertThrows; import android.app.Instrumentation; import android.content.Context; +import android.graphics.Bitmap; import androidx.media3.common.MediaItem; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.NullableType; @@ -38,7 +40,10 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestName; import org.junit.runner.RunWith; /** End-to-end instrumentation test for {@link ExperimentalFrameExtractor}. */ @@ -48,10 +53,18 @@ public class FrameExtractorTest { "asset:///media/mp4/sample_with_increasing_timestamps_360p.mp4"; private static final long TIMEOUT_SECONDS = 10; + @Rule public final TestName testName = new TestName(); + private final Context context = ApplicationProvider.getApplicationContext(); + private String testId; private @MonotonicNonNull ExperimentalFrameExtractor frameExtractor; + @Before + public void setUpTestId() { + testId = testName.getMethodName(); + } + @After public void tearDown() { if (frameExtractor != null) { @@ -64,8 +77,15 @@ public class FrameExtractorTest { frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500); + Bitmap bitmap = frameFuture.get(TIMEOUT_SECONDS, SECONDS).bitmap; + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", bitmap, /* path= */ null); assertThat(frameFuture.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_531); + // TODO: b/350498258 - Actually check Bitmap contents. Due to bugs in hardware decoders, + // such a test would require a too high tolerance. + assertThat(bitmap.getWidth()).isEqualTo(640); + assertThat(bitmap.getHeight()).isEqualTo(360); + assertThat(bitmap.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888); } @Test diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java index 38eb92aa93..6d7ac47cb1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java @@ -24,6 +24,9 @@ import static androidx.media3.common.util.Util.usToMs; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Matrix; +import android.opengl.GLES20; import android.os.Handler; import android.os.Looper; import androidx.annotation.Nullable; @@ -34,9 +37,11 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.PlaybackException; import androidx.media3.common.Player; import androidx.media3.common.util.ConditionVariable; +import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.NullableType; import androidx.media3.effect.GlEffect; import androidx.media3.effect.GlShaderProgram; +import androidx.media3.effect.MatrixTransformation; import androidx.media3.effect.PassthroughShaderProgram; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.analytics.AnalyticsListener; @@ -45,6 +50,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -59,14 +65,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* package */ final class ExperimentalFrameExtractor implements AnalyticsListener { /** Stores an extracted and decoded video frame. */ - // TODO: b/350498258 - Add a Bitmap field to Frame. public static final class Frame { /** The presentation timestamp of the extracted frame, in milliseconds. */ public final long presentationTimeMs; - private Frame(long presentationTimeMs) { + /** The extracted frame contents. */ + public final Bitmap bitmap; + + private Frame(long presentationTimeMs, Bitmap bitmap) { this.presentationTimeMs = presentationTimeMs; + this.bitmap = bitmap; } } @@ -222,7 +231,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private ImmutableList buildVideoEffects() { - return ImmutableList.of(new FrameReader()); + return ImmutableList.of( + (MatrixTransformation) + presentationTimeUs -> { + Matrix mirrorY = new Matrix(); + mirrorY.setScale(/* sx= */ 1, /* sy= */ -1); + return mirrorY; + }, + new FrameReader()); } private final class FrameReader implements GlEffect { @@ -234,13 +250,45 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private final class FrameReadingGlShaderProgram extends PassthroughShaderProgram { + private static final int BYTES_PER_PIXEL = 4; + + private ByteBuffer byteBuffer = ByteBuffer.allocateDirect(0); + @Override public void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + int pixelBufferSize = inputTexture.width * inputTexture.height * BYTES_PER_PIXEL; + if (byteBuffer.capacity() != pixelBufferSize) { + byteBuffer = ByteBuffer.allocateDirect(pixelBufferSize); + } + byteBuffer.clear(); + try { + GlUtil.focusFramebufferUsingCurrentContext( + inputTexture.fboId, inputTexture.width, inputTexture.height); + GlUtil.checkGlError(); + GLES20.glReadPixels( + /* x= */ 0, + /* y= */ 0, + inputTexture.width, + inputTexture.height, + GLES20.GL_RGBA, + GLES20.GL_UNSIGNED_BYTE, + byteBuffer); + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + onError(e); + return; + } + // According to https://www.khronos.org/opengl/wiki/Pixel_Transfer#Endian_issues, + // the colors will have the order RGBA in client memory. This is what the bitmap expects: + // https://developer.android.com/reference/android/graphics/Bitmap.Config. + Bitmap bitmap = + Bitmap.createBitmap(inputTexture.width, inputTexture.height, Bitmap.Config.ARGB_8888); + bitmap.copyPixelsFromBuffer(byteBuffer); + SettableFuture frameBeingExtractedFuture = checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null)); - // TODO: b/350498258 - Read the input texture contents into a Bitmap. - frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs))); + frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs), bitmap)); // Drop frame: do not call outputListener.onOutputFrameAvailable(). // Block effects pipeline: do not call inputListener.onReadyToAcceptInputFrame(). // The effects pipeline will unblock and receive new frames when flushed after a seek.