diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java
index 9360693fec..8082b33855 100644
--- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java
@@ -16,6 +16,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -23,6 +24,7 @@ import static org.junit.Assert.assertThrows;
import android.app.Instrumentation;
import android.content.Context;
+import android.graphics.Bitmap;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.ConditionVariable;
import androidx.media3.common.util.NullableType;
@@ -38,7 +40,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import org.junit.runner.RunWith;
/** End-to-end instrumentation test for {@link ExperimentalFrameExtractor}. */
@@ -48,10 +53,18 @@ public class FrameExtractorTest {
"asset:///media/mp4/sample_with_increasing_timestamps_360p.mp4";
private static final long TIMEOUT_SECONDS = 10;
+ @Rule public final TestName testName = new TestName();
+
private final Context context = ApplicationProvider.getApplicationContext();
+ private String testId;
private @MonotonicNonNull ExperimentalFrameExtractor frameExtractor;
+ @Before
+ public void setUpTestId() {
+ testId = testName.getMethodName();
+ }
+
@After
public void tearDown() {
if (frameExtractor != null) {
@@ -64,8 +77,15 @@ public class FrameExtractorTest {
frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH));
ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500);
+ Bitmap bitmap = frameFuture.get(TIMEOUT_SECONDS, SECONDS).bitmap;
+ maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", bitmap, /* path= */ null);
assertThat(frameFuture.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_531);
+ // TODO: b/350498258 - Actually check Bitmap contents. Due to bugs in hardware decoders,
+ // such a test would require a too high tolerance.
+ assertThat(bitmap.getWidth()).isEqualTo(640);
+ assertThat(bitmap.getHeight()).isEqualTo(360);
+ assertThat(bitmap.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888);
}
@Test
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java
index 38eb92aa93..6d7ac47cb1 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java
@@ -24,6 +24,9 @@ import static androidx.media3.common.util.Util.usToMs;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+import android.opengl.GLES20;
import android.os.Handler;
import android.os.Looper;
import androidx.annotation.Nullable;
@@ -34,9 +37,11 @@ import androidx.media3.common.MediaItem;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.Player;
import androidx.media3.common.util.ConditionVariable;
+import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.NullableType;
import androidx.media3.effect.GlEffect;
import androidx.media3.effect.GlShaderProgram;
+import androidx.media3.effect.MatrixTransformation;
import androidx.media3.effect.PassthroughShaderProgram;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.analytics.AnalyticsListener;
@@ -45,6 +50,7 @@ import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
+import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@@ -59,14 +65,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* package */ final class ExperimentalFrameExtractor implements AnalyticsListener {
/** Stores an extracted and decoded video frame. */
- // TODO: b/350498258 - Add a Bitmap field to Frame.
public static final class Frame {
/** The presentation timestamp of the extracted frame, in milliseconds. */
public final long presentationTimeMs;
- private Frame(long presentationTimeMs) {
+ /** The extracted frame contents. */
+ public final Bitmap bitmap;
+
+ private Frame(long presentationTimeMs, Bitmap bitmap) {
this.presentationTimeMs = presentationTimeMs;
+ this.bitmap = bitmap;
}
}
@@ -222,7 +231,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private ImmutableList buildVideoEffects() {
- return ImmutableList.of(new FrameReader());
+ return ImmutableList.of(
+ (MatrixTransformation)
+ presentationTimeUs -> {
+ Matrix mirrorY = new Matrix();
+ mirrorY.setScale(/* sx= */ 1, /* sy= */ -1);
+ return mirrorY;
+ },
+ new FrameReader());
}
private final class FrameReader implements GlEffect {
@@ -234,13 +250,45 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private final class FrameReadingGlShaderProgram extends PassthroughShaderProgram {
+ private static final int BYTES_PER_PIXEL = 4;
+
+ private ByteBuffer byteBuffer = ByteBuffer.allocateDirect(0);
+
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
+ int pixelBufferSize = inputTexture.width * inputTexture.height * BYTES_PER_PIXEL;
+ if (byteBuffer.capacity() != pixelBufferSize) {
+ byteBuffer = ByteBuffer.allocateDirect(pixelBufferSize);
+ }
+ byteBuffer.clear();
+ try {
+ GlUtil.focusFramebufferUsingCurrentContext(
+ inputTexture.fboId, inputTexture.width, inputTexture.height);
+ GlUtil.checkGlError();
+ GLES20.glReadPixels(
+ /* x= */ 0,
+ /* y= */ 0,
+ inputTexture.width,
+ inputTexture.height,
+ GLES20.GL_RGBA,
+ GLES20.GL_UNSIGNED_BYTE,
+ byteBuffer);
+ GlUtil.checkGlError();
+ } catch (GlUtil.GlException e) {
+ onError(e);
+ return;
+ }
+ // According to https://www.khronos.org/opengl/wiki/Pixel_Transfer#Endian_issues,
+ // the colors will have the order RGBA in client memory. This is what the bitmap expects:
+ // https://developer.android.com/reference/android/graphics/Bitmap.Config.
+ Bitmap bitmap =
+ Bitmap.createBitmap(inputTexture.width, inputTexture.height, Bitmap.Config.ARGB_8888);
+ bitmap.copyPixelsFromBuffer(byteBuffer);
+
SettableFuture frameBeingExtractedFuture =
checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null));
- // TODO: b/350498258 - Read the input texture contents into a Bitmap.
- frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs)));
+ frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs), bitmap));
// Drop frame: do not call outputListener.onOutputFrameAvailable().
// Block effects pipeline: do not call inputListener.onReadyToAcceptInputFrame().
// The effects pipeline will unblock and receive new frames when flushed after a seek.