loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Read Bitmap in ExperimentalFrameExtractor

Browse Source 
Add a MatrixTransformation GlEffect to flip between
OpenGL and Bitmap coordinates

PiperOrigin-RevId: 696029842
This commit is contained in:
dancho 2024-11-13 01:24:38 -08:00 committed by Copybara-Service
parent 64e92cb8e1
commit 175dca41df
2 changed files with 73 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND;
import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import static java.util.concurrent.TimeUnit.SECONDS;
@ -23,6 +24,7 @@ import static org.junit.Assert.assertThrows;
import android.app.Instrumentation;
import android.content.Context;
import android.graphics.Bitmap;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.ConditionVariable;
import androidx.media3.common.util.NullableType;
@ -38,7 +40,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
/** End-to-end instrumentation test for {@link ExperimentalFrameExtractor}. */
@ -48,10 +53,18 @@ public class FrameExtractorTest {
"asset:///media/mp4/sample_with_increasing_timestamps_360p.mp4";
private static final long TIMEOUT_SECONDS = 10;
@Rule public final TestName testName = new TestName();
private final Context context = ApplicationProvider.getApplicationContext();
private String testId;
private @MonotonicNonNull ExperimentalFrameExtractor frameExtractor;
@Before
public void setUpTestId() {
testId = testName.getMethodName();
}
@After
public void tearDown() {
if (frameExtractor != null) {
@ -64,8 +77,15 @@ public class FrameExtractorTest {
frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH));
ListenableFuture<Frame> frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500);
Bitmap bitmap = frameFuture.get(TIMEOUT_SECONDS, SECONDS).bitmap;
maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", bitmap, /* path= */ null);
assertThat(frameFuture.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_531);
// TODO: b/350498258 - Actually check Bitmap contents. Due to bugs in hardware decoders,
// such a test would require a too high tolerance.
assertThat(bitmap.getWidth()).isEqualTo(640);
assertThat(bitmap.getHeight()).isEqualTo(360);
assertThat(bitmap.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888);
}
@Test

58
libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java
View File

@ -24,6 +24,9 @@ import static androidx.media3.common.util.Util.usToMs;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.Looper;
import androidx.annotation.Nullable;
@ -34,9 +37,11 @@ import androidx.media3.common.MediaItem;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.Player;
import androidx.media3.common.util.ConditionVariable;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.NullableType;
import androidx.media3.effect.GlEffect;
import androidx.media3.effect.GlShaderProgram;
import androidx.media3.effect.MatrixTransformation;
import androidx.media3.effect.PassthroughShaderProgram;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.analytics.AnalyticsListener;
@ -45,6 +50,7 @@ import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -59,14 +65,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* package */ final class ExperimentalFrameExtractor implements AnalyticsListener {
/** Stores an extracted and decoded video frame. */
// TODO: b/350498258 - Add a Bitmap field to Frame.
public static final class Frame {
/** The presentation timestamp of the extracted frame, in milliseconds. */
public final long presentationTimeMs;
private Frame(long presentationTimeMs) {
/** The extracted frame contents. */
public final Bitmap bitmap;
private Frame(long presentationTimeMs, Bitmap bitmap) {
this.presentationTimeMs = presentationTimeMs;
this.bitmap = bitmap;
}
}
@ -222,7 +231,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private ImmutableList<Effect> buildVideoEffects() {
return ImmutableList.of(new FrameReader());
return ImmutableList.of(
(MatrixTransformation)
presentationTimeUs -> {
Matrix mirrorY = new Matrix();
mirrorY.setScale(/* sx= */ 1, /* sy= */ -1);
return mirrorY;
},
new FrameReader());
}
private final class FrameReader implements GlEffect {
@ -234,13 +250,45 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private final class FrameReadingGlShaderProgram extends PassthroughShaderProgram {
private static final int BYTES_PER_PIXEL = 4;
private ByteBuffer byteBuffer = ByteBuffer.allocateDirect(0);
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
int pixelBufferSize = inputTexture.width * inputTexture.height * BYTES_PER_PIXEL;
if (byteBuffer.capacity() != pixelBufferSize) {
byteBuffer = ByteBuffer.allocateDirect(pixelBufferSize);
}
byteBuffer.clear();
try {
GlUtil.focusFramebufferUsingCurrentContext(
inputTexture.fboId, inputTexture.width, inputTexture.height);
GlUtil.checkGlError();
GLES20.glReadPixels(
/* x= */ 0,
/* y= */ 0,
inputTexture.width,
inputTexture.height,
GLES20.GL_RGBA,
GLES20.GL_UNSIGNED_BYTE,
byteBuffer);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
onError(e);
return;
}
// According to https://www.khronos.org/opengl/wiki/Pixel_Transfer#Endian_issues,
// the colors will have the order RGBA in client memory. This is what the bitmap expects:
// https://developer.android.com/reference/android/graphics/Bitmap.Config.
Bitmap bitmap =
Bitmap.createBitmap(inputTexture.width, inputTexture.height, Bitmap.Config.ARGB_8888);
bitmap.copyPixelsFromBuffer(byteBuffer);
SettableFuture<Frame> frameBeingExtractedFuture =
checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null));
// TODO: b/350498258 - Read the input texture contents into a Bitmap.
frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs)));
frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs), bitmap));
// Drop frame: do not call outputListener.onOutputFrameAvailable().
// Block effects pipeline: do not call inputListener.onReadyToAcceptInputFrame().
// The effects pipeline will unblock and receive new frames when flushed after a seek.