loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Implement video effects for Frame Extraction

Browse Source 
Test that downscaled images match MediaMetadataRetriever.

PiperOrigin-RevId: 696862566
This commit is contained in:
dancho 2024-11-15 06:06:59 -08:00 committed by Copybara-Service
parent e6448f3498
commit 11fc0871ac
3 changed files with 52 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
libraries/test_data/src/test/assets/test-generated-goldens/FrameExtractorTest/sample_with_increasing_timestamps_360p_8.531_scaled_to_180p.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 106 KiB

47
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java
View File

@ -31,6 +31,7 @@ import android.graphics.Bitmap;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.ConditionVariable;
import androidx.media3.common.util.NullableType;
import androidx.media3.effect.Presentation;
import androidx.media3.exoplayer.ExoPlaybackException;
import androidx.media3.transformer.ExperimentalFrameExtractor.Frame;
import androidx.test.core.app.ApplicationProvider;
@ -91,7 +92,8 @@ public class FrameExtractorTest {
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(FILE_PATH));
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of());
ListenableFuture<Frame> frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500);
Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS);
@ -106,13 +108,36 @@ public class FrameExtractorTest {
assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD);
}
@Test
public void extractFrame_oneFrameWithPresentationEffect_returnsScaledFrame() throws Exception {
frameExtractor =
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of(Presentation.createForHeight(180)));
ListenableFuture<Frame> frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500);
Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS);
Bitmap actualBitmap = frame.bitmap;
Bitmap expectedBitmap =
readBitmap(
/* assetString= */ GOLDEN_ASSET_FOLDER_PATH
+ "sample_with_increasing_timestamps_360p_8.531_scaled_to_180p.png");
maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null);
assertThat(frame.presentationTimeMs).isEqualTo(8_531);
assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD);
}
@Test
public void extractFrame_pastDuration_returnsLastFrame() throws Exception {
frameExtractor =
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(FILE_PATH));
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of());
ListenableFuture<Frame> frameFuture = frameExtractor.getFrame(/* positionMs= */ 200_000);
Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS);
@ -134,7 +159,8 @@ public class FrameExtractorTest {
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(FILE_PATH));
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of());
ImmutableList<Long> requestedFramePositionsMs = ImmutableList.of(0L, 0L, 33L, 34L, 34L);
ImmutableList<Long> expectedFramePositionsMs = ImmutableList.of(0L, 0L, 33L, 66L, 66L);
List<ListenableFuture<Frame>> frameFutures = new ArrayList<>();
@ -164,7 +190,8 @@ public class FrameExtractorTest {
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(FILE_PATH));
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of());
ListenableFuture<Frame> frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000);
ListenableFuture<Frame> frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000);
@ -187,7 +214,8 @@ public class FrameExtractorTest {
new ExperimentalFrameExtractor.Configuration.Builder()
.setSeekParameters(CLOSEST_SYNC)
.build(),
MediaItem.fromUri(FILE_PATH));
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of());
ListenableFuture<Frame> frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000);
ListenableFuture<Frame> frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000);
@ -211,7 +239,8 @@ public class FrameExtractorTest {
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(filePath));
MediaItem.fromUri(filePath),
/* effects= */ ImmutableList.of());
ListenableFuture<Frame> frame0 = frameExtractor.getFrame(/* positionMs= */ 0);
@ -228,7 +257,8 @@ public class FrameExtractorTest {
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(FILE_PATH));
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of());
AtomicReference<@NullableType Frame> frameAtomicReference = new AtomicReference<>();
AtomicReference<@NullableType Throwable> throwableAtomicReference = new AtomicReference<>();
ConditionVariable frameReady = new ConditionVariable();
@ -262,7 +292,8 @@ public class FrameExtractorTest {
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(FILE_PATH));
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of());
Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
instrumentation.runOnMainSync(frameExtractor::release);

20
libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java
View File

@ -53,6 +53,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -148,12 +149,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* Creates an instance.
*
* @param context {@link Context}.
* @param configuration The {@link Configuration} for this frame extractor.
* @param mediaItem The {@link MediaItem} from which frames are extracted.
* @param effects The {@link List} of {@linkplain Effect video effects} to apply to the extracted
* video frames.
*/
// TODO: b/350498258 - Support changing the MediaItem.
// TODO: b/350498258 - Support video effects.
public ExperimentalFrameExtractor(
Context context, Configuration configuration, MediaItem mediaItem) {
Context context, Configuration configuration, MediaItem mediaItem, List<Effect> effects) {
player = new ExoPlayer.Builder(context).setSeekParameters(configuration.seekParameters).build();
playerApplicationThreadHandler = new Handler(player.getApplicationLooper());
lastRequestedFrameFuture = SettableFuture.create();
@ -168,7 +171,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
playerApplicationThreadHandler.post(
() -> {
player.addAnalyticsListener(thisRef);
player.setVideoEffects(buildVideoEffects());
player.setVideoEffects(buildVideoEffects(effects));
player.setMediaItem(mediaItem);
player.setPlayWhenReady(false);
player.prepare();
@ -272,15 +275,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
private ImmutableList<Effect> buildVideoEffects() {
return ImmutableList.of(
private ImmutableList<Effect> buildVideoEffects(List<Effect> effects) {
ImmutableList.Builder<Effect> listBuilder = new ImmutableList.Builder<>();
listBuilder.addAll(effects);
listBuilder.add(
(MatrixTransformation)
presentationTimeUs -> {
Matrix mirrorY = new Matrix();
mirrorY.setScale(/* sx= */ 1, /* sy= */ -1);
return mirrorY;
},
new FrameReader());
});
listBuilder.add(new FrameReader());
return listBuilder.build();
}
private final class FrameReader implements GlEffect {