From 16a15b94caaa928e22da995426f02fc1d3adf5d6 Mon Sep 17 00:00:00 2001 From: dancho Date: Thu, 14 Nov 2024 02:44:50 -0800 Subject: [PATCH] Add Configration for Frame Extraction for specific SeekParameters Expose ExoPlayer seek parameters via FrameExtractor API PiperOrigin-RevId: 696449874 --- .../transformer/FrameExtractorTest.java | 70 ++++++++++++++++--- .../ExperimentalFrameExtractor.java | 48 ++++++++++++- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java index 8a404b1813..5c69a91c2f 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; import static androidx.media3.common.PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND; +import static androidx.media3.exoplayer.SeekParameters.CLOSEST_SYNC; import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; import static androidx.media3.test.utils.TestUtil.assertBitmapsAreSimilar; @@ -86,7 +87,11 @@ public class FrameExtractorTest { @Test public void extractFrame_oneFrame_returnsNearest() throws Exception { - frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(FILE_PATH)); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500); Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); @@ -103,7 +108,11 @@ public class FrameExtractorTest { @Test public void extractFrame_pastDuration_returnsLastFrame() throws Exception { - frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(FILE_PATH)); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 200_000); Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); @@ -121,7 +130,11 @@ public class FrameExtractorTest { @Test public void extractFrame_repeatedPositionMs_returnsTheSameFrame() throws Exception { - frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(FILE_PATH)); ImmutableList requestedFramePositionsMs = ImmutableList.of(0L, 0L, 33L, 34L, 34L); ImmutableList expectedFramePositionsMs = ImmutableList.of(0L, 0L, 33L, 66L, 66L); List> frameFutures = new ArrayList<>(); @@ -147,7 +160,11 @@ public class FrameExtractorTest { @Test public void extractFrame_randomAccess_returnsCorrectFrames() throws Exception { - frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(FILE_PATH)); ListenableFuture frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000); ListenableFuture frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000); @@ -162,10 +179,39 @@ public class FrameExtractorTest { assertThat(frame8.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_031); } + @Test + public void extractFrame_closestSyncRandomAccess_returnsCorrectFrames() throws Exception { + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder() + .setSeekParameters(CLOSEST_SYNC) + .build(), + MediaItem.fromUri(FILE_PATH)); + + ListenableFuture frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000); + ListenableFuture frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000); + ListenableFuture frame7 = frameExtractor.getFrame(/* positionMs= */ 7_000); + ListenableFuture frame2 = frameExtractor.getFrame(/* positionMs= */ 2_000); + ListenableFuture frame8 = frameExtractor.getFrame(/* positionMs= */ 8_000); + + // The input video has sync points at 0s, 8.331s, and 9.198s. Verify with: + // ffprobe IN -select_streams v -show_entries frame=pict_type,pts_time -of csv -skip_frame nokey + assertThat(frame5.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_331); + assertThat(frame3.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(0); + assertThat(frame7.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_331); + assertThat(frame2.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(0); + assertThat(frame8.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_331); + } + @Test public void extractFrame_invalidInput_reportsErrorViaFuture() { String filePath = "asset:///nonexistent"; - frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(filePath)); + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(filePath)); ListenableFuture frame0 = frameExtractor.getFrame(/* positionMs= */ 0); @@ -178,7 +224,11 @@ public class FrameExtractorTest { @Test public void extractFrame_oneFrame_completesViaCallback() throws Exception { - frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(FILE_PATH)); AtomicReference<@NullableType Frame> frameAtomicReference = new AtomicReference<>(); AtomicReference<@NullableType Throwable> throwableAtomicReference = new AtomicReference<>(); ConditionVariable frameReady = new ConditionVariable(); @@ -207,8 +257,12 @@ public class FrameExtractorTest { } @Test - public void frameExtractor_releaseOnPlayerLooper_returns() throws Exception { - frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + public void frameExtractor_releaseOnPlayerLooper_returns() { + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(FILE_PATH)); Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); instrumentation.runOnMainSync(frameExtractor::release); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java index 6d7ac47cb1..ca5aae1695 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java @@ -44,12 +44,14 @@ import androidx.media3.effect.GlShaderProgram; import androidx.media3.effect.MatrixTransformation; import androidx.media3.effect.PassthroughShaderProgram; import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.SeekParameters; import androidx.media3.exoplayer.analytics.AnalyticsListener; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.initialization.qual.Initialized; @@ -64,6 +66,46 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; */ /* package */ final class ExperimentalFrameExtractor implements AnalyticsListener { + /** Configuration for the frame extractor. */ + // TODO: b/350498258 - Add configuration for decoder selection. + public static final class Configuration { + + /** A builder for {@link Configuration} instances. */ + public static final class Builder { + private SeekParameters seekParameters; + + /** Creates a new instance with default values. */ + public Builder() { + seekParameters = SeekParameters.DEFAULT; + } + + /** + * Sets the parameters that control how seek operations are performed. Defaults to {@link + * SeekParameters#DEFAULT}. + * + * @param seekParameters The {@link SeekParameters}. + * @return This builder. + */ + @CanIgnoreReturnValue + public Builder setSeekParameters(SeekParameters seekParameters) { + this.seekParameters = seekParameters; + return this; + } + + /** Builds a new {@link Configuration} instance. */ + public Configuration build() { + return new Configuration(seekParameters); + } + } + + /** The {@link SeekParameters}. */ + public final SeekParameters seekParameters; + + private Configuration(SeekParameters seekParameters) { + this.seekParameters = seekParameters; + } + } + /** Stores an extracted and decoded video frame. */ public static final class Frame { @@ -109,10 +151,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param mediaItem The {@link MediaItem} from which frames are extracted. */ // TODO: b/350498258 - Support changing the MediaItem. - // TODO: b/350498258 - Add configuration options such as SeekParameters. // TODO: b/350498258 - Support video effects. - public ExperimentalFrameExtractor(Context context, MediaItem mediaItem) { - player = new ExoPlayer.Builder(context).build(); + public ExperimentalFrameExtractor( + Context context, Configuration configuration, MediaItem mediaItem) { + player = new ExoPlayer.Builder(context).setSeekParameters(configuration.seekParameters).build(); playerApplicationThreadHandler = new Handler(player.getApplicationLooper()); lastRequestedFrameFuture = SettableFuture.create(); // TODO: b/350498258 - Extracting the first frame is a workaround for ExoPlayer.setVideoEffects