diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java index 7f8b56f1f8..694d565d69 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java @@ -258,7 +258,8 @@ public final class MediaCodecUtil { /** * Returns a copy of the provided decoder list sorted such that software decoders are listed - * first. + * first. Break ties by listing non-{@link MediaCodecInfo#vendor} decoders first, due to issues + * with decoder reuse with some software vendor codecs. See b/382447848. * *

The returned list is not modifiable. */ @@ -266,7 +267,9 @@ public final class MediaCodecUtil { public static List getDecoderInfosSortedBySoftwareOnly( List decoderInfos) { decoderInfos = new ArrayList<>(decoderInfos); - sortByScore(decoderInfos, decoderInfo -> decoderInfo.softwareOnly ? 1 : 0); + sortByScore( + decoderInfos, + decoderInfo -> (decoderInfo.softwareOnly ? 2 : 0) + (decoderInfo.vendor ? 0 : 1)); return ImmutableList.copyOf(decoderInfos); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java index 9ed8c6b4cf..a3eb12ed4e 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; import static androidx.media3.common.PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND; +import static androidx.media3.common.PlaybackException.ERROR_CODE_SETUP_REQUIRED; import static androidx.media3.exoplayer.SeekParameters.CLOSEST_SYNC; import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; @@ -30,6 +31,7 @@ import android.app.Instrumentation; import android.content.Context; import android.graphics.Bitmap; import androidx.media3.common.MediaItem; +import androidx.media3.common.PlaybackException; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.NullableType; import androidx.media3.effect.Presentation; @@ -92,10 +94,8 @@ public class FrameExtractorTest { public void extractFrame_oneFrame_returnsNearest() throws Exception { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500); Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); @@ -120,10 +120,10 @@ public class FrameExtractorTest { public void extractFrame_oneFrameWithPresentationEffect_returnsScaledFrame() throws Exception { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of(Presentation.createForHeight(180))); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem( + MediaItem.fromUri(FILE_PATH), + /* effects= */ ImmutableList.of(Presentation.createForHeight(180))); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500); Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); @@ -148,10 +148,8 @@ public class FrameExtractorTest { public void extractFrame_pastDuration_returnsLastFrame() throws Exception { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 200_000); Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); @@ -177,10 +175,8 @@ public class FrameExtractorTest { public void extractFrame_repeatedPositionMs_returnsTheSameFrame() throws Exception { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); ImmutableList requestedFramePositionsMs = ImmutableList.of(0L, 0L, 33L, 34L, 34L); ImmutableList expectedFramePositionsMs = ImmutableList.of(0L, 0L, 33L, 66L, 66L); List> frameFutures = new ArrayList<>(); @@ -217,9 +213,8 @@ public class FrameExtractorTest { context, new ExperimentalFrameExtractor.Configuration.Builder() .setSeekParameters(CLOSEST_SYNC) - .build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + .build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); ImmutableList requestedFramePositionsMs = ImmutableList.of(0L, 0L, 33L, 34L, 34L); ImmutableList expectedFramePositionsMs = ImmutableList.of(0L, 0L, 0L, 0L, 0L); List> frameFutures = new ArrayList<>(); @@ -253,10 +248,8 @@ public class FrameExtractorTest { public void extractFrame_randomAccess_returnsCorrectFrames() throws Exception { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); ListenableFuture frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000); ListenableFuture frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000); @@ -284,9 +277,8 @@ public class FrameExtractorTest { context, new ExperimentalFrameExtractor.Configuration.Builder() .setSeekParameters(CLOSEST_SYNC) - .build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + .build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); ListenableFuture frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000); ListenableFuture frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000); @@ -314,10 +306,8 @@ public class FrameExtractorTest { String filePath = "asset:///nonexistent"; frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(filePath), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem(MediaItem.fromUri(filePath), /* effects= */ ImmutableList.of()); ListenableFuture frame0 = frameExtractor.getFrame(/* positionMs= */ 0); @@ -328,14 +318,27 @@ public class FrameExtractorTest { .isEqualTo(ERROR_CODE_IO_FILE_NOT_FOUND); } + @Test + public void getFrame_withoutMediaItem_throws() { + frameExtractor = + new ExperimentalFrameExtractor( + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + + ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500); + + ExecutionException thrown = + assertThrows(ExecutionException.class, () -> frameFuture.get(TIMEOUT_SECONDS, SECONDS)); + assertThat(thrown).hasCauseThat().isInstanceOf(PlaybackException.class); + assertThat(((PlaybackException) thrown.getCause()).errorCode) + .isEqualTo(ERROR_CODE_SETUP_REQUIRED); + } + @Test public void extractFrame_oneFrame_completesViaCallback() throws Exception { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); AtomicReference<@NullableType Frame> frameAtomicReference = new AtomicReference<>(); AtomicReference<@NullableType Throwable> throwableAtomicReference = new AtomicReference<>(); ConditionVariable frameReady = new ConditionVariable(); @@ -373,10 +376,8 @@ public class FrameExtractorTest { public void frameExtractor_releaseOnPlayerLooper_returns() { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); instrumentation.runOnMainSync(frameExtractor::release); @@ -387,10 +388,9 @@ public class FrameExtractorTest { public void extractFrame_oneFrameRotated_returnsFrameInCorrectOrientation() throws Exception { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(MP4_TRIM_OPTIMIZATION_270.uri), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem( + MediaItem.fromUri(MP4_TRIM_OPTIMIZATION_270.uri), /* effects= */ ImmutableList.of()); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 0); Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); @@ -415,10 +415,8 @@ public class FrameExtractorTest { public void extractFrame_randomAccessWithCancellation_returnsCorrectFrames() throws Exception { frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(FILE_PATH), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); ListenableFuture frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000); ListenableFuture frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000); @@ -440,4 +438,39 @@ public class FrameExtractorTest { .renderedOutputBufferCount) .isEqualTo(4); } + + @Test + public void extractFrame_changeMediaItem_extractsFrameFromTheCorrectItem() throws Exception { + frameExtractor = + new ExperimentalFrameExtractor( + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem( + MediaItem.fromUri(MP4_TRIM_OPTIMIZATION_270.uri), /* effects= */ ImmutableList.of()); + ListenableFuture frameFutureFirstItem = frameExtractor.getFrame(/* positionMs= */ 0); + frameExtractor.setMediaItem(MediaItem.fromUri(FILE_PATH), /* effects= */ ImmutableList.of()); + ListenableFuture frameFutureSecondItem = + frameExtractor.getFrame(/* positionMs= */ 8_500); + + Frame frameFirstItem = frameFutureFirstItem.get(TIMEOUT_SECONDS, SECONDS); + Bitmap actualBitmapFirstItem = frameFirstItem.bitmap; + Bitmap expectedBitmapFirstItem = + readBitmap( + /* assetString= */ GOLDEN_ASSET_FOLDER_PATH + + "internal_emulator_transformer_output_180_rotated_0.000.png"); + maybeSaveTestBitmap( + testId, /* bitmapLabel= */ "firstItem", actualBitmapFirstItem, /* path= */ null); + Frame frameSecondItem = frameFutureSecondItem.get(TIMEOUT_SECONDS, SECONDS); + Bitmap actualBitmapSecondItem = frameSecondItem.bitmap; + Bitmap expectedBitmapSecondItem = + readBitmap( + /* assetString= */ GOLDEN_ASSET_FOLDER_PATH + + "sample_with_increasing_timestamps_360p_8.531.png"); + maybeSaveTestBitmap( + testId, /* bitmapLabel= */ "secondItem", actualBitmapSecondItem, /* path= */ null); + + assertThat(frameFirstItem.presentationTimeMs).isEqualTo(0); + assertBitmapsAreSimilar(expectedBitmapFirstItem, actualBitmapFirstItem, PSNR_THRESHOLD); + assertThat(frameSecondItem.presentationTimeMs).isEqualTo(8_531); + assertBitmapsAreSimilar(expectedBitmapSecondItem, actualBitmapSecondItem, PSNR_THRESHOLD); + } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameExtractorHdrTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameExtractorHdrTest.java index de7ae48c0a..d86655a56b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameExtractorHdrTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameExtractorHdrTest.java @@ -23,6 +23,7 @@ import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; import static androidx.media3.test.utils.TestUtil.assertBitmapsAreSimilar; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_COLOR_TEST_1080P_HLG10; +import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_270; import static androidx.media3.transformer.mh.HdrCapabilitiesUtil.assumeDeviceSupportsOpenGlToneMapping; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.SECONDS; @@ -56,8 +57,8 @@ public class FrameExtractorHdrTest { "test-generated-goldens/sample_mp4_first_frame/electrical_colors/tone_map_hlg_to_sdr.png"; // File names in test-generated-goldens/FrameExtractorTest end with the presentation time of the // extracted frame in seconds and milliseconds (_0.000 for 0s ; _1.567 for 1.567 seconds). - private static final String EXTRACT_HLG_PNG_ASSET_PATH = - "test-generated-goldens/FrameExtractorTest/hlg10-color-test_0.000.png"; + private static final String GOLDEN_ASSET_FOLDER_PATH = + "test-generated-goldens/FrameExtractorTest/"; private static final long TIMEOUT_SECONDS = 10; private static final float PSNR_THRESHOLD = 25f; @@ -85,10 +86,9 @@ public class FrameExtractorHdrTest { assumeDeviceSupportsOpenGlToneMapping(testId, MP4_ASSET_COLOR_TEST_1080P_HLG10.videoFormat); frameExtractor = new ExperimentalFrameExtractor( - context, - new ExperimentalFrameExtractor.Configuration.Builder().build(), - MediaItem.fromUri(MP4_ASSET_COLOR_TEST_1080P_HLG10.uri), - /* effects= */ ImmutableList.of()); + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem( + MediaItem.fromUri(MP4_ASSET_COLOR_TEST_1080P_HLG10.uri), /* effects= */ ImmutableList.of()); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 0); @@ -111,16 +111,17 @@ public class FrameExtractorHdrTest { context, new ExperimentalFrameExtractor.Configuration.Builder() .setExtractHdrFrames(true) - .build(), - MediaItem.fromUri(MP4_ASSET_COLOR_TEST_1080P_HLG10.uri), - /* effects= */ ImmutableList.of()); + .build()); + frameExtractor.setMediaItem( + MediaItem.fromUri(MP4_ASSET_COLOR_TEST_1080P_HLG10.uri), /* effects= */ ImmutableList.of()); ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 0); ExperimentalFrameExtractor.Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); Bitmap actualBitmap = frame.bitmap; Bitmap actualBitmapDefaultColorSpace = removeColorSpace(actualBitmap); - Bitmap expectedBitmap = readBitmap(EXTRACT_HLG_PNG_ASSET_PATH); + Bitmap expectedBitmap = + readBitmap(/* assetString= */ GOLDEN_ASSET_FOLDER_PATH + "hlg10-color-test_0.000.png"); maybeSaveTestBitmap( testId, /* bitmapLabel= */ "actualBitmapDefaultColorSpace", @@ -133,6 +134,45 @@ public class FrameExtractorHdrTest { assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD); } + @Test + public void + extractFrame_changeMediaItemFromHdrToSdrWithToneMapping_extractsFrameFromTheCorrectItem() + throws Exception { + assumeDeviceSupportsOpenGlToneMapping(testId, MP4_ASSET_COLOR_TEST_1080P_HLG10.videoFormat); + frameExtractor = + new ExperimentalFrameExtractor( + context, new ExperimentalFrameExtractor.Configuration.Builder().build()); + frameExtractor.setMediaItem( + MediaItem.fromUri(MP4_ASSET_COLOR_TEST_1080P_HLG10.uri), /* effects= */ ImmutableList.of()); + ListenableFuture frameFutureFirstItem = + frameExtractor.getFrame(/* positionMs= */ 0); + frameExtractor.setMediaItem( + MediaItem.fromUri(MP4_TRIM_OPTIMIZATION_270.uri), /* effects= */ ImmutableList.of()); + ListenableFuture frameFutureSecondItem = + frameExtractor.getFrame(/* positionMs= */ 0); + + ExperimentalFrameExtractor.Frame frameFirstItem = + frameFutureFirstItem.get(TIMEOUT_SECONDS, SECONDS); + Bitmap actualBitmapFirstItem = frameFirstItem.bitmap; + Bitmap expectedBitmapFirstItem = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH); + maybeSaveTestBitmap( + testId, /* bitmapLabel= */ "firstItem", actualBitmapFirstItem, /* path= */ null); + ExperimentalFrameExtractor.Frame frameSecondItem = + frameFutureSecondItem.get(TIMEOUT_SECONDS, SECONDS); + Bitmap actualBitmapSecondItem = frameSecondItem.bitmap; + Bitmap expectedBitmapSecondItem = + readBitmap( + /* assetString= */ GOLDEN_ASSET_FOLDER_PATH + + "internal_emulator_transformer_output_180_rotated_0.000.png"); + maybeSaveTestBitmap( + testId, /* bitmapLabel= */ "secondItem", actualBitmapSecondItem, /* path= */ null); + + assertThat(frameFirstItem.presentationTimeMs).isEqualTo(0); + assertBitmapsAreSimilar(expectedBitmapFirstItem, actualBitmapFirstItem, PSNR_THRESHOLD); + assertThat(frameSecondItem.presentationTimeMs).isEqualTo(0); + assertBitmapsAreSimilar(expectedBitmapSecondItem, actualBitmapSecondItem, PSNR_THRESHOLD); + } + /** * Copy the contents of the input {@link Bitmap} into a {@link Bitmap} with {@link * Bitmap.Config#RGBA_F16} config and default {@link ColorSpace}. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java index ba709d4ed1..04f710866b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java @@ -24,11 +24,13 @@ import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED; import static androidx.media3.common.ColorInfo.isTransferHdr; import static androidx.media3.common.PlaybackException.ERROR_CODE_FAILED_RUNTIME_CHECK; import static androidx.media3.common.PlaybackException.ERROR_CODE_INVALID_STATE; +import static androidx.media3.common.PlaybackException.ERROR_CODE_SETUP_REQUIRED; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.GlUtil.createRgb10A2Texture; import static androidx.media3.common.util.Util.SDK_INT; import static androidx.media3.common.util.Util.usToMs; +import static com.google.common.util.concurrent.Futures.immediateCancelledFuture; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; @@ -77,6 +79,7 @@ import androidx.media3.exoplayer.SeekParameters; import androidx.media3.exoplayer.analytics.AnalyticsListener; import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter; import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; +import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.video.MediaCodecVideoRenderer; import androidx.media3.exoplayer.video.VideoRendererEventListener; import com.google.common.collect.ImmutableList; @@ -90,7 +93,6 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** @@ -120,7 +122,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * */ @UnstableApi -public final class ExperimentalFrameExtractor implements AnalyticsListener { +public final class ExperimentalFrameExtractor { /** Configuration for the frame extractor. */ public static final class Configuration { @@ -261,20 +263,15 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener { * The last {@link Frame} that was extracted successfully. Accessed on the {@linkplain * ExoPlayer#getApplicationLooper() ExoPlayer application thread}. */ - private @MonotonicNonNull Frame lastExtractedFrame; + @Nullable private Frame lastExtractedFrame; /** * Creates an instance. * * @param context {@link Context}. * @param configuration The {@link Configuration} for this frame extractor. - * @param mediaItem The {@link MediaItem} from which frames are extracted. - * @param effects The {@link List} of {@linkplain Effect video effects} to apply to the extracted - * video frames. */ - // TODO: b/350498258 - Support changing the MediaItem. - public ExperimentalFrameExtractor( - Context context, Configuration configuration, MediaItem mediaItem, List effects) { + public ExperimentalFrameExtractor(Context context, Configuration configuration) { player = new ExoPlayer.Builder( context, @@ -292,29 +289,41 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener { }) .setSeekParameters(configuration.seekParameters) .build(); + player.addAnalyticsListener(new PlayerListener()); playerApplicationThreadHandler = new Handler(player.getApplicationLooper()); extractedFrameNeedsRendering = new AtomicBoolean(); - // TODO: b/350498258 - Extracting the first frame is a workaround for ExoPlayer.setVideoEffects - // returning incorrect timestamps if we seek the player before rendering starts from zero. frameBeingExtractedCompleterAtomicReference = new AtomicReference<>(null); + lastRequestedFrameFuture = immediateCancelledFuture(); + } + + /** + * Sets a new {@link MediaItem}. + * + *

Changing between SDR and HDR {@link MediaItem}s is not supported when {@link + * Configuration#extractHdrFrames} is true. + * + * @param mediaItem The {@link MediaItem} from which frames will be extracted. + * @param effects The {@link List} of {@linkplain Effect video effects} to apply to the extracted + * video frames. + */ + public void setMediaItem(MediaItem mediaItem, List effects) { + ListenableFuture previousRequestedFrame = lastRequestedFrameFuture; + // TODO: b/350498258 - Extracting the first frame is a workaround for ExoPlayer.setVideoEffects + // returning incorrect timestamps if we seek the player before rendering starts from zero. lastRequestedFrameFuture = CallbackToFutureAdapter.getFuture( completer -> { - frameBeingExtractedCompleterAtomicReference.set(completer); - // TODO: b/350498258 - Refactor this and remove declaring this reference as - // initialized to satisfy the nullness checker. - @SuppressWarnings("nullness:assignment") - @Initialized - ExperimentalFrameExtractor thisRef = this; - playerApplicationThreadHandler.post( + previousRequestedFrame.addListener( () -> { - player.addAnalyticsListener(thisRef); - player.setVideoEffects(thisRef.buildVideoEffects(effects)); + frameBeingExtractedCompleterAtomicReference.set(completer); + lastExtractedFrame = null; + player.setVideoEffects(buildVideoEffects(effects)); player.setMediaItem(mediaItem); player.setPlayWhenReady(false); player.prepare(); - }); - return "ExperimentalFrameExtractor constructor"; + }, + playerApplicationThreadHandler::post); + return "ExperimentalFrameExtractor.setMediaItem"; }); } @@ -369,6 +378,12 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener { } if (playerError != null) { completer.setException(playerError); + } else if (player.getCurrentMediaItem() == null) { + completer.setException( + new PlaybackException( + "Player has no current item. Call setMediaItem before getFrame.", + null, + ERROR_CODE_SETUP_REQUIRED)); } else { checkState(frameBeingExtractedCompleterAtomicReference.compareAndSet(null, completer)); extractedFrameNeedsRendering.set(false); @@ -395,35 +410,6 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener { waitForRelease.blockUninterruptible(); } - // AnalyticsListener - - @Override - public void onPlayerError(EventTime eventTime, PlaybackException error) { - // Fail the next frame to be extracted. Errors will propagate to later pending requests via - // Future callbacks. - @Nullable - CallbackToFutureAdapter.Completer frameBeingExtractedCompleter = - frameBeingExtractedCompleterAtomicReference.getAndSet(null); - if (frameBeingExtractedCompleter != null) { - frameBeingExtractedCompleter.setException(error); - } - } - - @Override - public void onPlaybackStateChanged(EventTime eventTime, @Player.State int state) { - // The player enters STATE_BUFFERING at the start of a seek. - // At the end of a seek, the player enters STATE_READY after the video renderer position has - // been reset, and the renderer reports that it's ready. - if (state == Player.STATE_READY && !extractedFrameNeedsRendering.get()) { - // If the seek resolves to the current position, the renderer position will not be reset - // and extractedFrameNeedsRendering remains false. No frames are rendered. Repeat the - // previously returned frame. - CallbackToFutureAdapter.Completer frameBeingExtractedCompleter = - checkNotNull(frameBeingExtractedCompleterAtomicReference.getAndSet(null)); - frameBeingExtractedCompleter.set(checkNotNull(lastExtractedFrame)); - } - } - @VisibleForTesting /* package */ ListenableFuture<@NullableType DecoderCounters> getDecoderCounters() { SettableFuture<@NullableType DecoderCounters> decoderCountersSettableFuture = @@ -447,6 +433,35 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener { return listBuilder.build(); } + private final class PlayerListener implements AnalyticsListener { + @Override + public void onPlayerError(EventTime eventTime, PlaybackException error) { + // Fail the next frame to be extracted. Errors will propagate to later pending requests via + // Future callbacks. + @Nullable + CallbackToFutureAdapter.Completer frameBeingExtractedCompleter = + frameBeingExtractedCompleterAtomicReference.getAndSet(null); + if (frameBeingExtractedCompleter != null) { + frameBeingExtractedCompleter.setException(error); + } + } + + @Override + public void onPlaybackStateChanged(EventTime eventTime, @Player.State int state) { + // The player enters STATE_BUFFERING at the start of a seek. + // At the end of a seek, the player enters STATE_READY after the video renderer position has + // been reset, and the renderer reports that it's ready. + if (state == Player.STATE_READY && !extractedFrameNeedsRendering.get()) { + // If the seek resolves to the current position, the renderer position will not be reset + // and extractedFrameNeedsRendering remains false. No frames are rendered. Repeat the + // previously returned frame. + CallbackToFutureAdapter.Completer frameBeingExtractedCompleter = + checkNotNull(frameBeingExtractedCompleterAtomicReference.getAndSet(null)); + frameBeingExtractedCompleter.set(checkNotNull(lastExtractedFrame)); + } + } + } + private final class FrameReader implements GlEffect { @Override public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) @@ -609,7 +624,7 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener { private boolean frameRenderedSinceLastPositionReset; private List effectsFromPlayer; - private @MonotonicNonNull Effect rotation; + @Nullable private Effect rotation; public FrameExtractorRenderer( Context context, @@ -627,6 +642,18 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener { effectsFromPlayer = ImmutableList.of(); } + @Override + protected void onStreamChanged( + Format[] formats, + long startPositionUs, + long offsetUs, + MediaSource.MediaPeriodId mediaPeriodId) + throws ExoPlaybackException { + super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); + frameRenderedSinceLastPositionReset = false; + setRotation(null); + } + @Override public void setVideoEffects(List effects) { effectsFromPlayer = effects; @@ -653,17 +680,21 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener { // Some decoders do not apply rotation. It's no extra cost to rotate with a GL matrix // transformation effect instead. // https://developer.android.com/reference/android/media/MediaCodec#transformations-when-rendering-onto-surface - rotation = + setRotation( new ScaleAndRotateTransformation.Builder() .setRotationDegrees(360 - format.rotationDegrees) - .build(); - setEffectsWithRotation(); + .build()); formatHolder.format = format.buildUpon().setRotationDegrees(0).build(); } } return super.onInputFormatChanged(formatHolder); } + private void setRotation(@Nullable Effect rotation) { + this.rotation = rotation; + setEffectsWithRotation(); + } + private void setEffectsWithRotation() { ImmutableList.Builder effectBuilder = new ImmutableList.Builder<>(); if (rotation != null) {