loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Implement cancellation of FrameExtractor.getFrame

Browse Source 
Previously cancelling one FrameExtractor.getFrame
ListenableFuture caused the following requests to fail.
Implement ability to cancel Futures, and correctly issue
next player seek commands after all past requests complete.

Removes uses of SettableFuture, and replaces them with
CallbackToFutureAdapter.
Adds a dependency on androidx.concurrent:concurrent-futures

PiperOrigin-RevId: 701941403
This commit is contained in:
dancho 2024-12-02 06:06:06 -08:00 committed by Copybara-Service
parent 19b276d6a7
commit b2ba5da09b
3 changed files with 120 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libraries/transformer/build.gradle
View File

@ -48,6 +48,7 @@ android {
dependencies { dependencies {
implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
implementation 'androidx.concurrent:concurrent-futures:1.2.0'
implementation project(modulePrefix + 'lib-datasource') implementation project(modulePrefix + 'lib-datasource')
implementation project(modulePrefix + 'lib-container') implementation project(modulePrefix + 'lib-container')
api project(modulePrefix + 'lib-exoplayer') api project(modulePrefix + 'lib-exoplayer')

31
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java
View File

@ -45,6 +45,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -409,4 +410,34 @@ public class FrameExtractorTest {
.renderedOutputBufferCount) .renderedOutputBufferCount)
.isEqualTo(1); .isEqualTo(1);
} }
@Test
public void extractFrame_randomAccessWithCancellation_returnsCorrectFrames() throws Exception {
frameExtractor =
new ExperimentalFrameExtractor(
context,
new ExperimentalFrameExtractor.Configuration.Builder().build(),
MediaItem.fromUri(FILE_PATH),
/* effects= */ ImmutableList.of());
ListenableFuture<Frame> frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000);
ListenableFuture<Frame> frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000);
ListenableFuture<Frame> frame7 = frameExtractor.getFrame(/* positionMs= */ 7_000);
ListenableFuture<Frame> frame2 = frameExtractor.getFrame(/* positionMs= */ 2_000);
ListenableFuture<Frame> frame8 = frameExtractor.getFrame(/* positionMs= */ 8_000);
frame5.cancel(/* mayInterruptIfRunning= */ false);
frame7.cancel(/* mayInterruptIfRunning= */ false);
assertThat(frame3.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(3_032);
assertThat(frame2.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(2_032);
assertThat(frame8.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_031);
assertThrows(CancellationException.class, () -> frame5.get(TIMEOUT_SECONDS, SECONDS));
assertThrows(CancellationException.class, () -> frame7.get(TIMEOUT_SECONDS, SECONDS));
assertThat(
frameExtractor
.getDecoderCounters()
.get(TIMEOUT_SECONDS, SECONDS)
.renderedOutputBufferCount)
.isEqualTo(4);
}
} }

161
libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java
View File

@ -44,6 +44,7 @@ import androidx.annotation.CallSuper;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import androidx.concurrent.futures.CallbackToFutureAdapter;
import androidx.media3.common.Effect; import androidx.media3.common.Effect;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlObjectsProvider;
@ -220,18 +221,19 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener {
private final AtomicBoolean extractedFrameNeedsRendering; private final AtomicBoolean extractedFrameNeedsRendering;
/** /**
* A {@link SettableFuture} representing the frame currently being extracted. Accessed on both the * A {@link CallbackToFutureAdapter.Completer} corresponding to the frame currently being
* {@linkplain ExoPlayer#getApplicationLooper() ExoPlayer application thread}, and the video * extracted. Accessed on both the {@linkplain ExoPlayer#getApplicationLooper() ExoPlayer
* effects GL thread. * application thread}, and the video effects GL thread.
*/ */
private final AtomicReference<@NullableType SettableFuture<Frame>> private final AtomicReference<CallbackToFutureAdapter.@NullableType Completer<Frame>>
frameBeingExtractedFutureAtomicReference; frameBeingExtractedCompleterAtomicReference;
/** /**
* The last {@link SettableFuture} returned by {@link #getFrame(long)}. Accessed on the frame * A {@link ListenableFuture} that completes when all previous {@link #getFrame(long)} requests
* extractor application thread. * complete. Upon completion, the result corresponds to the last request to {@link
* #getFrame(long)}.
*/ */
private SettableFuture<Frame> lastRequestedFrameFuture; private ListenableFuture<Frame> lastRequestedFrameFuture;
/** /**
* The last {@link Frame} that was extracted successfully. Accessed on the {@linkplain * The last {@link Frame} that was extracted successfully. Accessed on the {@linkplain
@ -270,23 +272,28 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener {
.build(); .build();
playerApplicationThreadHandler = new Handler(player.getApplicationLooper()); playerApplicationThreadHandler = new Handler(player.getApplicationLooper());
extractedFrameNeedsRendering = new AtomicBoolean(); extractedFrameNeedsRendering = new AtomicBoolean();
lastRequestedFrameFuture = SettableFuture.create();
// TODO: b/350498258 - Extracting the first frame is a workaround for ExoPlayer.setVideoEffects // TODO: b/350498258 - Extracting the first frame is a workaround for ExoPlayer.setVideoEffects
// returning incorrect timestamps if we seek the player before rendering starts from zero. // returning incorrect timestamps if we seek the player before rendering starts from zero.
frameBeingExtractedFutureAtomicReference = new AtomicReference<>(lastRequestedFrameFuture); frameBeingExtractedCompleterAtomicReference = new AtomicReference<>(null);
// TODO: b/350498258 - Refactor this and remove declaring this reference as initialized lastRequestedFrameFuture =
// to satisfy the nullness checker. CallbackToFutureAdapter.getFuture(
@SuppressWarnings("nullness:assignment") completer -> {
@Initialized frameBeingExtractedCompleterAtomicReference.set(completer);
ExperimentalFrameExtractor thisRef = this; // TODO: b/350498258 - Refactor this and remove declaring this reference as
playerApplicationThreadHandler.post( // initialized to satisfy the nullness checker.
() -> { @SuppressWarnings("nullness:assignment")
player.addAnalyticsListener(thisRef); @Initialized
player.setVideoEffects(buildVideoEffects(effects)); ExperimentalFrameExtractor thisRef = this;
player.setMediaItem(mediaItem); playerApplicationThreadHandler.post(
player.setPlayWhenReady(false); () -> {
player.prepare(); player.addAnalyticsListener(thisRef);
}); player.setVideoEffects(thisRef.buildVideoEffects(effects));
player.setMediaItem(mediaItem);
player.setPlayWhenReady(false);
player.prepare();
});
return "ExperimentalFrameExtractor constructor";
});
} }
/** /**
@ -296,47 +303,55 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener {
* @return A {@link ListenableFuture} of the result. * @return A {@link ListenableFuture} of the result.
*/ */
public ListenableFuture<Frame> getFrame(long positionMs) { public ListenableFuture<Frame> getFrame(long positionMs) {
SettableFuture<Frame> frameSettableFuture = SettableFuture.create(); ListenableFuture<Frame> previousRequestedFrame = lastRequestedFrameFuture;
// Process frameSettableFuture after lastRequestedFrameFuture completes. ListenableFuture<Frame> frameListenableFuture =
// If lastRequestedFrameFuture is done, the callbacks are invoked immediately. CallbackToFutureAdapter.getFuture(
Futures.addCallback( completer -> {
lastRequestedFrameFuture, Futures.addCallback(
new FutureCallback<Frame>() { previousRequestedFrame,
@Override new FutureCallback<Frame>() {
public void onSuccess(Frame result) { @Override
playerApplicationThreadHandler.post( public void onSuccess(Frame result) {
() -> { lastExtractedFrame = result;
lastExtractedFrame = result; processNext(positionMs, completer);
@Nullable PlaybackException playerError; }
if (player.isReleased()) {
playerError =
new PlaybackException(
"The player is already released",
null,
ERROR_CODE_FAILED_RUNTIME_CHECK);
} else {
playerError = player.getPlayerError();
}
if (playerError != null) {
frameSettableFuture.setException(playerError);
} else {
checkState(
frameBeingExtractedFutureAtomicReference.compareAndSet(
null, frameSettableFuture));
extractedFrameNeedsRendering.set(false);
player.seekTo(positionMs);
}
});
}
@Override @Override
public void onFailure(Throwable t) { public void onFailure(Throwable t) {
frameSettableFuture.setException(t); processNext(positionMs, completer);
} }
}, },
directExecutor()); playerApplicationThreadHandler::post);
lastRequestedFrameFuture = frameSettableFuture; return "ExperimentalFrameExtractor.getFrame";
return lastRequestedFrameFuture; });
lastRequestedFrameFuture =
Futures.whenAllComplete(lastRequestedFrameFuture, frameListenableFuture)
.call(() -> Futures.getDone(frameListenableFuture), directExecutor());
return frameListenableFuture;
}
private void processNext(long positionMs, CallbackToFutureAdapter.Completer<Frame> completer) {
// Cancellation listener is invoked instantaneously if the returned future is already cancelled.
AtomicBoolean cancelled = new AtomicBoolean(false);
completer.addCancellationListener(() -> cancelled.set(true), directExecutor());
if (cancelled.get()) {
return;
}
@Nullable PlaybackException playerError;
if (player.isReleased()) {
playerError =
new PlaybackException(
"The player is already released", null, ERROR_CODE_FAILED_RUNTIME_CHECK);
} else {
playerError = player.getPlayerError();
}
if (playerError != null) {
completer.setException(playerError);
} else {
checkState(frameBeingExtractedCompleterAtomicReference.compareAndSet(null, completer));
extractedFrameNeedsRendering.set(false);
player.seekTo(positionMs);
}
} }
/** /**
@ -365,10 +380,10 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener {
// Fail the next frame to be extracted. Errors will propagate to later pending requests via // Fail the next frame to be extracted. Errors will propagate to later pending requests via
// Future callbacks. // Future callbacks.
@Nullable @Nullable
SettableFuture<Frame> frameBeingExtractedFuture = CallbackToFutureAdapter.Completer<Frame> frameBeingExtractedCompleter =
frameBeingExtractedFutureAtomicReference.getAndSet(null); frameBeingExtractedCompleterAtomicReference.getAndSet(null);
if (frameBeingExtractedFuture != null) { if (frameBeingExtractedCompleter != null) {
frameBeingExtractedFuture.setException(error); frameBeingExtractedCompleter.setException(error);
} }
} }
@ -381,9 +396,9 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener {
// If the seek resolves to the current position, the renderer position will not be reset // If the seek resolves to the current position, the renderer position will not be reset
// and extractedFrameNeedsRendering remains false. No frames are rendered. Repeat the // and extractedFrameNeedsRendering remains false. No frames are rendered. Repeat the
// previously returned frame. // previously returned frame.
SettableFuture<Frame> frameBeingExtractedFuture = CallbackToFutureAdapter.Completer<Frame> frameBeingExtractedCompleter =
checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null)); checkNotNull(frameBeingExtractedCompleterAtomicReference.getAndSet(null));
frameBeingExtractedFuture.set(checkNotNull(lastExtractedFrame)); frameBeingExtractedCompleter.set(checkNotNull(lastExtractedFrame));
} }
} }
@ -532,9 +547,9 @@ public final class ExperimentalFrameExtractor implements AnalyticsListener {
} }
bitmap.copyPixelsFromBuffer(byteBuffer); bitmap.copyPixelsFromBuffer(byteBuffer);
SettableFuture<Frame> frameBeingExtractedFuture = CallbackToFutureAdapter.Completer<Frame> frameBeingExtractedCompleter =
checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null)); checkNotNull(frameBeingExtractedCompleterAtomicReference.getAndSet(null));
frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs), bitmap)); frameBeingExtractedCompleter.set(new Frame(usToMs(presentationTimeUs), bitmap));
// Drop frame: do not call outputListener.onOutputFrameAvailable(). // Drop frame: do not call outputListener.onOutputFrameAvailable().
// Block effects pipeline: do not call inputListener.onReadyToAcceptInputFrame(). // Block effects pipeline: do not call inputListener.onReadyToAcceptInputFrame().
// The effects pipeline will unblock and receive new frames when flushed after a seek. // The effects pipeline will unblock and receive new frames when flushed after a seek.