From ac347983449fc414c5ea8110f22e974fb8046370 Mon Sep 17 00:00:00 2001 From: michaelkatz Date: Fri, 31 May 2024 02:19:25 -0700 Subject: [PATCH] Schedule doSomeWork when MediaCodec signals available buffers When running in asynchronous mode, MediaCodec will be running the CPU to signal input and output buffers being made available for use by the player. With ExoPlayer.experimentalSetDynamicSchedulingEnabled set to true, ExoPlayer will wakeup to make rendering progress when MediaCodec raises these signals. In this way, ExoPlayer work will align more closely with CPU wake-cycles. PiperOrigin-RevId: 638962108 --- RELEASENOTES.md | 6 ++ .../audio/MediaCodecAudioRenderer.java | 13 ++- .../AsynchronousMediaCodecAdapter.java | 6 ++ .../AsynchronousMediaCodecCallback.java | 24 ++++++ .../mediacodec/MediaCodecAdapter.java | 32 ++++++++ .../mediacodec/MediaCodecRenderer.java | 80 +++++++++++++++++++ .../AsynchronousMediaCodecCallbackTest.java | 36 +++++++++ 7 files changed, 190 insertions(+), 7 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index bc916de7f1..56440638f5 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -50,6 +50,12 @@ order for the renderer to progress. If `ExoPlayer` is set with `experimentalSetDynamicSchedulingEnabled` then `ExoPlayer` will call this method when calculating the time to schedule its work task. + * Add `MediaCodecAdapter#OnBufferAvailableListener` to alert when input + and output buffers are available for use by `MediaCodecRenderer`. + `MediaCodecRenderer` will signal `ExoPlayer` when receiving these + callbacks and if `ExoPlayer` is set with + `experimentalSetDynamicSchedulingEnabled`, then `ExoPlayer` will + schedule its work loop as renderers can make progress. * Transformer: * Work around a decoder bug where the number of audio channels was capped at stereo when handling PCM input. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java index 6d9668f243..24d4cf307b 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java @@ -119,8 +119,6 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media private long currentPositionUs; private boolean allowPositionDiscontinuity; private boolean audioSinkNeedsReset; - - @Nullable private WakeupListener wakeupListener; private boolean hasPendingReportedSkippedSilence; private int rendererPriority; private boolean isStarted; @@ -480,7 +478,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } @Override - public long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) { + public long getDurationToProgressUs( + boolean isOnBufferAvailableListenerRegistered, long positionUs, long elapsedRealtimeUs) { if (nextBufferToWritePresentationTimeUs != C.TIME_UNSET) { long durationUs = (long) @@ -493,7 +492,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } return max(DEFAULT_DURATION_TO_PROGRESS_US, durationUs); } - return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs); + return super.getDurationToProgressUs( + isOnBufferAvailableListenerRegistered, positionUs, elapsedRealtimeUs); } @Override @@ -854,9 +854,6 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media case MSG_SET_AUDIO_SESSION_ID: audioSink.setAudioSessionId((Integer) checkNotNull(message)); break; - case MSG_SET_WAKEUP_LISTENER: - this.wakeupListener = (WakeupListener) message; - break; case MSG_SET_PRIORITY: rendererPriority = (int) checkNotNull(message); updateCodecImportance(); @@ -1073,6 +1070,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media @Override public void onOffloadBufferEmptying() { + WakeupListener wakeupListener = getWakeupListener(); if (wakeupListener != null) { wakeupListener.onWakeup(); } @@ -1080,6 +1078,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media @Override public void onOffloadBufferFull() { + WakeupListener wakeupListener = getWakeupListener(); if (wakeupListener != null) { wakeupListener.onSleep(); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java index 01b9028fc3..f98b40deb5 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java @@ -274,6 +274,12 @@ import java.nio.ByteBuffer; handler); } + @Override + public boolean registerOnBufferAvailableListener(OnBufferAvailableListener listener) { + asynchronousMediaCodecCallback.setOnBufferAvailableListener(listener); + return true; + } + @Override public void setOutputSurface(Surface surface) { codec.setOutputSurface(surface); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallback.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallback.java index dd40266f4f..cb0c03de9b 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallback.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallback.java @@ -77,6 +77,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Nullable private IllegalStateException internalException; + @GuardedBy("lock") + @Nullable + private MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener; + /** * Creates a new instance. * @@ -210,6 +214,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public void onInputBufferAvailable(MediaCodec codec, int index) { synchronized (lock) { availableInputBuffers.addLast(index); + if (onBufferAvailableListener != null) { + onBufferAvailableListener.onInputBufferAvailable(); + } } } @@ -222,6 +229,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } availableOutputBuffers.addLast(index); bufferInfos.add(info); + if (onBufferAvailableListener != null) { + onBufferAvailableListener.onOutputBufferAvailable(); + } } } @@ -247,6 +257,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + /** + * Sets the {@link MediaCodecAdapter.OnBufferAvailableListener} that will be notified when {@link + * #onInputBufferAvailable} and {@link #onOutputBufferAvailable} are called. + * + * @param onBufferAvailableListener The listener that will be notified when {@link + * #onInputBufferAvailable} and {@link #onOutputBufferAvailable} are called. + */ + public void setOnBufferAvailableListener( + MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener) { + synchronized (lock) { + this.onBufferAvailableListener = onBufferAvailableListener; + } + } + private void onFlushCompleted() { synchronized (lock) { if (shutDown) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java index 1adc7b19f5..1599c4ae28 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java @@ -151,6 +151,23 @@ public interface MediaCodecAdapter { void onFrameRendered(MediaCodecAdapter codec, long presentationTimeUs, long nanoTime); } + /** Listener to be called when an input or output buffer becomes available. */ + interface OnBufferAvailableListener { + /** + * Called when an input buffer becomes available. + * + * @see MediaCodec.Callback#onInputBufferAvailable(MediaCodec, int) + */ + default void onInputBufferAvailable() {} + + /** + * Called when an output buffer becomes available. + * + * @see MediaCodec.Callback#onOutputBufferAvailable(MediaCodec, int, MediaCodec.BufferInfo) + */ + default void onOutputBufferAvailable() {} + } + /** * Returns the next available input buffer index from the underlying {@link MediaCodec} or {@link * MediaCodec#INFO_TRY_AGAIN_LATER} if no such buffer exists. @@ -252,6 +269,21 @@ public interface MediaCodecAdapter { @RequiresApi(23) void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler); + /** + * Registers a listener that will be called when an input or output buffer becomes available. + * + *

Returns false if listener was not successfully registered for callbacks. + * + * @see MediaCodec.Callback#onInputBufferAvailable + * @see MediaCodec.Callback#onOutputBufferAvailable + * @return Whether listener was successfully registered. + */ + @RequiresApi(21) + default boolean registerOnBufferAvailableListener( + MediaCodecAdapter.OnBufferAvailableListener listener) { + return false; + } + /** * Dynamically sets the output surface of a {@link MediaCodec}. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 302b76244f..979ed7a21f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -346,6 +346,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { @Nullable private Format outputFormat; @Nullable private DrmSession codecDrmSession; @Nullable private DrmSession sourceDrmSession; + @Nullable private WakeupListener wakeupListener; /** * A framework {@link MediaCrypto} for use with {@link MediaCodec#queueSecureInputBuffer(int, int, @@ -382,6 +383,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private boolean codecNeedsAdaptationWorkaroundBuffer; private boolean shouldSkipAdaptationWorkaroundOutputBuffer; private boolean codecNeedsEosPropagation; + private boolean codecRegisteredOnBufferAvailableListener; private long codecHotswapDeadlineMs; private int inputIndex; private int outputIndex; @@ -503,6 +505,37 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected abstract @Capabilities int supportsFormat( MediaCodecSelector mediaCodecSelector, Format format) throws DecoderQueryException; + @Override + public final long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) { + return getDurationToProgressUs( + /* isOnBufferAvailableListenerRegistered= */ codecRegisteredOnBufferAvailableListener, + positionUs, + elapsedRealtimeUs); + } + + /** + * Returns minimum time playback must advance in order for the {@link #render} call to make + * progress. + * + *

If the {@code Renderer} has a registered {@link + * MediaCodecAdapter.OnBufferAvailableListener}, then the {@code Renderer} will be notified when + * decoder input and output buffers become available. These callbacks may affect the calculated + * minimum time playback must advance before a {@link #render} call can make progress. + * + * @param isOnBufferAvailableListenerRegistered Whether the {@code Renderer} is using a {@link + * MediaCodecAdapter} with successfully registered {@link + * MediaCodecAdapter.OnBufferAvailableListener OnBufferAvailableListener}. + * @param positionUs The current media time in microseconds, measured at the start of the current + * iteration of the rendering loop. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * measured at the start of the current iteration of the rendering loop. + * @return minimum time playback must advance before renderer is able to make progress. + */ + protected long getDurationToProgressUs( + boolean isOnBufferAvailableListenerRegistered, long positionUs, long elapsedRealtimeUs) { + return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs); + } + /** * Returns a list of decoders that can decode media in the specified format, in priority order. * @@ -797,6 +830,16 @@ public abstract class MediaCodecRenderer extends BaseRenderer { // Do nothing. Overridden to remove throws clause. } + @Override + public void handleMessage(@MessageType int messageType, @Nullable Object message) + throws ExoPlaybackException { + if (messageType == MSG_SET_WAKEUP_LISTENER) { + this.wakeupListener = (WakeupListener) message; + } else { + super.handleMessage(messageType, message); + } + } + @Override public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { if (pendingOutputEndOfStream) { @@ -971,6 +1014,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { codecNeedsEosBufferTimestampWorkaround = false; codecNeedsMonoChannelCountWorkaround = false; codecNeedsEosPropagation = false; + codecRegisteredOnBufferAvailableListener = false; codecReconfigured = false; codecReconfigurationState = RECONFIGURATION_STATE_NONE; } @@ -1193,6 +1237,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer { try { TraceUtil.beginSection("createCodec:" + codecName); codec = codecAdapterFactory.createAdapter(configuration); + codecRegisteredOnBufferAvailableListener = + Util.SDK_INT >= 21 + && Api21.registerOnBufferAvailableListener( + codec, new MediaCodecRendererCodecAdapterListener()); } finally { TraceUtil.endSection(); } @@ -1813,6 +1861,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer { return CODEC_OPERATING_RATE_UNSET; } + /** Returns listener used to signal that {@link #render(long, long)} should be called. */ + @Nullable + protected final WakeupListener getWakeupListener() { + return wakeupListener; + } + /** * Updates the codec operating rate, or triggers codec release and re-initialization if a * previously set operating rate needs to be cleared. @@ -2691,6 +2745,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } } + @RequiresApi(21) + private static final class Api21 { + @DoNotInline + public static boolean registerOnBufferAvailableListener( + MediaCodecAdapter codec, MediaCodecRendererCodecAdapterListener listener) { + return codec.registerOnBufferAvailableListener(listener); + } + } + @RequiresApi(31) private static final class Api31 { private Api31() {} @@ -2704,4 +2767,21 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } } } + + private final class MediaCodecRendererCodecAdapterListener + implements MediaCodecAdapter.OnBufferAvailableListener { + @Override + public void onInputBufferAvailable() { + if (wakeupListener != null) { + wakeupListener.onWakeup(); + } + } + + @Override + public void onOutputBufferAvailable() { + if (wakeupListener != null) { + wakeupListener.onWakeup(); + } + } + } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallbackTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallbackTest.java index eacb81ceb2..a16830baae 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallbackTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallbackTest.java @@ -530,6 +530,42 @@ public class AsynchronousMediaCodecCallbackTest { asynchronousMediaCodecCallback.shutdown(); } + @Test + public void onInputBufferAvailable_withOnBufferAvailableListener_callsOnInputBufferAvailable() { + AtomicInteger onInputBufferAvailableCounter = new AtomicInteger(); + MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener = + new MediaCodecAdapter.OnBufferAvailableListener() { + @Override + public void onInputBufferAvailable() { + onInputBufferAvailableCounter.getAndIncrement(); + } + }; + asynchronousMediaCodecCallback.setOnBufferAvailableListener(onBufferAvailableListener); + + // Send an input buffer to the callback. + asynchronousMediaCodecCallback.onInputBufferAvailable(codec, 0); + + assertThat(onInputBufferAvailableCounter.get()).isEqualTo(1); + } + + @Test + public void onOutputBufferAvailable_withOnBufferAvailableListener_callsOnOutputBufferAvailable() { + AtomicInteger onOutputBufferAvailableCounter = new AtomicInteger(); + MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener = + new MediaCodecAdapter.OnBufferAvailableListener() { + @Override + public void onOutputBufferAvailable() { + onOutputBufferAvailableCounter.getAndIncrement(); + } + }; + asynchronousMediaCodecCallback.setOnBufferAvailableListener(onBufferAvailableListener); + + // Send an output buffer to the callback. + asynchronousMediaCodecCallback.onOutputBufferAvailable(codec, 0, new MediaCodec.BufferInfo()); + + assertThat(onOutputBufferAvailableCounter.get()).isEqualTo(1); + } + /** Reflectively create a {@link MediaCodec.CodecException}. */ private static MediaCodec.CodecException createCodecException() throws NoSuchMethodException,