diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 0073540f1c..196074485a 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -164,6 +164,7 @@ * No longer use a `MediaCodec` in audio passthrough mode. * Check `DefaultAudioSink` supports passthrough, in addition to checking the `AudioCapabilities` + * Add an experimental scheduling mode to save power in offload. ([#7404](https://github.com/google/ExoPlayer/issues/7404)). * Adjust input timestamps in `MediaCodecRenderer` to account for the Codec2 MP3 decoder having lower timestamps on the output side. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java index bd56974b32..3913922c3c 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java @@ -219,12 +219,20 @@ public class DefaultRenderersFactory implements RenderersFactory { } /** - * Sets whether audio should be played using the offload path. Audio offload disables audio - * processors (for example speed adjustment). + * Sets whether audio should be played using the offload path. + * + *

Audio offload disables ExoPlayer audio processing, but significantly reduces the energy + * consumption of the playback when {@link + * ExoPlayer#experimental_enableOffloadScheduling(boolean)} is enabled. + * + *

Most Android devices can only support one offload {@link android.media.AudioTrack} at a time + * and can invalidate it at any time. Thus an app can never be guaranteed that it will be able to + * play in offload. * *

The default value is {@code false}. * - * @param enableOffload If audio offload should be used. + * @param enableOffload Whether to enable use of audio offload for supported formats, if + * available. * @return This factory, for convenience. */ public DefaultRenderersFactory setEnableAudioOffload(boolean enableOffload) { @@ -423,7 +431,8 @@ public class DefaultRenderersFactory implements RenderersFactory { * before output. May be empty. * @param eventHandler A handler to use when invoking event listeners and outputs. * @param eventListener An event listener. - * @param enableOffload If the renderer should use audio offload for all supported formats. + * @param enableOffload Whether to enable use of audio offload for supported formats, if + * available. * @param out An array to which the built renderers should be appended. */ protected void buildAudioRenderers( diff --git a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayer.java b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayer.java index b4cd9a399d..b3b369b68e 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayer.java @@ -20,6 +20,8 @@ import android.os.Looper; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import com.google.android.exoplayer2.analytics.AnalyticsCollector; +import com.google.android.exoplayer2.audio.AudioCapabilities; +import com.google.android.exoplayer2.audio.DefaultAudioSink; import com.google.android.exoplayer2.audio.MediaCodecAudioRenderer; import com.google.android.exoplayer2.metadata.MetadataRenderer; import com.google.android.exoplayer2.source.ClippingMediaSource; @@ -597,4 +599,39 @@ public interface ExoPlayer extends Player { * @see #setPauseAtEndOfMediaItems(boolean) */ boolean getPauseAtEndOfMediaItems(); + + /** + * Enables audio offload scheduling, which runs ExoPlayer's main loop as rarely as possible when + * playing an audio stream using audio offload. + * + *

Only use this scheduling mode if the player is not displaying anything to the user. For + * example when the application is in the background, or the screen is off. The player state + * (including position) is rarely updated (between 10s and 1min). + * + *

While offload scheduling is enabled, player events may be delivered severely delayed and + * apps should not interact with the player. When returning to the foreground, disable offload + * scheduling before interacting with the player + * + *

This mode should save significant power when the phone is playing offload audio with the + * screen off. + * + *

This mode only has an effect when playing an audio track in offload mode, which requires all + * the following: + * + *

+ * + *

This method is experimental, and will be renamed or removed in a future release. + * + * @param enableOffloadScheduling Whether to enable offload scheduling. + */ + void experimental_enableOffloadScheduling(boolean enableOffloadScheduling); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java index 26357a18dc..51c8a9ea60 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java @@ -202,6 +202,11 @@ import java.util.concurrent.TimeoutException; internalPlayer.experimental_throwWhenStuckBuffering(); } + @Override + public void experimental_enableOffloadScheduling(boolean enableOffloadScheduling) { + internalPlayer.experimental_enableOffloadScheduling(enableOffloadScheduling); + } + @Override @Nullable public AudioComponent getAudioComponent() { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java index 53c8a5d080..c5e6b06c19 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java @@ -94,6 +94,15 @@ import java.util.concurrent.atomic.AtomicBoolean; private static final int ACTIVE_INTERVAL_MS = 10; private static final int IDLE_INTERVAL_MS = 1000; + /** + * Duration under which pausing the main DO_SOME_WORK loop is not expected to yield significant + * power saving. + * + *

This value is probably too high, power measurements are needed adjust it, but as renderer + * sleep is currently only implemented for audio offload, which uses buffer much bigger than 2s, + * this does not matter for now. + */ + private static final long MIN_RENDERER_SLEEP_DURATION_MS = 2000; private final Renderer[] renderers; private final RendererCapabilities[] rendererCapabilities; @@ -127,6 +136,8 @@ import java.util.concurrent.atomic.AtomicBoolean; @Player.RepeatMode private int repeatMode; private boolean shuffleModeEnabled; private boolean foregroundMode; + private boolean requestForRendererSleep; + private boolean offloadSchedulingEnabled; private int enabledRendererCount; @Nullable private SeekPosition pendingInitialSeekPosition; @@ -199,6 +210,13 @@ import java.util.concurrent.atomic.AtomicBoolean; throwWhenStuckBuffering = true; } + public void experimental_enableOffloadScheduling(boolean enableOffloadScheduling) { + offloadSchedulingEnabled = enableOffloadScheduling; + if (!enableOffloadScheduling) { + handler.sendEmptyMessage(MSG_DO_SOME_WORK); + } + } + public void prepare() { handler.obtainMessage(MSG_PREPARE).sendToTarget(); } @@ -885,12 +903,13 @@ import java.util.concurrent.atomic.AtomicBoolean; if ((shouldPlayWhenReady() && playbackInfo.playbackState == Player.STATE_READY) || playbackInfo.playbackState == Player.STATE_BUFFERING) { - scheduleNextWork(operationStartTimeMs, ACTIVE_INTERVAL_MS); + maybeScheduleWakeup(operationStartTimeMs, ACTIVE_INTERVAL_MS); } else if (enabledRendererCount != 0 && playbackInfo.playbackState != Player.STATE_ENDED) { scheduleNextWork(operationStartTimeMs, IDLE_INTERVAL_MS); } else { handler.removeMessages(MSG_DO_SOME_WORK); } + requestForRendererSleep = false; // A sleep request is only valid for the current doSomeWork. TraceUtil.endSection(); } @@ -900,6 +919,14 @@ import java.util.concurrent.atomic.AtomicBoolean; handler.sendEmptyMessageAtTime(MSG_DO_SOME_WORK, thisOperationStartTimeMs + intervalMs); } + private void maybeScheduleWakeup(long operationStartTimeMs, long intervalMs) { + if (offloadSchedulingEnabled && requestForRendererSleep) { + return; + } + + scheduleNextWork(operationStartTimeMs, intervalMs); + } + private void seekToInternal(SeekPosition seekPosition) throws ExoPlaybackException { playbackInfoUpdate.incrementPendingOperationAcks(/* operationAcks= */ 1); @@ -2068,6 +2095,24 @@ import java.util.concurrent.atomic.AtomicBoolean; joining, mayRenderStartOfStream, periodHolder.getRendererOffset()); + + renderer.handleMessage( + Renderer.MSG_SET_WAKEUP_LISTENER, + new Renderer.WakeupListener() { + @Override + public void onSleep(long wakeupDeadlineMs) { + // Do not sleep if the expected sleep time is not long enough to save significant power. + if (wakeupDeadlineMs >= MIN_RENDERER_SLEEP_DURATION_MS) { + requestForRendererSleep = true; + } + } + + @Override + public void onWakeup() { + handler.sendEmptyMessage(MSG_DO_SOME_WORK); + } + }); + mediaClock.onRendererEnabled(renderer); // Start the renderer if playing. if (playing) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/Renderer.java b/library/core/src/main/java/com/google/android/exoplayer2/Renderer.java index fa73f9257d..8620c2d752 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/Renderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/Renderer.java @@ -46,6 +46,30 @@ import java.lang.annotation.RetentionPolicy; */ public interface Renderer extends PlayerMessage.Target { + /** + * Some renderers can signal when {@link #render(long, long)} should be called. + * + *

That allows the player to sleep until the next wakeup, instead of calling {@link + * #render(long, long)} in a tight loop. The aim of this interrupt based scheduling is to save + * power. + */ + interface WakeupListener { + + /** + * The renderer no longer needs to render until the next wakeup. + * + * @param wakeupDeadlineMs Maximum time in milliseconds until {@link #onWakeup()} will be + * called. + */ + void onSleep(long wakeupDeadlineMs); + + /** + * The renderer needs to render some frames. The client should call {@link #render(long, long)} + * at its earliest convenience. + */ + void onWakeup(); + } + /** * The type of a message that can be passed to a video renderer via {@link * ExoPlayer#createMessage(Target)}. The message payload should be the target {@link Surface}, or @@ -137,6 +161,14 @@ public interface Renderer extends PlayerMessage.Target { * representing the audio session ID that will be attached to the underlying audio track. */ int MSG_SET_AUDIO_SESSION_ID = 102; + /** + * A type of a message that can be passed to a {@link Renderer} via {@link + * ExoPlayer#createMessage(Target)}, to inform the renderer that it can schedule waking up another + * component. + * + *

The message payload must be a {@link WakeupListener} instance. + */ + int MSG_SET_WAKEUP_LISTENER = 103; /** * Applications or extensions may define custom {@code MSG_*} constants that can be passed to * renderers. These custom constants must be greater than or equal to this value. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java index d1f0cfc798..4c36f9fc99 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java @@ -633,6 +633,11 @@ public class SimpleExoPlayer extends BasePlayer C.TRACK_TYPE_AUDIO, Renderer.MSG_SET_SKIP_SILENCE_ENABLED, skipSilenceEnabled); } + @Override + public void experimental_enableOffloadScheduling(boolean enableOffloadScheduling) { + player.experimental_enableOffloadScheduling(enableOffloadScheduling); + } + @Override @Nullable public AudioComponent getAudioComponent() { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java index c4fa25d6bf..8bebd97a67 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java @@ -90,6 +90,17 @@ public interface AudioSink { * @param skipSilenceEnabled Whether skipping silences is enabled. */ void onSkipSilenceEnabledChanged(boolean skipSilenceEnabled); + + /** Called when the offload buffer has been partially emptied. */ + default void onOffloadBufferEmptying() {} + + /** + * Called when the offload buffer has been filled completely. + * + * @param bufferEmptyingDeadlineMs Maximum time in milliseconds until {@link + * #onOffloadBufferEmptying()} will be called. + */ + default void onOffloadBufferFull(long bufferEmptyingDeadlineMs) {} } /** diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioTrackPositionTracker.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioTrackPositionTracker.java index d15fe44fc0..ae2eb92044 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioTrackPositionTracker.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioTrackPositionTracker.java @@ -335,6 +335,11 @@ import java.lang.reflect.Method; return bufferSize - bytesPending; } + /** Returns the duration of audio that is buffered but unplayed. */ + public long getPendingBufferDurationMs(long writtenFrames) { + return C.usToMs(framesToDurationUs(writtenFrames - getPlaybackHeadPosition())); + } + /** Returns whether the track is in an invalid state and must be recreated. */ public boolean isStalled(long writtenFrames) { return forceResetWorkaroundTimeMs != C.TIME_UNSET diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java index bc3c321cac..fdd684a269 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java @@ -20,6 +20,7 @@ import android.media.AudioFormat; import android.media.AudioManager; import android.media.AudioTrack; import android.os.ConditionVariable; +import android.os.Handler; import android.os.SystemClock; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; @@ -274,6 +275,7 @@ public final class DefaultAudioSink implements AudioSink { private final AudioTrackPositionTracker audioTrackPositionTracker; private final ArrayDeque mediaPositionParametersCheckpoints; private final boolean enableOffload; + @MonotonicNonNull private StreamEventCallback offloadStreamEventCallback; @Nullable private Listener listener; /** Used to keep the audio session active on pre-V21 builds (see {@link #initialize(long)}). */ @@ -304,7 +306,7 @@ public final class DefaultAudioSink implements AudioSink { @Nullable private ByteBuffer inputBuffer; private int inputBufferAccessUnitCount; @Nullable private ByteBuffer outputBuffer; - private byte[] preV21OutputBuffer; + @MonotonicNonNull private byte[] preV21OutputBuffer; private int preV21OutputBufferOffset; private int drainingAudioProcessorIndex; private boolean handledEndOfStream; @@ -366,7 +368,10 @@ public final class DefaultAudioSink implements AudioSink { * be available when float output is in use. * @param enableOffload Whether audio offloading is enabled. If an audio format can be both played * with offload and encoded audio passthrough, it will be played in offload. Audio offload is - * supported starting with API 29 ({@link android.os.Build.VERSION_CODES#Q}). + * supported starting with API 29 ({@link android.os.Build.VERSION_CODES#Q}). Most Android + * devices can only support one offload {@link android.media.AudioTrack} at a time and can + * invalidate it at any time. Thus an app can never be guaranteed that it will be able to play + * in offload. */ public DefaultAudioSink( @Nullable AudioCapabilities audioCapabilities, @@ -404,6 +409,7 @@ public final class DefaultAudioSink implements AudioSink { activeAudioProcessors = new AudioProcessor[0]; outputBuffers = new ByteBuffer[0]; mediaPositionParametersCheckpoints = new ArrayDeque<>(); + offloadStreamEventCallback = Util.SDK_INT >= 29 ? new StreamEventCallback() : null; } // AudioSink implementation. @@ -563,6 +569,9 @@ public final class DefaultAudioSink implements AudioSink { audioTrack = Assertions.checkNotNull(configuration) .buildAudioTrack(tunneling, audioAttributes, audioSessionId); + if (isOffloadedPlayback(audioTrack)) { + registerStreamEventCallback(audioTrack); + } int audioSessionId = audioTrack.getAudioSessionId(); if (enablePreV21AudioSessionWorkaround) { if (Util.SDK_INT < 21) { @@ -744,6 +753,16 @@ public final class DefaultAudioSink implements AudioSink { return false; } + @RequiresApi(29) + private void registerStreamEventCallback(AudioTrack audioTrack) { + if (offloadStreamEventCallback == null) { + // Must be lazily initialized to receive stream event callbacks on the current (playback) + // thread as the constructor is not called in the playback thread. + offloadStreamEventCallback = new StreamEventCallback(); + } + offloadStreamEventCallback.register(audioTrack); + } + private void processBuffers(long avSyncPresentationTimeUs) throws WriteException { int count = activeAudioProcessors.length; int index = count; @@ -822,6 +841,15 @@ public final class DefaultAudioSink implements AudioSink { throw new WriteException(bytesWritten); } + if (playing + && listener != null + && bytesWritten < bytesRemaining + && isOffloadedPlayback(audioTrack)) { + long pendingDurationMs = + audioTrackPositionTracker.getPendingBufferDurationMs(writtenEncodedFrames); + listener.onOffloadBufferFull(pendingDurationMs); + } + if (configuration.isInputPcm) { writtenPcmBytes += bytesWritten; } @@ -1040,6 +1068,9 @@ public final class DefaultAudioSink implements AudioSink { if (audioTrackPositionTracker.isPlaying()) { audioTrack.pause(); } + if (isOffloadedPlayback(audioTrack)) { + Assertions.checkNotNull(offloadStreamEventCallback).unregister(audioTrack); + } // AudioTrack.release can take some time, so we call it on a background thread. final AudioTrack toRelease = audioTrack; audioTrack = null; @@ -1229,6 +1260,36 @@ public final class DefaultAudioSink implements AudioSink { audioFormat, audioAttributes.getAudioAttributesV21()); } + private static boolean isOffloadedPlayback(AudioTrack audioTrack) { + return Util.SDK_INT >= 29 && audioTrack.isOffloadedPlayback(); + } + + @RequiresApi(29) + private final class StreamEventCallback extends AudioTrack.StreamEventCallback { + private final Handler handler; + + public StreamEventCallback() { + handler = new Handler(); + } + + @Override + public void onDataRequest(AudioTrack track, int size) { + Assertions.checkState(track == DefaultAudioSink.this.audioTrack); + if (listener != null) { + listener.onOffloadBufferEmptying(); + } + } + + public void register(AudioTrack audioTrack) { + audioTrack.registerStreamEventCallback(handler::post, this); + } + + public void unregister(AudioTrack audioTrack) { + audioTrack.unregisterStreamEventCallback(this); + handler.removeCallbacksAndMessages(/* token= */ null); + } + } + private static AudioTrack initializeKeepSessionIdAudioTrack(int audioSessionId) { int sampleRate = 4000; // Equal to private AudioTrack.MIN_SAMPLE_RATE. int channelConfig = AudioFormat.CHANNEL_OUT_MONO; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index a4816c5372..a2a48d6f09 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -92,6 +92,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media private boolean allowFirstBufferPositionDiscontinuity; private boolean allowPositionDiscontinuity; + @Nullable private WakeupListener wakeupListener; + /** * @param context A context. * @param mediaCodecSelector A decoder selector. @@ -696,6 +698,9 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media case MSG_SET_AUDIO_SESSION_ID: audioSink.setAudioSessionId((Integer) message); break; + case MSG_SET_WAKEUP_LISTENER: + this.wakeupListener = (WakeupListener) message; + break; default: super.handleMessage(messageType, message); break; @@ -875,5 +880,19 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media eventDispatcher.skipSilenceEnabledChanged(skipSilenceEnabled); onAudioTrackSkipSilenceEnabledChanged(skipSilenceEnabled); } + + @Override + public void onOffloadBufferEmptying() { + if (wakeupListener != null) { + wakeupListener.onWakeup(); + } + } + + @Override + public void onOffloadBufferFull(long bufferEmptyingDeadlineMs) { + if (wakeupListener != null) { + wakeupListener.onSleep(bufferEmptyingDeadlineMs); + } + } } } diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/StubExoPlayer.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/StubExoPlayer.java index b4678cb7cf..c79a128f81 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/StubExoPlayer.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/StubExoPlayer.java @@ -465,4 +465,9 @@ public abstract class StubExoPlayer extends BasePlayer implements ExoPlayer { public boolean getPauseAtEndOfMediaItems() { throw new UnsupportedOperationException(); } + + @Override + public void experimental_enableOffloadScheduling(boolean enableOffloadScheduling) { + throw new UnsupportedOperationException(); + } }