From d0676245b527e51c056f3471749209013b1ce42d Mon Sep 17 00:00:00 2001 From: tonihei Date: Tue, 27 Aug 2024 07:05:45 -0700 Subject: [PATCH] Add AnalyticsListener.onRendererReadyChanged This callback allows listeners to track when individual renderers allow or prevent playback from being ready. For example, this is useful to figure out which renderer blocked the playback the longest. PiperOrigin-RevId: 667970933 --- RELEASENOTES.md | 4 ++ .../exoplayer/ExoPlayerImplInternal.java | 45 ++++++++++---- .../analytics/AnalyticsCollector.java | 15 ++++- .../analytics/AnalyticsListener.java | 22 ++++++- .../analytics/DefaultAnalyticsCollector.java | 28 +++++---- .../media3/exoplayer/util/EventLogger.java | 19 ++++++ .../DefaultAnalyticsCollectorTest.java | 58 +++++++++++++++++++ 7 files changed, 165 insertions(+), 26 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index d296cf877e..582527a796 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -37,6 +37,10 @@ handling is enabled. This ensures the blocking call isn't done if audio focus handling is not enabled ([#1616](https://github.com/androidx/media/pull/1616)). + * Allow playback regardless of buffered duration when loading fails + ([#1571](https://github.com/androidx/media/issues/1571)). + * Add `AnalyticsListener.onRendererReadyChanged()` to signal when + individual renderers allow playback to be ready. * Transformer: * Add `SurfaceAssetLoader`, which supports queueing video data to Transformer via a `Surface`. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java index c3ee83c5ef..7bab7d178a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java @@ -185,6 +185,7 @@ import java.util.concurrent.atomic.AtomicBoolean; private final Renderer[] renderers; private final Set renderersToReset; private final RendererCapabilities[] rendererCapabilities; + private final boolean[] rendererReportedReady; private final TrackSelector trackSelector; private final TrackSelectorResult emptyTrackSelectorResult; private final LoadControl loadControl; @@ -206,6 +207,8 @@ import java.util.concurrent.atomic.AtomicBoolean; private final long releaseTimeoutMs; private final PlayerId playerId; private final boolean dynamicSchedulingEnabled; + private final AnalyticsCollector analyticsCollector; + private final HandlerWrapper applicationLooperHandler; @SuppressWarnings("unused") private SeekParameters seekParameters; @@ -253,7 +256,7 @@ import java.util.concurrent.atomic.AtomicBoolean; Clock clock, PlaybackInfoUpdateListener playbackInfoUpdateListener, PlayerId playerId, - Looper playbackLooper, + @Nullable Looper playbackLooper, PreloadConfiguration preloadConfiguration) { this.playbackInfoUpdateListener = playbackInfoUpdateListener; this.renderers = renderers; @@ -272,6 +275,7 @@ import java.util.concurrent.atomic.AtomicBoolean; this.clock = clock; this.playerId = playerId; this.preloadConfiguration = preloadConfiguration; + this.analyticsCollector = analyticsCollector; playbackMaybeBecameStuckAtMs = C.TIME_UNSET; lastRebufferRealtimeMs = C.TIME_UNSET; @@ -282,6 +286,7 @@ import java.util.concurrent.atomic.AtomicBoolean; playbackInfo = PlaybackInfo.createDummy(emptyTrackSelectorResult); playbackInfoUpdate = new PlaybackInfoUpdate(playbackInfo); rendererCapabilities = new RendererCapabilities[renderers.length]; + rendererReportedReady = new boolean[renderers.length]; @Nullable RendererCapabilities.Listener rendererCapabilitiesListener = trackSelector.getRendererCapabilitiesListener(); @@ -301,12 +306,16 @@ import java.util.concurrent.atomic.AtomicBoolean; deliverPendingMessageAtStartPositionRequired = true; - HandlerWrapper eventHandler = clock.createHandler(applicationLooper, /* callback= */ null); + applicationLooperHandler = clock.createHandler(applicationLooper, /* callback= */ null); queue = new MediaPeriodQueue( - analyticsCollector, eventHandler, this::createMediaPeriodHolder, preloadConfiguration); + analyticsCollector, + applicationLooperHandler, + this::createMediaPeriodHolder, + preloadConfiguration); mediaSourceList = - new MediaSourceList(/* listener= */ this, analyticsCollector, eventHandler, playerId); + new MediaSourceList( + /* listener= */ this, analyticsCollector, applicationLooperHandler, playerId); if (playbackLooper != null) { internalPlaybackThread = null; @@ -1128,6 +1137,7 @@ import java.util.concurrent.atomic.AtomicBoolean; for (int i = 0; i < renderers.length; i++) { Renderer renderer = renderers[i]; if (!isRendererEnabled(renderer)) { + maybeTriggerOnRendererReadyChanged(/* rendererIndex= */ i, /* allowsPlayback= */ false); continue; } // TODO: Each renderer should return the maximum delay before which it wishes to be called @@ -1144,6 +1154,7 @@ import java.util.concurrent.atomic.AtomicBoolean; boolean isWaitingForNextStream = !isReadingAhead && renderer.hasReadStreamToEnd(); boolean allowsPlayback = isReadingAhead || isWaitingForNextStream || renderer.isReady() || renderer.isEnded(); + maybeTriggerOnRendererReadyChanged(/* rendererIndex= */ i, allowsPlayback); renderersAllowPlayback = renderersAllowPlayback && allowsPlayback; if (!allowsPlayback) { renderer.maybeThrowStreamError(); @@ -1240,6 +1251,16 @@ import java.util.concurrent.atomic.AtomicBoolean; TraceUtil.endSection(); } + private void maybeTriggerOnRendererReadyChanged(int rendererIndex, boolean allowsPlayback) { + if (rendererReportedReady[rendererIndex] != allowsPlayback) { + rendererReportedReady[rendererIndex] = allowsPlayback; + applicationLooperHandler.post( + () -> + analyticsCollector.onRendererReadyChanged( + rendererIndex, renderers[rendererIndex].getTrackType(), allowsPlayback)); + } + } + private long getCurrentLiveOffsetUs() { return getLiveOffsetUs( playbackInfo.timeline, playbackInfo.periodId.periodUid, playbackInfo.positionUs); @@ -1435,8 +1456,8 @@ import java.util.concurrent.atomic.AtomicBoolean; || oldPlayingPeriodHolder != newPlayingPeriodHolder || (newPlayingPeriodHolder != null && newPlayingPeriodHolder.toRendererTime(periodPositionUs) < 0)) { - for (Renderer renderer : renderers) { - disableRenderer(renderer); + for (int i = 0; i < renderers.length; i++) { + disableRenderer(/* rendererIndex= */ i); } if (newPlayingPeriodHolder != null) { // Update the queue and reenable renderers if the requested media period already exists. @@ -1561,9 +1582,9 @@ import java.util.concurrent.atomic.AtomicBoolean; updateRebufferingState(/* isRebuffering= */ false, /* resetLastRebufferRealtimeMs= */ true); mediaClock.stop(); rendererPositionUs = MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US; - for (Renderer renderer : renderers) { + for (int i = 0; i < renderers.length; i++) { try { - disableRenderer(renderer); + disableRenderer(/* rendererIndex= */ i); } catch (ExoPlaybackException | RuntimeException e) { // There's nothing we can do. Log.e(TAG, "Disable failed.", e); @@ -1837,10 +1858,12 @@ import java.util.concurrent.atomic.AtomicBoolean; } } - private void disableRenderer(Renderer renderer) throws ExoPlaybackException { + private void disableRenderer(int rendererIndex) throws ExoPlaybackException { + Renderer renderer = renderers[rendererIndex]; if (!isRendererEnabled(renderer)) { return; } + maybeTriggerOnRendererReadyChanged(rendererIndex, /* allowsPlayback= */ false); mediaClock.onRendererDisabled(renderer); ensureStopped(renderer); renderer.disable(); @@ -1916,7 +1939,7 @@ import java.util.concurrent.atomic.AtomicBoolean; if (rendererWasEnabledFlags[i]) { if (sampleStream != renderer.getStream()) { // We need to disable the renderer. - disableRenderer(renderer); + disableRenderer(/* rendererIndex= */ i); } else if (streamResetFlags[i]) { // The renderer will continue to consume from its current stream, but needs to be reset. renderer.resetPosition(rendererPositionUs); @@ -2361,7 +2384,7 @@ import java.util.concurrent.atomic.AtomicBoolean; } } else if (renderer.isEnded()) { // The renderer has finished playback, so we can disable it now. - disableRenderer(renderer); + disableRenderer(/* rendererIndex= */ i); } else { // We need to wait until rendering finished before disabling the renderer. needsToWaitForRendererToEnd = true; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java index b1939179b8..3b36813f35 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java @@ -89,11 +89,22 @@ public interface AnalyticsCollector void updateMediaPeriodQueueInfo(List queue, @Nullable MediaPeriodId readingPeriod); /** - * Notify analytics collector that a seek operation will start. Should be called before the player - * adjusts its state and position to the seek. + * Notifies the analytics collector that a seek operation will start. Should be called before the + * player adjusts its state and position to the seek. */ void notifySeekStarted(); + /** + * Called each time a renderer starts or stops allowing playback to be ready. + * + * @param rendererIndex The index of the renderer in the {@link + * androidx.media3.exoplayer.ExoPlayer} instance. + * @param rendererTrackType The {@link C.TrackType} of the renderer. + * @param isRendererReady Whether the renderer allows playback to be ready. + */ + void onRendererReadyChanged( + int rendererIndex, @C.TrackType int rendererTrackType, boolean isRendererReady); + // Audio events. /** diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java index b352fea965..4418a683a6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java @@ -234,7 +234,8 @@ public interface AnalyticsListener { EVENT_AUDIO_CODEC_ERROR, EVENT_VIDEO_CODEC_ERROR, EVENT_AUDIO_TRACK_INITIALIZED, - EVENT_AUDIO_TRACK_RELEASED + EVENT_AUDIO_TRACK_RELEASED, + EVENT_RENDERER_READY_CHANGED }) @interface EventFlags {} @@ -444,6 +445,9 @@ public interface AnalyticsListener { /** An audio track has been released. */ @UnstableApi int EVENT_AUDIO_TRACK_RELEASED = 1032; + /** A renderer changed its readiness for playback. */ + @UnstableApi int EVENT_RENDERER_READY_CHANGED = 1033; + /** Time information of an event. */ @UnstableApi final class EventTime { @@ -1390,6 +1394,22 @@ public interface AnalyticsListener { @UnstableApi default void onDrmSessionReleased(EventTime eventTime) {} + /** + * Called each time a renderer starts or stops allowing playback to be ready. + * + * @param eventTime The event time. + * @param rendererIndex The index of the renderer in the {@link + * androidx.media3.exoplayer.ExoPlayer} instance. + * @param rendererTrackType The {@link C.TrackType} of the renderer. + * @param isRendererReady Whether the renderer allows playback to be ready. + */ + @UnstableApi + default void onRendererReadyChanged( + EventTime eventTime, + int rendererIndex, + @C.TrackType int rendererTrackType, + boolean isRendererReady) {} + /** * Called when the {@link Player} is released. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java index e91df8cf73..e9c2b7b9a5 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java @@ -164,6 +164,18 @@ public class DefaultAnalyticsCollector implements AnalyticsCollector { } } + @Override + public void onRendererReadyChanged( + int rendererIndex, @C.TrackType int rendererTrackType, boolean isRendererReady) { + EventTime eventTime = generateReadingMediaPeriodEventTime(); + sendEvent( + eventTime, + AnalyticsListener.EVENT_RENDERER_READY_CHANGED, + listener -> + listener.onRendererReadyChanged( + eventTime, rendererIndex, rendererTrackType, isRendererReady)); + } + // Audio events. @Override @@ -172,9 +184,7 @@ public class DefaultAnalyticsCollector implements AnalyticsCollector { sendEvent( eventTime, AnalyticsListener.EVENT_AUDIO_ENABLED, - listener -> { - listener.onAudioEnabled(eventTime, counters); - }); + listener -> listener.onAudioEnabled(eventTime, counters)); } @SuppressWarnings("deprecation") // Calling deprecated listener method. @@ -237,9 +247,7 @@ public class DefaultAnalyticsCollector implements AnalyticsCollector { sendEvent( eventTime, AnalyticsListener.EVENT_AUDIO_DISABLED, - listener -> { - listener.onAudioDisabled(eventTime, counters); - }); + listener -> listener.onAudioDisabled(eventTime, counters)); } @Override @@ -295,9 +303,7 @@ public class DefaultAnalyticsCollector implements AnalyticsCollector { sendEvent( eventTime, AnalyticsListener.EVENT_VIDEO_ENABLED, - listener -> { - listener.onVideoEnabled(eventTime, counters); - }); + listener -> listener.onVideoEnabled(eventTime, counters)); } @Override @@ -349,9 +355,7 @@ public class DefaultAnalyticsCollector implements AnalyticsCollector { sendEvent( eventTime, AnalyticsListener.EVENT_VIDEO_DISABLED, - listener -> { - listener.onVideoDisabled(eventTime, counters); - }); + listener -> listener.onVideoDisabled(eventTime, counters)); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java index 61dede687c..bc9ffcdf05 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java @@ -16,6 +16,7 @@ package androidx.media3.exoplayer.util; import static androidx.media3.common.util.Util.getFormatSupportString; +import static androidx.media3.common.util.Util.getTrackTypeString; import static java.lang.Math.min; import android.os.SystemClock; @@ -567,6 +568,24 @@ public class EventLogger implements AnalyticsListener { logd(eventTime, "drmSessionReleased"); } + @UnstableApi + @Override + public void onRendererReadyChanged( + EventTime eventTime, + int rendererIndex, + @C.TrackType int rendererTrackType, + boolean isRendererReady) { + logd( + eventTime, + "rendererReady", + "rendererIndex=" + + rendererIndex + + ", " + + getTrackTypeString(rendererTrackType) + + ", " + + isRendererReady); + } + /** * Logs a debug message. * diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollectorTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollectorTest.java index 8a21917d32..d5a65b1c0f 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollectorTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollectorTest.java @@ -39,6 +39,7 @@ import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_PLAYER import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_PLAY_WHEN_READY_CHANGED; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_POSITION_DISCONTINUITY; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_RENDERED_FIRST_FRAME; +import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_RENDERER_READY_CHANGED; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_TIMELINE_CHANGED; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_TRACKS_CHANGED; import static androidx.media3.exoplayer.analytics.AnalyticsListener.EVENT_VIDEO_DECODER_INITIALIZED; @@ -280,6 +281,9 @@ public final class DefaultAnalyticsCollectorTest { assertThat(listener.getEvents(EVENT_VIDEO_SIZE_CHANGED)).containsExactly(period0); assertThat(listener.getEvents(EVENT_RENDERED_FIRST_FRAME)).containsExactly(period0); assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)).containsExactly(period0); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly(period0 /* audioTrue */, period0 /* videoTrue */) + .inOrder(); listener.assertNoMoreEvents(); } @@ -361,6 +365,9 @@ public final class DefaultAnalyticsCollectorTest { .containsExactly(period0, period1) .inOrder(); assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)).containsExactly(period1); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly(period0 /* audioTrue */, period0 /* videoTrue */) + .inOrder(); listener.assertNoMoreEvents(); } @@ -430,6 +437,9 @@ public final class DefaultAnalyticsCollectorTest { .inOrder(); assertThat(listener.getEvents(EVENT_RENDERED_FIRST_FRAME)).containsExactly(period0); assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)).containsExactly(period0); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly(period0 /* videoTrue */, period1 /* videoFalse */, period1 /* audioTrue */) + .inOrder(); listener.assertNoMoreEvents(); } @@ -514,6 +524,14 @@ public final class DefaultAnalyticsCollectorTest { period1) // width=0, height=0 for audio only media source .inOrder(); assertThat(listener.getEvents(EVENT_RENDERED_FIRST_FRAME)).containsExactly(period0); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly( + period0 /* videoTrue */, + period0 /* audioTrue */, + period1 /* videoFalse */, + period1 /* audioFalse */, + period1 /* audioTrue */) + .inOrder(); listener.assertNoMoreEvents(); } @@ -619,6 +637,15 @@ public final class DefaultAnalyticsCollectorTest { assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)) .containsExactly(period0, period1Seq2) .inOrder(); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly( + period0 /* videoTrue */, + period1Seq1 /* audioTrue */, + period1Seq1 /* audioFalse */, + period1Seq1 /* videoFalse */, + period0 /* videoTrue */, + period1Seq2 /* audioTrue */) + .inOrder(); listener.assertNoMoreEvents(); } @@ -714,6 +741,9 @@ public final class DefaultAnalyticsCollectorTest { .inOrder(); assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)) .containsExactly(period0Seq1); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly( + period0Seq0 /* videoTrue */, period0Seq0 /* videoFalse */, period0Seq1 /* videoTrue */); listener.assertNoMoreEvents(); } @@ -795,6 +825,10 @@ public final class DefaultAnalyticsCollectorTest { .containsExactly(period0Seq0, period0Seq0); assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)) .containsExactly(period0Seq0); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly( + period0Seq0 /* videoTrue */, period0Seq0 /* videoFalse */, period0Seq0 /* videoTrue */) + .inOrder(); listener.assertNoMoreEvents(); } @@ -872,6 +906,11 @@ public final class DefaultAnalyticsCollectorTest { assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)) .containsExactly(period1Seq0, period1Seq0) .inOrder(); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly( + window0Period1Seq0 /* videoTrue */, + period1Seq0 /* videoFalse */, + period1Seq0 /* videoTrue */); listener.assertNoMoreEvents(); } @@ -961,6 +1000,9 @@ public final class DefaultAnalyticsCollectorTest { .containsExactly(period0Seq0, period1Seq1, period0Seq1); assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)) .containsExactly(period0Seq1); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly( + period0Seq0 /* videoTrue */, period0Seq1 /* videoFalse */, period0Seq1 /* videoTrue */); listener.assertNoMoreEvents(); } @@ -1190,6 +1232,8 @@ public final class DefaultAnalyticsCollectorTest { .inOrder(); assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)) .containsExactly(contentAfterPostroll); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly(prerollAd /* videoTrue */); listener.assertNoMoreEvents(); } @@ -1326,6 +1370,11 @@ public final class DefaultAnalyticsCollectorTest { .inOrder(); assertThat(listener.getEvents(EVENT_VIDEO_FRAME_PROCESSING_OFFSET)) .containsExactly(contentAfterMidroll); + assertThat(listener.getEvents(EVENT_RENDERER_READY_CHANGED)) + .containsExactly( + contentBeforeMidroll /* videoTrue */, + midrollAd /* videoFalse */, + midrollAd /* videoTrue */); listener.assertNoMoreEvents(); } @@ -2319,6 +2368,15 @@ public final class DefaultAnalyticsCollectorTest { reportedEvents.add(new ReportedEvent(EVENT_DRM_SESSION_RELEASED, eventTime)); } + @Override + public void onRendererReadyChanged( + EventTime eventTime, + int rendererIndex, + @C.TrackType int rendererTrackType, + boolean isRendererReady) { + reportedEvents.add(new ReportedEvent(EVENT_RENDERER_READY_CHANGED, eventTime)); + } + private static final class ReportedEvent { public final long eventType;