Schedule doSomeWork when MediaCodec signals available buffers

When running in asynchronous mode, MediaCodec will be running the CPU to signal input and output buffers being made available for use by the player. With ExoPlayer.experimentalSetDynamicSchedulingEnabled set to true, ExoPlayer will wakeup to make rendering progress when MediaCodec raises these signals. In this way, ExoPlayer work will align more closely with CPU wake-cycles.

PiperOrigin-RevId: 638962108

RELEASENOTES.md

@@ -50,6 +50,12 @@
         order for the renderer to progress. If `ExoPlayer` is set with
         `experimentalSetDynamicSchedulingEnabled` then `ExoPlayer` will call
         this method when calculating the time to schedule its work task.
+    *   Add `MediaCodecAdapter#OnBufferAvailableListener` to alert when input
+        and output buffers are available for use by `MediaCodecRenderer`.
+        `MediaCodecRenderer` will signal `ExoPlayer` when receiving these
+        callbacks and if `ExoPlayer` is set with
+        `experimentalSetDynamicSchedulingEnabled`, then `ExoPlayer` will
+        schedule its work loop as renderers can make progress.
 *   Transformer:
     *   Work around a decoder bug where the number of audio channels was capped
         at stereo when handling PCM input.

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java

@@ -119,8 +119,6 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   private long currentPositionUs;
   private boolean allowPositionDiscontinuity;
   private boolean audioSinkNeedsReset;
-
-  @Nullable private WakeupListener wakeupListener;
   private boolean hasPendingReportedSkippedSilence;
   private int rendererPriority;
   private boolean isStarted;
@@ -480,7 +478,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   }
 
   @Override
-  public long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) {
+  public long getDurationToProgressUs(
+      boolean isOnBufferAvailableListenerRegistered, long positionUs, long elapsedRealtimeUs) {
     if (nextBufferToWritePresentationTimeUs != C.TIME_UNSET) {
       long durationUs =
           (long)
@@ -493,7 +492,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
       }
       return max(DEFAULT_DURATION_TO_PROGRESS_US, durationUs);
     }
-    return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs);
+    return super.getDurationToProgressUs(
+        isOnBufferAvailableListenerRegistered, positionUs, elapsedRealtimeUs);
   }
 
   @Override
@@ -854,9 +854,6 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
       case MSG_SET_AUDIO_SESSION_ID:
         audioSink.setAudioSessionId((Integer) checkNotNull(message));
         break;
-      case MSG_SET_WAKEUP_LISTENER:
-        this.wakeupListener = (WakeupListener) message;
-        break;
       case MSG_SET_PRIORITY:
         rendererPriority = (int) checkNotNull(message);
         updateCodecImportance();
@@ -1073,6 +1070,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
 
     @Override
     public void onOffloadBufferEmptying() {
+      WakeupListener wakeupListener = getWakeupListener();
       if (wakeupListener != null) {
         wakeupListener.onWakeup();
       }
@@ -1080,6 +1078,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
 
     @Override
     public void onOffloadBufferFull() {
+      WakeupListener wakeupListener = getWakeupListener();
       if (wakeupListener != null) {
         wakeupListener.onSleep();
       }

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java

@@ -274,6 +274,12 @@ import java.nio.ByteBuffer;
         handler);
   }
 
+  @Override
+  public boolean registerOnBufferAvailableListener(OnBufferAvailableListener listener) {
+    asynchronousMediaCodecCallback.setOnBufferAvailableListener(listener);
+    return true;
+  }
+
   @Override
   public void setOutputSurface(Surface surface) {
     codec.setOutputSurface(surface);

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallback.java

@@ -77,6 +77,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   @Nullable
   private IllegalStateException internalException;
 
+  @GuardedBy("lock")
+  @Nullable
+  private MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener;
+
   /**
    * Creates a new instance.
    *
@@ -210,6 +214,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   public void onInputBufferAvailable(MediaCodec codec, int index) {
     synchronized (lock) {
       availableInputBuffers.addLast(index);
+      if (onBufferAvailableListener != null) {
+        onBufferAvailableListener.onInputBufferAvailable();
+      }
     }
   }
 
@@ -222,6 +229,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       }
       availableOutputBuffers.addLast(index);
       bufferInfos.add(info);
+      if (onBufferAvailableListener != null) {
+        onBufferAvailableListener.onOutputBufferAvailable();
+      }
     }
   }
 
@@ -247,6 +257,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
   }
 
+  /**
+   * Sets the {@link MediaCodecAdapter.OnBufferAvailableListener} that will be notified when {@link
+   * #onInputBufferAvailable} and {@link #onOutputBufferAvailable} are called.
+   *
+   * @param onBufferAvailableListener The listener that will be notified when {@link
+   *     #onInputBufferAvailable} and {@link #onOutputBufferAvailable} are called.
+   */
+  public void setOnBufferAvailableListener(
+      MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener) {
+    synchronized (lock) {
+      this.onBufferAvailableListener = onBufferAvailableListener;
+    }
+  }
+
   private void onFlushCompleted() {
     synchronized (lock) {
       if (shutDown) {

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java

@@ -151,6 +151,23 @@ public interface MediaCodecAdapter {
     void onFrameRendered(MediaCodecAdapter codec, long presentationTimeUs, long nanoTime);
   }
 
+  /** Listener to be called when an input or output buffer becomes available. */
+  interface OnBufferAvailableListener {
+    /**
+     * Called when an input buffer becomes available.
+     *
+     * @see MediaCodec.Callback#onInputBufferAvailable(MediaCodec, int)
+     */
+    default void onInputBufferAvailable() {}
+
+    /**
+     * Called when an output buffer becomes available.
+     *
+     * @see MediaCodec.Callback#onOutputBufferAvailable(MediaCodec, int, MediaCodec.BufferInfo)
+     */
+    default void onOutputBufferAvailable() {}
+  }
+
   /**
    * Returns the next available input buffer index from the underlying {@link MediaCodec} or {@link
    * MediaCodec#INFO_TRY_AGAIN_LATER} if no such buffer exists.
@@ -252,6 +269,21 @@ public interface MediaCodecAdapter {
   @RequiresApi(23)
   void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler);
 
+  /**
+   * Registers a listener that will be called when an input or output buffer becomes available.
+   *
+   * <p>Returns false if listener was not successfully registered for callbacks.
+   *
+   * @see MediaCodec.Callback#onInputBufferAvailable
+   * @see MediaCodec.Callback#onOutputBufferAvailable
+   * @return Whether listener was successfully registered.
+   */
+  @RequiresApi(21)
+  default boolean registerOnBufferAvailableListener(
+      MediaCodecAdapter.OnBufferAvailableListener listener) {
+    return false;
+  }
+
   /**
    * Dynamically sets the output surface of a {@link MediaCodec}.
    *

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java

@@ -346,6 +346,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   @Nullable private Format outputFormat;
   @Nullable private DrmSession codecDrmSession;
   @Nullable private DrmSession sourceDrmSession;
+  @Nullable private WakeupListener wakeupListener;
 
   /**
    * A framework {@link MediaCrypto} for use with {@link MediaCodec#queueSecureInputBuffer(int, int,
@@ -382,6 +383,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   private boolean codecNeedsAdaptationWorkaroundBuffer;
   private boolean shouldSkipAdaptationWorkaroundOutputBuffer;
   private boolean codecNeedsEosPropagation;
+  private boolean codecRegisteredOnBufferAvailableListener;
   private long codecHotswapDeadlineMs;
   private int inputIndex;
   private int outputIndex;
@@ -503,6 +505,37 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   protected abstract @Capabilities int supportsFormat(
       MediaCodecSelector mediaCodecSelector, Format format) throws DecoderQueryException;
 
+  @Override
+  public final long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) {
+    return getDurationToProgressUs(
+        /* isOnBufferAvailableListenerRegistered= */ codecRegisteredOnBufferAvailableListener,
+        positionUs,
+        elapsedRealtimeUs);
+  }
+
+  /**
+   * Returns minimum time playback must advance in order for the {@link #render} call to make
+   * progress.
+   *
+   * <p>If the {@code Renderer} has a registered {@link
+   * MediaCodecAdapter.OnBufferAvailableListener}, then the {@code Renderer} will be notified when
+   * decoder input and output buffers become available. These callbacks may affect the calculated
+   * minimum time playback must advance before a {@link #render} call can make progress.
+   *
+   * @param isOnBufferAvailableListenerRegistered Whether the {@code Renderer} is using a {@link
+   *     MediaCodecAdapter} with successfully registered {@link
+   *     MediaCodecAdapter.OnBufferAvailableListener OnBufferAvailableListener}.
+   * @param positionUs The current media time in microseconds, measured at the start of the current
+   *     iteration of the rendering loop.
+   * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
+   *     measured at the start of the current iteration of the rendering loop.
+   * @return minimum time playback must advance before renderer is able to make progress.
+   */
+  protected long getDurationToProgressUs(
+      boolean isOnBufferAvailableListenerRegistered, long positionUs, long elapsedRealtimeUs) {
+    return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs);
+  }
+
   /**
    * Returns a list of decoders that can decode media in the specified format, in priority order.
    *
@@ -797,6 +830,16 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     // Do nothing. Overridden to remove throws clause.
   }
 
+  @Override
+  public void handleMessage(@MessageType int messageType, @Nullable Object message)
+      throws ExoPlaybackException {
+    if (messageType == MSG_SET_WAKEUP_LISTENER) {
+      this.wakeupListener = (WakeupListener) message;
+    } else {
+      super.handleMessage(messageType, message);
+    }
+  }
+
   @Override
   public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
     if (pendingOutputEndOfStream) {
@@ -971,6 +1014,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     codecNeedsEosBufferTimestampWorkaround = false;
     codecNeedsMonoChannelCountWorkaround = false;
     codecNeedsEosPropagation = false;
+    codecRegisteredOnBufferAvailableListener = false;
     codecReconfigured = false;
     codecReconfigurationState = RECONFIGURATION_STATE_NONE;
   }
@@ -1193,6 +1237,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     try {
       TraceUtil.beginSection("createCodec:" + codecName);
       codec = codecAdapterFactory.createAdapter(configuration);
+      codecRegisteredOnBufferAvailableListener =
+          Util.SDK_INT >= 21
+              && Api21.registerOnBufferAvailableListener(
+                  codec, new MediaCodecRendererCodecAdapterListener());
     } finally {
       TraceUtil.endSection();
     }
@@ -1813,6 +1861,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     return CODEC_OPERATING_RATE_UNSET;
   }
 
+  /** Returns listener used to signal that {@link #render(long, long)} should be called. */
+  @Nullable
+  protected final WakeupListener getWakeupListener() {
+    return wakeupListener;
+  }
+
   /**
    * Updates the codec operating rate, or triggers codec release and re-initialization if a
    * previously set operating rate needs to be cleared.
@@ -2691,6 +2745,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     }
   }
 
+  @RequiresApi(21)
+  private static final class Api21 {
+    @DoNotInline
+    public static boolean registerOnBufferAvailableListener(
+        MediaCodecAdapter codec, MediaCodecRendererCodecAdapterListener listener) {
+      return codec.registerOnBufferAvailableListener(listener);
+    }
+  }
+
   @RequiresApi(31)
   private static final class Api31 {
     private Api31() {}
@@ -2704,4 +2767,21 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
       }
     }
   }
+
+  private final class MediaCodecRendererCodecAdapterListener
+      implements MediaCodecAdapter.OnBufferAvailableListener {
+    @Override
+    public void onInputBufferAvailable() {
+      if (wakeupListener != null) {
+        wakeupListener.onWakeup();
+      }
+    }
+
+    @Override
+    public void onOutputBufferAvailable() {
+      if (wakeupListener != null) {
+        wakeupListener.onWakeup();
+      }
+    }
+  }
 }

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallbackTest.java

@@ -530,6 +530,42 @@ public class AsynchronousMediaCodecCallbackTest {
     asynchronousMediaCodecCallback.shutdown();
   }
 
+  @Test
+  public void onInputBufferAvailable_withOnBufferAvailableListener_callsOnInputBufferAvailable() {
+    AtomicInteger onInputBufferAvailableCounter = new AtomicInteger();
+    MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener =
+        new MediaCodecAdapter.OnBufferAvailableListener() {
+          @Override
+          public void onInputBufferAvailable() {
+            onInputBufferAvailableCounter.getAndIncrement();
+          }
+        };
+    asynchronousMediaCodecCallback.setOnBufferAvailableListener(onBufferAvailableListener);
+
+    // Send an input buffer to the callback.
+    asynchronousMediaCodecCallback.onInputBufferAvailable(codec, 0);
+
+    assertThat(onInputBufferAvailableCounter.get()).isEqualTo(1);
+  }
+
+  @Test
+  public void onOutputBufferAvailable_withOnBufferAvailableListener_callsOnOutputBufferAvailable() {
+    AtomicInteger onOutputBufferAvailableCounter = new AtomicInteger();
+    MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener =
+        new MediaCodecAdapter.OnBufferAvailableListener() {
+          @Override
+          public void onOutputBufferAvailable() {
+            onOutputBufferAvailableCounter.getAndIncrement();
+          }
+        };
+    asynchronousMediaCodecCallback.setOnBufferAvailableListener(onBufferAvailableListener);
+
+    // Send an output buffer to the callback.
+    asynchronousMediaCodecCallback.onOutputBufferAvailable(codec, 0, new MediaCodec.BufferInfo());
+
+    assertThat(onOutputBufferAvailableCounter.get()).isEqualTo(1);
+  }
+
   /** Reflectively create a {@link MediaCodec.CodecException}. */
   private static MediaCodec.CodecException createCodecException()
       throws NoSuchMethodException,