Calculate outputTime based on inputTime and audio speed change

PiperOrigin-RevId: 613894863

libraries/common/src/main/java/androidx/media3/common/audio/SonicAudioProcessor.java

@@ -141,6 +141,36 @@ public class SonicAudioProcessor implements AudioProcessor {
     }
   }
 
+  /**
+   * Returns the playout duration corresponding to the specified media duration, taking speed
+   * adjustment into account.
+   *
+   * <p>The scaling performed by this method will use the actual playback speed achieved by the
+   * audio processor, on average, since it was last flushed. This may differ very slightly from the
+   * target playback speed.
+   *
+   * @param mediaDuration The media duration to scale.
+   * @return The corresponding playout duration, in the same units as {@code mediaDuration}.
+   */
+  public final long getPlayoutDuration(long mediaDuration) {
+    if (outputBytes >= MIN_BYTES_FOR_DURATION_SCALING_CALCULATION) {
+      long processedInputBytes = inputBytes - checkNotNull(sonic).getPendingInputBytes();
+      return outputAudioFormat.sampleRate == inputAudioFormat.sampleRate
+          ? Util.scaleLargeTimestamp(mediaDuration, outputBytes, processedInputBytes)
+          : Util.scaleLargeTimestamp(
+              mediaDuration,
+              outputBytes * inputAudioFormat.sampleRate,
+              processedInputBytes * outputAudioFormat.sampleRate);
+    } else {
+      return (long) (mediaDuration / (double) speed);
+    }
+  }
+
+  /** Returns the number of bytes processed since last flush or reset. */
+  public final long getProcessedInputBytes() {
+    return inputBytes - checkNotNull(sonic).getPendingInputBytes();
+  }
+
   @Override
   public final AudioFormat configure(AudioFormat inputAudioFormat)
       throws UnhandledAudioFormatException {

libraries/common/src/main/java/androidx/media3/common/audio/SpeedChangingAudioProcessor.java

@@ -19,9 +19,14 @@ package androidx.media3.common.audio;
 import static java.lang.Math.min;
 
 import androidx.media3.common.C;
+import androidx.media3.common.util.LongArray;
+import androidx.media3.common.util.LongArrayQueue;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Queue;
+import java.util.function.LongConsumer;
 
 /**
  * An {@link AudioProcessor} that changes the speed of audio samples depending on their timestamp.
@@ -41,13 +46,31 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor {
    */
   private final SonicAudioProcessor sonicAudioProcessor;
 
+  private final Object pendingCallbacksLock;
+
+  // Elements in the same positions in the queues are associated.
+  private final LongArrayQueue pendingCallbackInputTimesUs;
+  private final Queue<LongConsumer> pendingCallbacks;
+
+  // Elements in the same positions in the arrays are associated.
+  private final LongArray inputSegmentStartTimesUs;
+  private final LongArray outputSegmentStartTimesUs;
+
   private float currentSpeed;
   private long bytesRead;
+  private long lastProcessedInputTime;
   private boolean endOfStreamQueuedToSonic;
 
   public SpeedChangingAudioProcessor(SpeedProvider speedProvider) {
     this.speedProvider = speedProvider;
     sonicAudioProcessor = new SonicAudioProcessor();
+    pendingCallbacksLock = new Object();
+    pendingCallbackInputTimesUs = new LongArrayQueue();
+    pendingCallbacks = new ArrayDeque<>();
+    inputSegmentStartTimesUs = new LongArray();
+    outputSegmentStartTimesUs = new LongArray();
+    inputSegmentStartTimesUs.add(0);
+    outputSegmentStartTimesUs.add(0);
     currentSpeed = 1f;
   }
 
@@ -66,6 +89,7 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor {
             /* divisor= */ (long) inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame);
     float newSpeed = speedProvider.getSpeed(timeUs);
     if (newSpeed != currentSpeed) {
+      updateSpeedChangeArrays(timeUs);
       currentSpeed = newSpeed;
       if (isUsingSonic()) {
         sonicAudioProcessor.setSpeed(newSpeed);
@@ -112,6 +136,7 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor {
       buffer.flip();
     }
     bytesRead += inputBuffer.position() - startPosition;
+    lastProcessedInputTime = updateLastProcessedInputTime();
     inputBuffer.limit(inputBufferLimit);
   }
 
@@ -125,7 +150,9 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor {
 
   @Override
   public ByteBuffer getOutput() {
-    return isUsingSonic() ? sonicAudioProcessor.getOutput() : super.getOutput();
+    ByteBuffer output = isUsingSonic() ? sonicAudioProcessor.getOutput() : super.getOutput();
+    processPendingCallbacks();
+    return output;
   }
 
   @Override
@@ -147,6 +174,99 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor {
     endOfStreamQueuedToSonic = false;
   }
 
+  /**
+   * Calculates the time at which the {@code inputTimeUs} is outputted at after the speed changes
+   * has been applied.
+   *
+   * <p>Calls {@linkplain LongConsumer#accept(long) the callback} with the output time as soon as
+   * enough audio has been processed to calculate it.
+   *
+   * <p>Successive calls must have monotonically increasing {@code inputTimeUs}.
+   *
+   * <p>Can be called from any thread.
+   *
+   * @param inputTimeUs The input time, in microseconds.
+   * @param callback The callback called with the output time. May be called on a different thread
+   *     from the caller of this method.
+   */
+  public void getSpeedAdjustedTimeAsync(long inputTimeUs, LongConsumer callback) {
+    synchronized (pendingCallbacksLock) {
+      if (inputTimeUs <= lastProcessedInputTime) {
+        callback.accept(calculateSpeedAdjustedTime(inputTimeUs));
+        return;
+      }
+      pendingCallbackInputTimesUs.add(inputTimeUs);
+      pendingCallbacks.add(callback);
+    }
+  }
+
+  /**
+   * Assuming enough audio has been processed, calculates the time at which the {@code inputTimeUs}
+   * is outputted at after the speed changes has been applied.
+   */
+  private long calculateSpeedAdjustedTime(long inputTimeUs) {
+    int floorIndex = inputSegmentStartTimesUs.size() - 1;
+    while (floorIndex > 0 && inputSegmentStartTimesUs.get(floorIndex) > inputTimeUs) {
+      floorIndex--;
+    }
+    long lastSegmentInputDuration = inputTimeUs - inputSegmentStartTimesUs.get(floorIndex);
+    long lastSegmentOutputDuration =
+        floorIndex == inputSegmentStartTimesUs.size() - 1
+            ? getPlayoutDurationAtCurrentSpeed(lastSegmentInputDuration)
+            : lastSegmentInputDuration
+                * (inputSegmentStartTimesUs.get(floorIndex + 1)
+                    - inputSegmentStartTimesUs.get(floorIndex))
+                / (outputSegmentStartTimesUs.get(floorIndex + 1)
+                    - outputSegmentStartTimesUs.get(floorIndex));
+    return outputSegmentStartTimesUs.get(floorIndex) + lastSegmentOutputDuration;
+  }
+
+  private void processPendingCallbacks() {
+    synchronized (pendingCallbacksLock) {
+      while (!pendingCallbacks.isEmpty()
+          && pendingCallbackInputTimesUs.element() <= lastProcessedInputTime) {
+        pendingCallbacks
+            .remove()
+            .accept(calculateSpeedAdjustedTime(pendingCallbackInputTimesUs.remove()));
+      }
+    }
+  }
+
+  private void updateSpeedChangeArrays(long currentSpeedChangeInputTimeUs) {
+    long lastSpeedChangeOutputTimeUs =
+        outputSegmentStartTimesUs.get(outputSegmentStartTimesUs.size() - 1);
+    long lastSpeedChangeInputTimeUs =
+        inputSegmentStartTimesUs.get(inputSegmentStartTimesUs.size() - 1);
+    long lastSpeedSegmentMediaDurationUs =
+        currentSpeedChangeInputTimeUs - lastSpeedChangeInputTimeUs;
+    inputSegmentStartTimesUs.add(currentSpeedChangeInputTimeUs);
+    outputSegmentStartTimesUs.add(
+        lastSpeedChangeOutputTimeUs
+            + getPlayoutDurationAtCurrentSpeed(lastSpeedSegmentMediaDurationUs));
+  }
+
+  private long getPlayoutDurationAtCurrentSpeed(long mediaDuration) {
+    return isUsingSonic() ? sonicAudioProcessor.getPlayoutDuration(mediaDuration) : mediaDuration;
+  }
+
+  private long updateLastProcessedInputTime() {
+    if (isUsingSonic()) {
+      // TODO - b/320242819: Investigate whether bytesRead can be used here rather than
+      //  sonicAudioProcessor.getProcessedInputBytes().
+      long currentProcessedInputDurationUs =
+          Util.scaleLargeTimestamp(
+              /* timestamp= */ sonicAudioProcessor.getProcessedInputBytes(),
+              /* multiplier= */ C.MICROS_PER_SECOND,
+              /* divisor= */ (long) inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame);
+      return inputSegmentStartTimesUs.get(inputSegmentStartTimesUs.size() - 1)
+          + currentProcessedInputDurationUs;
+    }
+    return Util.scaleLargeTimestamp(
+        /* timestamp= */ bytesRead,
+        /* multiplier= */ C.MICROS_PER_SECOND,
+        /* divisor= */ (long) inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame);
+  }
+
   private boolean isUsingSonic() {
     return currentSpeed != 1f;
   }

libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java

@@ -23,6 +23,7 @@ import androidx.media3.test.utils.TestSpeedProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.util.ArrayList;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
@@ -350,6 +351,32 @@ public class SpeedChangingAudioProcessorTest {
     assertThat(speedChangingAudioProcessor.isEnded()).isFalse();
   }
 
+  @Test
+  public void getSpeedAdjustedTimeAsync_callbacksCalledWithCorrectParameters() throws Exception {
+    ArrayList<Long> outputTimesUs = new ArrayList<>();
+    // The speed change is at 113Us (5*MICROS_PER_SECOND/sampleRate).
+    SpeedProvider speedProvider =
+        TestSpeedProvider.createWithFrameCounts(
+            AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
+    SpeedChangingAudioProcessor speedChangingAudioProcessor =
+        getConfiguredSpeedChangingAudioProcessor(speedProvider);
+    ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
+
+    speedChangingAudioProcessor.getSpeedAdjustedTimeAsync(
+        /* inputTimeUs= */ 50L, outputTimesUs::add);
+    speedChangingAudioProcessor.queueInput(inputBuffer);
+    getAudioProcessorOutput(speedChangingAudioProcessor);
+    inputBuffer.rewind();
+    speedChangingAudioProcessor.queueInput(inputBuffer);
+    getAudioProcessorOutput(speedChangingAudioProcessor);
+    speedChangingAudioProcessor.getSpeedAdjustedTimeAsync(
+        /* inputTimeUs= */ 100L, outputTimesUs::add);
+    speedChangingAudioProcessor.getSpeedAdjustedTimeAsync(
+        /* inputTimeUs= */ 150L, outputTimesUs::add);
+
+    assertThat(outputTimesUs).containsExactly(50L, 100L, 131L);
+  }
+
   private static SpeedChangingAudioProcessor getConfiguredSpeedChangingAudioProcessor(
       SpeedProvider speedProvider) throws AudioProcessor.UnhandledAudioFormatException {
     SpeedChangingAudioProcessor speedChangingAudioProcessor =