Add SpeedChangingAudioProcessor

PiperOrigin-RevId: 425562875

libraries/transformer/src/main/java/androidx/media3/transformer/SegmentSpeedProvider.java

@@ -59,6 +59,13 @@ import java.util.TreeMap;
     return entry != null ? entry.getValue() : baseSpeedMultiplier;
   }
 
+  @Override
+  public long getNextSpeedChangeTimeUs(long timeUs) {
+    checkArgument(timeUs >= 0);
+    @Nullable Long nextTimeUs = speedsByStartTimeUs.higherKey(timeUs);
+    return nextTimeUs != null ? nextTimeUs : C.TIME_UNSET;
+  }
+
   private static ImmutableSortedMap<Long, Float> buildSpeedByStartTimeUsMap(
       Format format, float baseSpeed) {
     List<Segment> segments = extractSlowMotionSegments(format);

libraries/transformer/src/main/java/androidx/media3/transformer/SpeedChangingAudioProcessor.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.transformer;
+
+import static java.lang.Math.min;
+
+import androidx.media3.common.C;
+import androidx.media3.common.util.Util;
+import androidx.media3.exoplayer.audio.AudioProcessor;
+import androidx.media3.exoplayer.audio.BaseAudioProcessor;
+import androidx.media3.exoplayer.audio.SonicAudioProcessor;
+import java.nio.ByteBuffer;
+
+/**
+ * An {@link AudioProcessor} that changes the speed of audio samples depending on their timestamp.
+ */
+/* package */ final class SpeedChangingAudioProcessor extends BaseAudioProcessor {
+
+  /** The speed provider that provides the speed for each timestamp. */
+  private final SpeedProvider speedProvider;
+  /**
+   * The {@link SonicAudioProcessor} used to change the speed, when needed. If there is no speed
+   * change required, the input buffer is copied to the output buffer and this processor is not
+   * used.
+   */
+  private final SonicAudioProcessor sonicAudioProcessor;
+
+  private float currentSpeed;
+  private long bytesRead;
+  private boolean endOfStreamQueuedToSonic;
+
+  public SpeedChangingAudioProcessor(SpeedProvider speedProvider) {
+    this.speedProvider = speedProvider;
+    sonicAudioProcessor = new SonicAudioProcessor();
+    currentSpeed = 1f;
+  }
+
+  @Override
+  public AudioFormat onConfigure(AudioFormat inputAudioFormat)
+      throws UnhandledAudioFormatException {
+    return sonicAudioProcessor.configure(inputAudioFormat);
+  }
+
+  @Override
+  public void queueInput(ByteBuffer inputBuffer) {
+    long timeUs =
+        Util.scaleLargeTimestamp(
+            /* timestamp= */ bytesRead,
+            /* multiplier= */ C.MICROS_PER_SECOND,
+            /* divisor= */ (long) inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame);
+    float newSpeed = speedProvider.getSpeed(timeUs);
+    if (newSpeed != currentSpeed) {
+      currentSpeed = newSpeed;
+      if (isUsingSonic()) {
+        sonicAudioProcessor.setSpeed(newSpeed);
+        sonicAudioProcessor.setPitch(newSpeed);
+        sonicAudioProcessor.flush();
+        endOfStreamQueuedToSonic = false;
+      }
+    }
+
+    int inputBufferLimit = inputBuffer.limit();
+    long nextSpeedChangeTimeUs = speedProvider.getNextSpeedChangeTimeUs(timeUs);
+    int bytesToNextSpeedChange;
+    if (nextSpeedChangeTimeUs != C.TIME_UNSET) {
+      bytesToNextSpeedChange =
+          (int)
+              Util.scaleLargeTimestamp(
+                  /* timestamp= */ nextSpeedChangeTimeUs - timeUs,
+                  /* multiplier= */ (long) inputAudioFormat.sampleRate
+                      * inputAudioFormat.bytesPerFrame,
+                  /* divisor= */ C.MICROS_PER_SECOND);
+      int bytesToNextFrame =
+          inputAudioFormat.bytesPerFrame - bytesToNextSpeedChange % inputAudioFormat.bytesPerFrame;
+      if (bytesToNextFrame != inputAudioFormat.bytesPerFrame) {
+        bytesToNextSpeedChange += bytesToNextFrame;
+      }
+      // Update the input buffer limit to make sure that all samples processed have the same speed.
+      inputBuffer.limit(min(inputBufferLimit, inputBuffer.position() + bytesToNextSpeedChange));
+    } else {
+      bytesToNextSpeedChange = C.LENGTH_UNSET;
+    }
+
+    long startPosition = inputBuffer.position();
+    if (isUsingSonic()) {
+      sonicAudioProcessor.queueInput(inputBuffer);
+      if (bytesToNextSpeedChange != C.LENGTH_UNSET
+          && (inputBuffer.position() - startPosition) == bytesToNextSpeedChange) {
+        sonicAudioProcessor.queueEndOfStream();
+        endOfStreamQueuedToSonic = true;
+      }
+    } else {
+      ByteBuffer buffer = replaceOutputBuffer(/* count= */ inputBuffer.remaining());
+      buffer.put(inputBuffer);
+      buffer.flip();
+    }
+    bytesRead += inputBuffer.position() - startPosition;
+    inputBuffer.limit(inputBufferLimit);
+  }
+
+  @Override
+  protected void onQueueEndOfStream() {
+    if (!endOfStreamQueuedToSonic) {
+      sonicAudioProcessor.queueEndOfStream();
+      endOfStreamQueuedToSonic = true;
+    }
+  }
+
+  @Override
+  public ByteBuffer getOutput() {
+    return isUsingSonic() ? sonicAudioProcessor.getOutput() : super.getOutput();
+  }
+
+  @Override
+  public boolean isEnded() {
+    return super.isEnded() && sonicAudioProcessor.isEnded();
+  }
+
+  @Override
+  protected void onFlush() {
+    sonicAudioProcessor.flush();
+    endOfStreamQueuedToSonic = false;
+  }
+
+  @Override
+  protected void onReset() {
+    currentSpeed = 1f;
+    bytesRead = 0;
+    sonicAudioProcessor.reset();
+    endOfStreamQueuedToSonic = false;
+  }
+
+  private boolean isUsingSonic() {
+    return currentSpeed != 1f;
+  }
+}

libraries/transformer/src/main/java/androidx/media3/transformer/SpeedProvider.java

@@ -15,6 +15,8 @@
  */
 package androidx.media3.transformer;
 
+import androidx.media3.common.C;
+
 /** A custom interface that determines the speed for media at specific timestamps. */
 /* package */ interface SpeedProvider {
 
@@ -25,4 +27,14 @@ package androidx.media3.transformer;
    * @return The speed that the media should be played at, based on the timeUs.
    */
   float getSpeed(long timeUs);
+
+  /**
+   * Returns the timestamp of the next speed change, if there is any.
+   *
+   * @param timeUs A timestamp, in microseconds.
+   * @return The timestamp of the next speed change, in microseconds, or {@link C#TIME_UNSET} if
+   *     there is no next speed change. If {@code timeUs} corresponds to a speed change, the
+   *     returned value corresponds to the following speed change.
+   */
+  long getNextSpeedChangeTimeUs(long timeUs);
 }