diff --git a/libraries/common/src/main/java/androidx/media3/common/audio/SonicAudioProcessor.java b/libraries/common/src/main/java/androidx/media3/common/audio/SonicAudioProcessor.java index 8762944cc8..9a55137524 100644 --- a/libraries/common/src/main/java/androidx/media3/common/audio/SonicAudioProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/audio/SonicAudioProcessor.java @@ -141,6 +141,36 @@ public class SonicAudioProcessor implements AudioProcessor { } } + /** + * Returns the playout duration corresponding to the specified media duration, taking speed + * adjustment into account. + * + *

The scaling performed by this method will use the actual playback speed achieved by the + * audio processor, on average, since it was last flushed. This may differ very slightly from the + * target playback speed. + * + * @param mediaDuration The media duration to scale. + * @return The corresponding playout duration, in the same units as {@code mediaDuration}. + */ + public final long getPlayoutDuration(long mediaDuration) { + if (outputBytes >= MIN_BYTES_FOR_DURATION_SCALING_CALCULATION) { + long processedInputBytes = inputBytes - checkNotNull(sonic).getPendingInputBytes(); + return outputAudioFormat.sampleRate == inputAudioFormat.sampleRate + ? Util.scaleLargeTimestamp(mediaDuration, outputBytes, processedInputBytes) + : Util.scaleLargeTimestamp( + mediaDuration, + outputBytes * inputAudioFormat.sampleRate, + processedInputBytes * outputAudioFormat.sampleRate); + } else { + return (long) (mediaDuration / (double) speed); + } + } + + /** Returns the number of bytes processed since last flush or reset. */ + public final long getProcessedInputBytes() { + return inputBytes - checkNotNull(sonic).getPendingInputBytes(); + } + @Override public final AudioFormat configure(AudioFormat inputAudioFormat) throws UnhandledAudioFormatException { diff --git a/libraries/common/src/main/java/androidx/media3/common/audio/SpeedChangingAudioProcessor.java b/libraries/common/src/main/java/androidx/media3/common/audio/SpeedChangingAudioProcessor.java index fb2cb2472f..26852f38b2 100644 --- a/libraries/common/src/main/java/androidx/media3/common/audio/SpeedChangingAudioProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/audio/SpeedChangingAudioProcessor.java @@ -19,9 +19,14 @@ package androidx.media3.common.audio; import static java.lang.Math.min; import androidx.media3.common.C; +import androidx.media3.common.util.LongArray; +import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.Queue; +import java.util.function.LongConsumer; /** * An {@link AudioProcessor} that changes the speed of audio samples depending on their timestamp. @@ -41,13 +46,31 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor { */ private final SonicAudioProcessor sonicAudioProcessor; + private final Object pendingCallbacksLock; + + // Elements in the same positions in the queues are associated. + private final LongArrayQueue pendingCallbackInputTimesUs; + private final Queue pendingCallbacks; + + // Elements in the same positions in the arrays are associated. + private final LongArray inputSegmentStartTimesUs; + private final LongArray outputSegmentStartTimesUs; + private float currentSpeed; private long bytesRead; + private long lastProcessedInputTime; private boolean endOfStreamQueuedToSonic; public SpeedChangingAudioProcessor(SpeedProvider speedProvider) { this.speedProvider = speedProvider; sonicAudioProcessor = new SonicAudioProcessor(); + pendingCallbacksLock = new Object(); + pendingCallbackInputTimesUs = new LongArrayQueue(); + pendingCallbacks = new ArrayDeque<>(); + inputSegmentStartTimesUs = new LongArray(); + outputSegmentStartTimesUs = new LongArray(); + inputSegmentStartTimesUs.add(0); + outputSegmentStartTimesUs.add(0); currentSpeed = 1f; } @@ -66,6 +89,7 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor { /* divisor= */ (long) inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame); float newSpeed = speedProvider.getSpeed(timeUs); if (newSpeed != currentSpeed) { + updateSpeedChangeArrays(timeUs); currentSpeed = newSpeed; if (isUsingSonic()) { sonicAudioProcessor.setSpeed(newSpeed); @@ -112,6 +136,7 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor { buffer.flip(); } bytesRead += inputBuffer.position() - startPosition; + lastProcessedInputTime = updateLastProcessedInputTime(); inputBuffer.limit(inputBufferLimit); } @@ -125,7 +150,9 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor { @Override public ByteBuffer getOutput() { - return isUsingSonic() ? sonicAudioProcessor.getOutput() : super.getOutput(); + ByteBuffer output = isUsingSonic() ? sonicAudioProcessor.getOutput() : super.getOutput(); + processPendingCallbacks(); + return output; } @Override @@ -147,6 +174,99 @@ public final class SpeedChangingAudioProcessor extends BaseAudioProcessor { endOfStreamQueuedToSonic = false; } + /** + * Calculates the time at which the {@code inputTimeUs} is outputted at after the speed changes + * has been applied. + * + *

Calls {@linkplain LongConsumer#accept(long) the callback} with the output time as soon as + * enough audio has been processed to calculate it. + * + *

Successive calls must have monotonically increasing {@code inputTimeUs}. + * + *

Can be called from any thread. + * + * @param inputTimeUs The input time, in microseconds. + * @param callback The callback called with the output time. May be called on a different thread + * from the caller of this method. + */ + public void getSpeedAdjustedTimeAsync(long inputTimeUs, LongConsumer callback) { + synchronized (pendingCallbacksLock) { + if (inputTimeUs <= lastProcessedInputTime) { + callback.accept(calculateSpeedAdjustedTime(inputTimeUs)); + return; + } + pendingCallbackInputTimesUs.add(inputTimeUs); + pendingCallbacks.add(callback); + } + } + + /** + * Assuming enough audio has been processed, calculates the time at which the {@code inputTimeUs} + * is outputted at after the speed changes has been applied. + */ + private long calculateSpeedAdjustedTime(long inputTimeUs) { + int floorIndex = inputSegmentStartTimesUs.size() - 1; + while (floorIndex > 0 && inputSegmentStartTimesUs.get(floorIndex) > inputTimeUs) { + floorIndex--; + } + long lastSegmentInputDuration = inputTimeUs - inputSegmentStartTimesUs.get(floorIndex); + long lastSegmentOutputDuration = + floorIndex == inputSegmentStartTimesUs.size() - 1 + ? getPlayoutDurationAtCurrentSpeed(lastSegmentInputDuration) + : lastSegmentInputDuration + * (inputSegmentStartTimesUs.get(floorIndex + 1) + - inputSegmentStartTimesUs.get(floorIndex)) + / (outputSegmentStartTimesUs.get(floorIndex + 1) + - outputSegmentStartTimesUs.get(floorIndex)); + return outputSegmentStartTimesUs.get(floorIndex) + lastSegmentOutputDuration; + } + + private void processPendingCallbacks() { + synchronized (pendingCallbacksLock) { + while (!pendingCallbacks.isEmpty() + && pendingCallbackInputTimesUs.element() <= lastProcessedInputTime) { + pendingCallbacks + .remove() + .accept(calculateSpeedAdjustedTime(pendingCallbackInputTimesUs.remove())); + } + } + } + + private void updateSpeedChangeArrays(long currentSpeedChangeInputTimeUs) { + long lastSpeedChangeOutputTimeUs = + outputSegmentStartTimesUs.get(outputSegmentStartTimesUs.size() - 1); + long lastSpeedChangeInputTimeUs = + inputSegmentStartTimesUs.get(inputSegmentStartTimesUs.size() - 1); + long lastSpeedSegmentMediaDurationUs = + currentSpeedChangeInputTimeUs - lastSpeedChangeInputTimeUs; + inputSegmentStartTimesUs.add(currentSpeedChangeInputTimeUs); + outputSegmentStartTimesUs.add( + lastSpeedChangeOutputTimeUs + + getPlayoutDurationAtCurrentSpeed(lastSpeedSegmentMediaDurationUs)); + } + + private long getPlayoutDurationAtCurrentSpeed(long mediaDuration) { + return isUsingSonic() ? sonicAudioProcessor.getPlayoutDuration(mediaDuration) : mediaDuration; + } + + private long updateLastProcessedInputTime() { + if (isUsingSonic()) { + // TODO - b/320242819: Investigate whether bytesRead can be used here rather than + // sonicAudioProcessor.getProcessedInputBytes(). + long currentProcessedInputDurationUs = + Util.scaleLargeTimestamp( + /* timestamp= */ sonicAudioProcessor.getProcessedInputBytes(), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ (long) inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame); + return inputSegmentStartTimesUs.get(inputSegmentStartTimesUs.size() - 1) + + currentProcessedInputDurationUs; + } + return Util.scaleLargeTimestamp( + /* timestamp= */ bytesRead, + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ (long) inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame); + } + private boolean isUsingSonic() { return currentSpeed != 1f; } diff --git a/libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java b/libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java index 9dc725cc9e..74caffbe42 100644 --- a/libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java @@ -23,6 +23,7 @@ import androidx.media3.test.utils.TestSpeedProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; import org.junit.Test; import org.junit.runner.RunWith; @@ -350,6 +351,32 @@ public class SpeedChangingAudioProcessorTest { assertThat(speedChangingAudioProcessor.isEnded()).isFalse(); } + @Test + public void getSpeedAdjustedTimeAsync_callbacksCalledWithCorrectParameters() throws Exception { + ArrayList outputTimesUs = new ArrayList<>(); + // The speed change is at 113Us (5*MICROS_PER_SECOND/sampleRate). + SpeedProvider speedProvider = + TestSpeedProvider.createWithFrameCounts( + AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2}); + SpeedChangingAudioProcessor speedChangingAudioProcessor = + getConfiguredSpeedChangingAudioProcessor(speedProvider); + ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); + + speedChangingAudioProcessor.getSpeedAdjustedTimeAsync( + /* inputTimeUs= */ 50L, outputTimesUs::add); + speedChangingAudioProcessor.queueInput(inputBuffer); + getAudioProcessorOutput(speedChangingAudioProcessor); + inputBuffer.rewind(); + speedChangingAudioProcessor.queueInput(inputBuffer); + getAudioProcessorOutput(speedChangingAudioProcessor); + speedChangingAudioProcessor.getSpeedAdjustedTimeAsync( + /* inputTimeUs= */ 100L, outputTimesUs::add); + speedChangingAudioProcessor.getSpeedAdjustedTimeAsync( + /* inputTimeUs= */ 150L, outputTimesUs::add); + + assertThat(outputTimesUs).containsExactly(50L, 100L, 131L); + } + private static SpeedChangingAudioProcessor getConfiguredSpeedChangingAudioProcessor( SpeedProvider speedProvider) throws AudioProcessor.UnhandledAudioFormatException { SpeedChangingAudioProcessor speedChangingAudioProcessor =