diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SegmentSpeedProvider.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SegmentSpeedProvider.java index aa06e8dfc6..42aa74423e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SegmentSpeedProvider.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SegmentSpeedProvider.java @@ -59,6 +59,13 @@ import java.util.TreeMap; return entry != null ? entry.getValue() : baseSpeedMultiplier; } + @Override + public long getNextSpeedChangeTimeUs(long timeUs) { + checkArgument(timeUs >= 0); + @Nullable Long nextTimeUs = speedsByStartTimeUs.higherKey(timeUs); + return nextTimeUs != null ? nextTimeUs : C.TIME_UNSET; + } + private static ImmutableSortedMap buildSpeedByStartTimeUsMap( Format format, float baseSpeed) { List segments = extractSlowMotionSegments(format); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedChangingAudioProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedChangingAudioProcessor.java new file mode 100644 index 0000000000..16a55089fd --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedChangingAudioProcessor.java @@ -0,0 +1,150 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static java.lang.Math.min; + +import androidx.media3.common.C; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.audio.AudioProcessor; +import androidx.media3.exoplayer.audio.BaseAudioProcessor; +import androidx.media3.exoplayer.audio.SonicAudioProcessor; +import java.nio.ByteBuffer; + +/** + * An {@link AudioProcessor} that changes the speed of audio samples depending on their timestamp. + */ +/* package */ final class SpeedChangingAudioProcessor extends BaseAudioProcessor { + + /** The speed provider that provides the speed for each timestamp. */ + private final SpeedProvider speedProvider; + /** + * The {@link SonicAudioProcessor} used to change the speed, when needed. If there is no speed + * change required, the input buffer is copied to the output buffer and this processor is not + * used. + */ + private final SonicAudioProcessor sonicAudioProcessor; + + private float currentSpeed; + private long bytesRead; + private boolean endOfStreamQueuedToSonic; + + public SpeedChangingAudioProcessor(SpeedProvider speedProvider) { + this.speedProvider = speedProvider; + sonicAudioProcessor = new SonicAudioProcessor(); + currentSpeed = 1f; + } + + @Override + public AudioFormat onConfigure(AudioFormat inputAudioFormat) + throws UnhandledAudioFormatException { + return sonicAudioProcessor.configure(inputAudioFormat); + } + + @Override + public void queueInput(ByteBuffer inputBuffer) { + long timeUs = + Util.scaleLargeTimestamp( + /* timestamp= */ bytesRead, + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ (long) inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame); + float newSpeed = speedProvider.getSpeed(timeUs); + if (newSpeed != currentSpeed) { + currentSpeed = newSpeed; + if (isUsingSonic()) { + sonicAudioProcessor.setSpeed(newSpeed); + sonicAudioProcessor.setPitch(newSpeed); + sonicAudioProcessor.flush(); + endOfStreamQueuedToSonic = false; + } + } + + int inputBufferLimit = inputBuffer.limit(); + long nextSpeedChangeTimeUs = speedProvider.getNextSpeedChangeTimeUs(timeUs); + int bytesToNextSpeedChange; + if (nextSpeedChangeTimeUs != C.TIME_UNSET) { + bytesToNextSpeedChange = + (int) + Util.scaleLargeTimestamp( + /* timestamp= */ nextSpeedChangeTimeUs - timeUs, + /* multiplier= */ (long) inputAudioFormat.sampleRate + * inputAudioFormat.bytesPerFrame, + /* divisor= */ C.MICROS_PER_SECOND); + int bytesToNextFrame = + inputAudioFormat.bytesPerFrame - bytesToNextSpeedChange % inputAudioFormat.bytesPerFrame; + if (bytesToNextFrame != inputAudioFormat.bytesPerFrame) { + bytesToNextSpeedChange += bytesToNextFrame; + } + // Update the input buffer limit to make sure that all samples processed have the same speed. + inputBuffer.limit(min(inputBufferLimit, inputBuffer.position() + bytesToNextSpeedChange)); + } else { + bytesToNextSpeedChange = C.LENGTH_UNSET; + } + + long startPosition = inputBuffer.position(); + if (isUsingSonic()) { + sonicAudioProcessor.queueInput(inputBuffer); + if (bytesToNextSpeedChange != C.LENGTH_UNSET + && (inputBuffer.position() - startPosition) == bytesToNextSpeedChange) { + sonicAudioProcessor.queueEndOfStream(); + endOfStreamQueuedToSonic = true; + } + } else { + ByteBuffer buffer = replaceOutputBuffer(/* count= */ inputBuffer.remaining()); + buffer.put(inputBuffer); + buffer.flip(); + } + bytesRead += inputBuffer.position() - startPosition; + inputBuffer.limit(inputBufferLimit); + } + + @Override + protected void onQueueEndOfStream() { + if (!endOfStreamQueuedToSonic) { + sonicAudioProcessor.queueEndOfStream(); + endOfStreamQueuedToSonic = true; + } + } + + @Override + public ByteBuffer getOutput() { + return isUsingSonic() ? sonicAudioProcessor.getOutput() : super.getOutput(); + } + + @Override + public boolean isEnded() { + return super.isEnded() && sonicAudioProcessor.isEnded(); + } + + @Override + protected void onFlush() { + sonicAudioProcessor.flush(); + endOfStreamQueuedToSonic = false; + } + + @Override + protected void onReset() { + currentSpeed = 1f; + bytesRead = 0; + sonicAudioProcessor.reset(); + endOfStreamQueuedToSonic = false; + } + + private boolean isUsingSonic() { + return currentSpeed != 1f; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedProvider.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedProvider.java index 74c8a52a93..dc85d90023 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedProvider.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SpeedProvider.java @@ -15,6 +15,8 @@ */ package androidx.media3.transformer; +import androidx.media3.common.C; + /** A custom interface that determines the speed for media at specific timestamps. */ /* package */ interface SpeedProvider { @@ -25,4 +27,14 @@ package androidx.media3.transformer; * @return The speed that the media should be played at, based on the timeUs. */ float getSpeed(long timeUs); + + /** + * Returns the timestamp of the next speed change, if there is any. + * + * @param timeUs A timestamp, in microseconds. + * @return The timestamp of the next speed change, in microseconds, or {@link C#TIME_UNSET} if + * there is no next speed change. If {@code timeUs} corresponds to a speed change, the + * returned value corresponds to the following speed change. + */ + long getNextSpeedChangeTimeUs(long timeUs); }