mirror of
https://github.com/androidx/media.git
synced 2025-04-30 06:46:50 +08:00
Implement getExpectedFrameCountAfterProcessorApplied()
in Sonic
This method allows `Sonic` to statically and accurately report the expected number of output frames for any given parameter configuration. This change is required prework for `SpeedChangingAudioProcessor` to implement a similar static method and allow precise, non-blocking timestamp adjustments for the experimental speed changing effect. PiperOrigin-RevId: 690669627
This commit is contained in:
parent
772bd20f7d
commit
7e7764de5e
@ -72,6 +72,45 @@ import java.util.Arrays;
|
|||||||
private int maxDiff;
|
private int maxDiff;
|
||||||
private double accumulatedSpeedAdjustmentError;
|
private double accumulatedSpeedAdjustmentError;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the estimated output frame count for a given configuration and input frame count.
|
||||||
|
*
|
||||||
|
* <p>Please note that the returned value might not be mathematically exact, as Sonic incurs in
|
||||||
|
* truncation and precision errors that accumulate on the output.
|
||||||
|
*/
|
||||||
|
public static long getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
int inputSampleRateHz,
|
||||||
|
int outputSampleRateHz,
|
||||||
|
float speed,
|
||||||
|
float pitch,
|
||||||
|
long inputFrameCount) {
|
||||||
|
float resamplingRate = (float) inputSampleRateHz / outputSampleRateHz;
|
||||||
|
resamplingRate *= pitch;
|
||||||
|
double speedRate = speed / pitch;
|
||||||
|
BigDecimal bigResamplingRate = new BigDecimal(String.valueOf(resamplingRate));
|
||||||
|
|
||||||
|
BigDecimal length = BigDecimal.valueOf(inputFrameCount);
|
||||||
|
BigDecimal framesAfterTimeStretching;
|
||||||
|
if (speedRate > 1.00001 || speedRate < 0.99999) {
|
||||||
|
framesAfterTimeStretching =
|
||||||
|
length.divide(BigDecimal.valueOf(speedRate), RoundingMode.HALF_EVEN);
|
||||||
|
} else {
|
||||||
|
// If speed is almost 1, then just copy the buffers without modifying them.
|
||||||
|
framesAfterTimeStretching = length;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (resamplingRate == 1.0f) {
|
||||||
|
return framesAfterTimeStretching.longValueExact();
|
||||||
|
}
|
||||||
|
|
||||||
|
BigDecimal framesAfterResampling =
|
||||||
|
framesAfterTimeStretching.divide(bigResamplingRate, RoundingMode.HALF_EVEN);
|
||||||
|
|
||||||
|
return framesAfterResampling.longValueExact()
|
||||||
|
- calculateAccumulatedTruncationErrorForResampling(
|
||||||
|
framesAfterTimeStretching, BigDecimal.valueOf(inputSampleRateHz), bigResamplingRate);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns expected accumulated truncation error for {@link Sonic}'s resampling algorithm, given
|
* Returns expected accumulated truncation error for {@link Sonic}'s resampling algorithm, given
|
||||||
* an input length, input sample rate, and resampling rate.
|
* an input length, input sample rate, and resampling rate.
|
||||||
|
@ -15,7 +15,6 @@
|
|||||||
*/
|
*/
|
||||||
package androidx.media3.common.audio;
|
package androidx.media3.common.audio;
|
||||||
|
|
||||||
import static androidx.media3.common.audio.Sonic.calculateAccumulatedTruncationErrorForResampling;
|
|
||||||
import static androidx.media3.test.utils.TestUtil.generateFloatInRange;
|
import static androidx.media3.test.utils.TestUtil.generateFloatInRange;
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
import static com.google.common.truth.Truth.assertThat;
|
||||||
import static java.lang.Math.max;
|
import static java.lang.Math.max;
|
||||||
@ -164,18 +163,10 @@ public final class RandomParameterizedSonicTest {
|
|||||||
}
|
}
|
||||||
sonic.flush();
|
sonic.flush();
|
||||||
|
|
||||||
BigDecimal bigLength = new BigDecimal(String.valueOf(streamLength));
|
long expectedSamples =
|
||||||
// The scale of expectedSize will be bigLength.scale() - speed.scale(). Thus, the result should
|
Sonic.getExpectedFrameCountAfterProcessorApplied(
|
||||||
// always yield an integer.
|
SAMPLE_RATE, SAMPLE_RATE, speed.floatValue(), speed.floatValue(), streamLength);
|
||||||
BigDecimal expectedSize = bigLength.divide(speed, RoundingMode.HALF_EVEN);
|
assertThat(readSampleCount).isWithin(1).of(expectedSamples);
|
||||||
|
|
||||||
long accumulatedTruncationError =
|
|
||||||
calculateAccumulatedTruncationErrorForResampling(
|
|
||||||
bigLength, new BigDecimal(SAMPLE_RATE), speed);
|
|
||||||
|
|
||||||
assertThat(readSampleCount)
|
|
||||||
.isWithin(1)
|
|
||||||
.of(expectedSize.longValueExact() - accumulatedTruncationError);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -208,20 +199,18 @@ public final class RandomParameterizedSonicTest {
|
|||||||
}
|
}
|
||||||
sonic.flush();
|
sonic.flush();
|
||||||
|
|
||||||
BigDecimal bigLength = new BigDecimal(String.valueOf(streamLength));
|
long expectedSamples =
|
||||||
// The scale of expectedSampleCount will be bigLength.scale() - speed.scale(). Thus, the result
|
Sonic.getExpectedFrameCountAfterProcessorApplied(
|
||||||
// should always yield an integer.
|
SAMPLE_RATE, SAMPLE_RATE, speed.floatValue(), 1, streamLength);
|
||||||
BigDecimal expectedSampleCount = bigLength.divide(speed, RoundingMode.HALF_EVEN);
|
|
||||||
|
|
||||||
// Calculate allowed tolerance and round to nearest integer.
|
// Calculate allowed tolerance and round to nearest integer.
|
||||||
BigDecimal allowedTolerance =
|
BigDecimal allowedTolerance =
|
||||||
TIME_STRETCHING_SAMPLE_DRIFT_TOLERANCE
|
TIME_STRETCHING_SAMPLE_DRIFT_TOLERANCE
|
||||||
.multiply(expectedSampleCount)
|
.multiply(BigDecimal.valueOf(expectedSamples))
|
||||||
.setScale(/* newScale= */ 0, RoundingMode.HALF_EVEN);
|
.setScale(/* newScale= */ 0, RoundingMode.HALF_EVEN);
|
||||||
|
|
||||||
// Always allow at least 1 sample of tolerance.
|
// Always allow at least 1 sample of tolerance.
|
||||||
long tolerance = max(allowedTolerance.longValue(), 1);
|
long tolerance = max(allowedTolerance.longValue(), 1);
|
||||||
|
assertThat(readSampleCount).isWithin(tolerance).of(expectedSamples);
|
||||||
assertThat(readSampleCount).isWithin(tolerance).of(expectedSampleCount.longValueExact());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -15,7 +15,6 @@
|
|||||||
*/
|
*/
|
||||||
package androidx.media3.common.audio;
|
package androidx.media3.common.audio;
|
||||||
|
|
||||||
import static androidx.media3.common.audio.Sonic.calculateAccumulatedTruncationErrorForResampling;
|
|
||||||
import static androidx.media3.test.utils.TestUtil.buildTestData;
|
import static androidx.media3.test.utils.TestUtil.buildTestData;
|
||||||
import static androidx.media3.test.utils.TestUtil.generateFloatInRange;
|
import static androidx.media3.test.utils.TestUtil.generateFloatInRange;
|
||||||
import static androidx.media3.test.utils.TestUtil.generateLong;
|
import static androidx.media3.test.utils.TestUtil.generateLong;
|
||||||
@ -27,6 +26,7 @@ import androidx.media3.test.utils.TestSpeedProvider;
|
|||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.Range;
|
import com.google.common.collect.Range;
|
||||||
import com.google.common.primitives.Floats;
|
import com.google.common.primitives.Floats;
|
||||||
|
import com.google.common.primitives.Ints;
|
||||||
import java.math.BigDecimal;
|
import java.math.BigDecimal;
|
||||||
import java.math.RoundingMode;
|
import java.math.RoundingMode;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
@ -102,30 +102,26 @@ public class RandomParameterizedSpeedChangingAudioProcessorTest {
|
|||||||
ByteBuffer.wrap(
|
ByteBuffer.wrap(
|
||||||
buildTestData(/* length= */ BUFFER_SIZE * AUDIO_FORMAT.bytesPerFrame, random));
|
buildTestData(/* length= */ BUFFER_SIZE * AUDIO_FORMAT.bytesPerFrame, random));
|
||||||
ByteBuffer outBuffer;
|
ByteBuffer outBuffer;
|
||||||
BigDecimal expectedTotalOutputFrameCount = BigDecimal.ZERO;
|
|
||||||
long outputFrameCount = 0;
|
long outputFrameCount = 0;
|
||||||
long totalInputFrameCount = 0;
|
long totalInputFrameCount = 0;
|
||||||
long expectedResamplingError = 0;
|
long expectedOutputFrames = 0;
|
||||||
|
|
||||||
for (int i = 0; i < frameCounts.size(); i++) {
|
for (int i = 0; i < frameCounts.size(); i++) {
|
||||||
totalInputFrameCount += frameCounts.get(i);
|
totalInputFrameCount += frameCounts.get(i);
|
||||||
BigDecimal frameCount = BigDecimal.valueOf(frameCounts.get(i));
|
float speed = speeds.get(i).floatValue();
|
||||||
BigDecimal speed = speeds.get(i);
|
expectedOutputFrames +=
|
||||||
BigDecimal expectedOutputFrameCountForSection =
|
Sonic.getExpectedFrameCountAfterProcessorApplied(
|
||||||
frameCount.divide(speed, RoundingMode.HALF_EVEN);
|
/* inputSampleRateHz= */ AUDIO_FORMAT.sampleRate,
|
||||||
expectedTotalOutputFrameCount =
|
/* outputSampleRateHz= */ AUDIO_FORMAT.sampleRate,
|
||||||
expectedTotalOutputFrameCount.add(expectedOutputFrameCountForSection);
|
/* speed= */ speed,
|
||||||
// SpeedChangingAudioProcessor currently uses resampling on Sonic, instead of time-stretching.
|
/* pitch= */ speed,
|
||||||
// See b/359649531.
|
/* inputFrameCount= */ frameCounts.get(i));
|
||||||
expectedResamplingError +=
|
|
||||||
calculateAccumulatedTruncationErrorForResampling(
|
|
||||||
frameCount, BigDecimal.valueOf(AUDIO_FORMAT.sampleRate), speed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
SpeedProvider speedProvider =
|
SpeedProvider speedProvider =
|
||||||
TestSpeedProvider.createWithFrameCounts(
|
TestSpeedProvider.createWithFrameCounts(
|
||||||
AUDIO_FORMAT,
|
AUDIO_FORMAT,
|
||||||
/* frameCounts= */ frameCounts.stream().mapToInt(Math::toIntExact).toArray(),
|
/* frameCounts= */ Ints.toArray(frameCounts),
|
||||||
/* speeds= */ Floats.toArray(speeds));
|
/* speeds= */ Floats.toArray(speeds));
|
||||||
|
|
||||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||||
@ -152,8 +148,6 @@ public class RandomParameterizedSpeedChangingAudioProcessorTest {
|
|||||||
outputFrameCount += outBuffer.remaining() / AUDIO_FORMAT.bytesPerFrame;
|
outputFrameCount += outBuffer.remaining() / AUDIO_FORMAT.bytesPerFrame;
|
||||||
|
|
||||||
// We allow 1 frame of tolerance per speed change.
|
// We allow 1 frame of tolerance per speed change.
|
||||||
assertThat(outputFrameCount)
|
assertThat(outputFrameCount).isWithin(frameCounts.size()).of(expectedOutputFrames);
|
||||||
.isWithin(frameCounts.size())
|
|
||||||
.of(expectedTotalOutputFrameCount.longValueExact() - expectedResamplingError);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -16,6 +16,7 @@
|
|||||||
package androidx.media3.common.audio;
|
package androidx.media3.common.audio;
|
||||||
|
|
||||||
import static androidx.media3.common.audio.Sonic.calculateAccumulatedTruncationErrorForResampling;
|
import static androidx.media3.common.audio.Sonic.calculateAccumulatedTruncationErrorForResampling;
|
||||||
|
import static androidx.media3.common.audio.Sonic.getExpectedFrameCountAfterProcessorApplied;
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
import static com.google.common.truth.Truth.assertThat;
|
||||||
|
|
||||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||||
@ -110,6 +111,141 @@ public class SonicTest {
|
|||||||
assertThat(outputBuffer.array()).isEqualTo(new short[] {0, 4, 8});
|
assertThat(outputBuffer.array()).isEqualTo(new short[] {0, 4, 8});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_timeStretchingFaster_returnsExpectedSampleCount() {
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 44100,
|
||||||
|
/* outputSampleRateHz= */ 44100,
|
||||||
|
/* speed= */ 2,
|
||||||
|
/* pitch= */ 1,
|
||||||
|
/* inputFrameCount= */ 88200);
|
||||||
|
assertThat(samples).isEqualTo(44100);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_timeStretchingSlower_returnsExpectedSampleCount() {
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 44100,
|
||||||
|
/* outputSampleRateHz= */ 44100,
|
||||||
|
/* speed= */ 0.5f,
|
||||||
|
/* pitch= */ 1,
|
||||||
|
/* inputFrameCount= */ 88200);
|
||||||
|
assertThat(samples).isEqualTo(176400);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_resamplingHigherSampleRate_returnsExpectedSampleCount() {
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 44100,
|
||||||
|
/* outputSampleRateHz= */ 88200,
|
||||||
|
/* speed= */ 1f,
|
||||||
|
/* pitch= */ 1,
|
||||||
|
/* inputFrameCount= */ 88200);
|
||||||
|
assertThat(samples).isEqualTo(176400);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_resamplingLowerSampleRate_returnsExpectedSampleCount() {
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 44100,
|
||||||
|
/* outputSampleRateHz= */ 22050,
|
||||||
|
/* speed= */ 1f,
|
||||||
|
/* pitch= */ 1,
|
||||||
|
/* inputFrameCount= */ 88200);
|
||||||
|
assertThat(samples).isEqualTo(44100);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_resamplingLowerPitch_returnsExpectedSampleCount() {
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 44100,
|
||||||
|
/* outputSampleRateHz= */ 44100,
|
||||||
|
/* speed= */ 0.5f,
|
||||||
|
/* pitch= */ 0.5f,
|
||||||
|
/* inputFrameCount= */ 88200);
|
||||||
|
assertThat(samples).isEqualTo(176400);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_resamplingHigherPitch_returnsExpectedSampleCount() {
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 44100,
|
||||||
|
/* outputSampleRateHz= */ 44100,
|
||||||
|
/* speed= */ 2f,
|
||||||
|
/* pitch= */ 2f,
|
||||||
|
/* inputFrameCount= */ 88200);
|
||||||
|
assertThat(samples).isEqualTo(44100);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_resamplePitchAndSampleRateChange_returnsExpectedSampleCount() {
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 44100,
|
||||||
|
/* outputSampleRateHz= */ 88200,
|
||||||
|
/* speed= */ 1f,
|
||||||
|
/* pitch= */ 2f,
|
||||||
|
/* inputFrameCount= */ 88200);
|
||||||
|
// First time stretch at speed / pitch = 0.5.
|
||||||
|
// Then resample at (inputSampleRateHz / outputSampleRateHz) * pitch = 0.5 * 2.
|
||||||
|
// Final sample count is 88200 / 0.5 / (0.5 * 2) = 176400.
|
||||||
|
assertThat(samples).isEqualTo(176400);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_pitchSpeedAndSampleRateChange_returnsExpectedSampleCount() {
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 48000,
|
||||||
|
/* outputSampleRateHz= */ 192000,
|
||||||
|
/* speed= */ 5f,
|
||||||
|
/* pitch= */ 0.5f,
|
||||||
|
/* inputFrameCount= */ 88200);
|
||||||
|
// First time stretch at speed / pitch = 10.
|
||||||
|
// Then resample at (inputSampleRateHz / outputSampleRateHz) * pitch = 0.25 * 0.5.
|
||||||
|
// Final sample count is 88200 / 10 / (0.25 * 0.5) = 176400.
|
||||||
|
assertThat(samples).isEqualTo(70560);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
getExpectedFrameCountAfterProcessorApplied_withPeriodicResamplingRate_adjustsForTruncationError() {
|
||||||
|
long length = 26902000;
|
||||||
|
float resamplingRate = 0.33f;
|
||||||
|
long samples =
|
||||||
|
getExpectedFrameCountAfterProcessorApplied(
|
||||||
|
/* inputSampleRateHz= */ 48000,
|
||||||
|
/* outputSampleRateHz= */ 48000,
|
||||||
|
/* speed= */ resamplingRate,
|
||||||
|
/* pitch= */ resamplingRate,
|
||||||
|
/* inputFrameCount= */ length);
|
||||||
|
|
||||||
|
long truncationError =
|
||||||
|
calculateAccumulatedTruncationErrorForResampling(
|
||||||
|
BigDecimal.valueOf(length),
|
||||||
|
BigDecimal.valueOf(48000),
|
||||||
|
new BigDecimal(String.valueOf(resamplingRate)));
|
||||||
|
// Sonic incurs on accumulated truncation errors when the input sample rate is not exactly
|
||||||
|
// divisible by the resampling rate (pitch * inputSampleRateHz / outputSampleRateHz). This error
|
||||||
|
// is more prominent on larger stream lengths and inputSampleRateHz + resamplingRate
|
||||||
|
// combinations that result in higher truncated decimal values.
|
||||||
|
assertThat(samples).isEqualTo(81521212 - truncationError);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void calculateAccumulatedTruncationErrorForResampling_returnsExpectedSampleCount() {
|
public void calculateAccumulatedTruncationErrorForResampling_returnsExpectedSampleCount() {
|
||||||
long error =
|
long error =
|
||||||
|
Loading…
x
Reference in New Issue
Block a user