Support segment-specific speed changes in SpeedChangeEffect
PiperOrigin-RevId: 606204479
This commit is contained in:
tofunmi
Copybara-Service
parent
0b0c419c73
commit
1a5eb4eecd
@ -18,6 +18,8 @@
|
||||
* DRM:
|
||||
* Effect:
|
||||
* Improved PQ to SDR tone-mapping by converting color spaces.
|
||||
* Support multiple speed changes within the same `EditedMediaItem` or
|
||||
`Composition` in `SpeedChangeEffect`.
|
||||
* Muxers:
|
||||
* IMA extension:
|
||||
* Session:
|
||||
|
||||
@ -16,11 +16,10 @@
|
||||
package androidx.media3.common.audio;
|
||||
|
||||
import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER;
|
||||
import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.util.Util;
|
||||
import androidx.media3.test.utils.TestSpeedProvider;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
@ -39,7 +38,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
public void queueInput_noSpeedChange_doesNotOverwriteInput() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -54,7 +53,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
public void queueInput_speedChange_doesNotOverwriteInput() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -69,7 +68,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
public void queueInput_noSpeedChange_copiesSamples() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -86,7 +85,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
public void queueInput_speedChange_modifiesSamples() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -104,7 +103,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
public void queueInput_noSpeedChangeAfterSpeedChange_copiesSamples() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -124,7 +123,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -137,7 +136,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
|
||||
speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
inputBuffer.rewind();
|
||||
speedChangingAudioProcessor.queueInput(inputBuffer);
|
||||
@ -152,7 +151,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {3, 2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {3, 2});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -165,7 +164,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
|
||||
speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
inputBuffer.rewind();
|
||||
speedChangingAudioProcessor.queueInput(inputBuffer);
|
||||
@ -180,7 +179,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 3});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 3});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -190,7 +189,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
|
||||
speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
inputBuffer.rewind();
|
||||
speedChangingAudioProcessor.queueInput(inputBuffer);
|
||||
@ -243,7 +242,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -262,7 +261,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -281,7 +280,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -297,7 +296,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -312,7 +311,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
public void queueEndOfStream_noInputQueued_endsProcessor() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
|
||||
@ -325,7 +324,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
public void isEnded_afterNoSpeedChangeAndOutputRetrieved_isFalse() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -340,7 +339,7 @@ public class SpeedChangingAudioProcessorTest {
|
||||
public void isEnded_afterSpeedChangeAndOutputRetrieved_isFalse() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithFrameCounts(
|
||||
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
|
||||
SpeedChangingAudioProcessor speedChangingAudioProcessor =
|
||||
getConfiguredSpeedChangingAudioProcessor(speedProvider);
|
||||
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
|
||||
@ -388,62 +387,4 @@ public class SpeedChangingAudioProcessorTest {
|
||||
}
|
||||
return concatenatedOutputBuffers;
|
||||
}
|
||||
|
||||
private static final class TestSpeedProvider implements SpeedProvider {
|
||||
|
||||
private final long[] startTimesUs;
|
||||
private final float[] speeds;
|
||||
|
||||
/**
|
||||
* Creates a {@code TestSpeedProvider} instance.
|
||||
*
|
||||
* @param startTimesUs The speed change start times, in microseconds. The values must be
|
||||
* distinct and in increasing order.
|
||||
* @param speeds The speeds corresponding to each start time. Consecutive values must be
|
||||
* distinct.
|
||||
* @return A {@code TestSpeedProvider}.
|
||||
*/
|
||||
public static TestSpeedProvider createWithStartTimes(long[] startTimesUs, float[] speeds) {
|
||||
return new TestSpeedProvider(startTimesUs, speeds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@code TestSpeedProvider} instance.
|
||||
*
|
||||
* @param frameCounts The frame counts for which the same speed should be applied.
|
||||
* @param speeds The speeds corresponding to each frame count. The values must be distinct.
|
||||
* @return A {@code TestSpeedProvider}.
|
||||
*/
|
||||
public static TestSpeedProvider createWithFrameCounts(int[] frameCounts, float[] speeds) {
|
||||
long[] startTimesUs = new long[frameCounts.length];
|
||||
int totalFrameCount = 0;
|
||||
for (int i = 0; i < frameCounts.length; i++) {
|
||||
startTimesUs[i] = totalFrameCount * C.MICROS_PER_SECOND / AUDIO_FORMAT.sampleRate;
|
||||
totalFrameCount += frameCounts[i];
|
||||
}
|
||||
return new TestSpeedProvider(startTimesUs, speeds);
|
||||
}
|
||||
|
||||
private TestSpeedProvider(long[] startTimesUs, float[] speeds) {
|
||||
checkArgument(startTimesUs.length == speeds.length);
|
||||
this.startTimesUs = startTimesUs;
|
||||
this.speeds = speeds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getSpeed(long timeUs) {
|
||||
int index =
|
||||
Util.binarySearchFloor(
|
||||
startTimesUs, timeUs, /* inclusive= */ true, /* stayInBounds= */ true);
|
||||
return speeds[index];
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNextSpeedChangeTimeUs(long timeUs) {
|
||||
int index =
|
||||
Util.binarySearchCeil(
|
||||
startTimesUs, timeUs, /* inclusive= */ false, /* stayInBounds= */ false);
|
||||
return index < startTimesUs.length ? startTimesUs[index] : C.TIME_UNSET;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,12 +16,19 @@
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
|
||||
import static androidx.media3.test.utils.VideoFrameProcessorTestRunner.createTimestampIterator;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
import android.util.Pair;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.Effect;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.media3.common.audio.SpeedProvider;
|
||||
import androidx.media3.test.utils.TestSpeedProvider;
|
||||
import androidx.media3.test.utils.VideoFrameProcessorTestRunner;
|
||||
import androidx.media3.test.utils.VideoFrameProcessorTestRunner.OnOutputFrameAvailableForRenderingListener;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.util.ArrayList;
|
||||
@ -41,29 +48,121 @@ public class SpeedChangeEffectTest {
|
||||
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
|
||||
|
||||
@Test
|
||||
public void changeSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception {
|
||||
String testId = testName.getMethodName();
|
||||
VideoFrameProcessor.Factory videoFrameProcessorFactory =
|
||||
new DefaultVideoFrameProcessor.Factory.Builder().build();
|
||||
ImmutableList<Effect> effects = ImmutableList.of(new SpeedChangeEffect(2f));
|
||||
public void increaseSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception {
|
||||
List<Long> outputPresentationTimesUs = new ArrayList<>();
|
||||
VideoFrameProcessorTestRunner.OnOutputFrameAvailableForRenderingListener
|
||||
onOutputFrameAvailableForRenderingListener = outputPresentationTimesUs::add;
|
||||
VideoFrameProcessorTestRunner videoFrameProcessorTestRunner =
|
||||
new VideoFrameProcessorTestRunner.Builder()
|
||||
.setTestId(testId)
|
||||
.setVideoFrameProcessorFactory(videoFrameProcessorFactory)
|
||||
.setEffects(effects)
|
||||
.setOnOutputFrameAvailableForRenderingListener(
|
||||
onOutputFrameAvailableForRenderingListener)
|
||||
.build();
|
||||
getVideoFrameProcessorTestRunner(
|
||||
testName.getMethodName(), new SpeedChangeEffect(2), outputPresentationTimesUs::add);
|
||||
|
||||
videoFrameProcessorTestRunner.queueInputBitmap(
|
||||
readBitmap(IMAGE_PATH), C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 5);
|
||||
readBitmap(IMAGE_PATH),
|
||||
/* durationUs= */ C.MICROS_PER_SECOND,
|
||||
/* offsetToAddUs= */ 0L,
|
||||
/* frameRate= */ 5);
|
||||
videoFrameProcessorTestRunner.endFrameProcessing();
|
||||
|
||||
assertThat(outputPresentationTimesUs)
|
||||
.containsExactly(0L, 100_000L, 200_000L, 300_000L, 400_000L)
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void decreaseSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception {
|
||||
List<Long> outputPresentationTimesUs = new ArrayList<>();
|
||||
VideoFrameProcessorTestRunner videoFrameProcessorTestRunner =
|
||||
getVideoFrameProcessorTestRunner(
|
||||
testName.getMethodName(), new SpeedChangeEffect(0.5f), outputPresentationTimesUs::add);
|
||||
|
||||
videoFrameProcessorTestRunner.queueInputBitmap(
|
||||
readBitmap(IMAGE_PATH),
|
||||
/* durationUs= */ C.MICROS_PER_SECOND,
|
||||
/* offsetToAddUs= */ 0L,
|
||||
/* frameRate= */ 5);
|
||||
videoFrameProcessorTestRunner.endFrameProcessing();
|
||||
|
||||
assertThat(outputPresentationTimesUs)
|
||||
.containsExactly(0L, 400_000L, 800_000L, 1_200_000L, 1_600_000L)
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void variableSpeedChange_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithStartTimes(
|
||||
/* startTimesUs= */ new long[] {0, 1_500_000, 3_000_000},
|
||||
/* speeds= */ new float[] {1, 2, 1});
|
||||
List<Long> outputPresentationTimesUs = new ArrayList<>();
|
||||
VideoFrameProcessorTestRunner videoFrameProcessorTestRunner =
|
||||
getVideoFrameProcessorTestRunner(
|
||||
testName.getMethodName(),
|
||||
new SpeedChangeEffect(speedProvider),
|
||||
outputPresentationTimesUs::add);
|
||||
|
||||
videoFrameProcessorTestRunner.queueInputBitmap(
|
||||
readBitmap(IMAGE_PATH),
|
||||
/* durationUs= */ 5_000_000L,
|
||||
/* offsetToAddUs= */ 0L,
|
||||
/* frameRate= */ 1);
|
||||
videoFrameProcessorTestRunner.queueInputBitmap(
|
||||
readBitmap(IMAGE_PATH),
|
||||
/* durationUs= */ 5_000_000L,
|
||||
/* offsetToAddUs= */ 5_000_000L,
|
||||
/* frameRate= */ 1);
|
||||
videoFrameProcessorTestRunner.endFrameProcessing();
|
||||
|
||||
ImmutableList<Long> firstStreamExpectedTimestamps =
|
||||
ImmutableList.of(0L, 1_000_000L, 1_750_000L, 2_250_000L, 3_250_000L);
|
||||
ImmutableList<Long> secondStreamExpectedTimestamps =
|
||||
ImmutableList.of(5_000_000L, 6_000_000L, 6_750_000L, 7_250_000L, 8_250_000L);
|
||||
ImmutableList<Long> allExpectedTimestamps =
|
||||
new ImmutableList.Builder<Long>()
|
||||
.addAll(firstStreamExpectedTimestamps)
|
||||
.addAll(secondStreamExpectedTimestamps)
|
||||
.build();
|
||||
assertThat(outputPresentationTimesUs)
|
||||
.containsExactlyElementsIn(allExpectedTimestamps)
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void
|
||||
variableSpeedChange_multipleSpeedChangesBetweenFrames_outputsFramesAtTheCorrectPresentationTimesUs()
|
||||
throws Exception {
|
||||
SpeedProvider speedProvider =
|
||||
TestSpeedProvider.createWithStartTimes(
|
||||
/* startTimesUs= */ new long[] {0, 1_000_000, 2_000_000},
|
||||
/* speeds= */ new float[] {4, 2, 1});
|
||||
List<Long> outputPresentationTimesUs = new ArrayList<>();
|
||||
VideoFrameProcessorTestRunner videoFrameProcessorTestRunner =
|
||||
getVideoFrameProcessorTestRunner(
|
||||
testName.getMethodName(),
|
||||
new SpeedChangeEffect(speedProvider),
|
||||
outputPresentationTimesUs::add);
|
||||
Bitmap bitmap = readBitmap(IMAGE_PATH);
|
||||
ImmutableList<Long> inputTimestamps = ImmutableList.of(0L, 4_000_000L, 5_000_000L);
|
||||
|
||||
videoFrameProcessorTestRunner.queueInputBitmaps(
|
||||
bitmap.getWidth(),
|
||||
bitmap.getHeight(),
|
||||
Pair.create(bitmap, createTimestampIterator(inputTimestamps)));
|
||||
videoFrameProcessorTestRunner.endFrameProcessing();
|
||||
|
||||
assertThat(outputPresentationTimesUs).containsExactly(0L, 2_750_000L, 3_750_000L).inOrder();
|
||||
}
|
||||
|
||||
private static VideoFrameProcessorTestRunner getVideoFrameProcessorTestRunner(
|
||||
String testId,
|
||||
GlEffect speedChangeEffect,
|
||||
OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableForRenderingListener)
|
||||
throws VideoFrameProcessingException {
|
||||
VideoFrameProcessor.Factory videoFrameProcessorFactory =
|
||||
new DefaultVideoFrameProcessor.Factory.Builder().build();
|
||||
ImmutableList<Effect> effects = ImmutableList.of(speedChangeEffect);
|
||||
return new VideoFrameProcessorTestRunner.Builder()
|
||||
.setTestId(testId)
|
||||
.setVideoFrameProcessorFactory(videoFrameProcessorFactory)
|
||||
.setEffects(effects)
|
||||
.setOnOutputFrameAvailableForRenderingListener(onOutputFrameAvailableForRenderingListener)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,33 +19,57 @@ import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
|
||||
import android.content.Context;
|
||||
import androidx.annotation.FloatRange;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.audio.SpeedProvider;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
|
||||
/**
|
||||
* Applies a speed change by updating the frame timestamps.
|
||||
*
|
||||
* <p>This effect doesn't drop any frames.
|
||||
*
|
||||
* <p>This effect is not supported for effects previewing.
|
||||
*/
|
||||
@UnstableApi
|
||||
public final class SpeedChangeEffect implements GlEffect {
|
||||
|
||||
private final float speed;
|
||||
private final SpeedProvider speedProvider;
|
||||
|
||||
/** Creates an instance. */
|
||||
/** Creates an instance that applies the same {@code speed} change to all the timestamps. */
|
||||
public SpeedChangeEffect(@FloatRange(from = 0, fromInclusive = false) float speed) {
|
||||
checkArgument(speed > 0f);
|
||||
this.speed = speed;
|
||||
speedProvider =
|
||||
new SpeedProvider() {
|
||||
@Override
|
||||
public float getSpeed(long timeUs) {
|
||||
return speed;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNextSpeedChangeTimeUs(long timeUs) {
|
||||
return C.TIME_UNSET;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an instance.
|
||||
*
|
||||
* @param speedProvider The {@link SpeedProvider} specifying the speed changes. Applied on each
|
||||
* stream assuming the first frame timestamp of the input media is 0.
|
||||
*/
|
||||
public SpeedChangeEffect(SpeedProvider speedProvider) {
|
||||
this.speedProvider = speedProvider;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr)
|
||||
throws VideoFrameProcessingException {
|
||||
return new SpeedChangeShaderProgram(speed);
|
||||
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
|
||||
return new SpeedChangeShaderProgram(speedProvider);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isNoOp(int inputWidth, int inputHeight) {
|
||||
return speed == 1f;
|
||||
return speedProvider.getSpeed(/* timeUs= */ 0) == 1
|
||||
&& speedProvider.getNextSpeedChangeTimeUs(/* timeUs= */ 0) == C.TIME_UNSET;
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,24 +15,95 @@
|
||||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.GlObjectsProvider;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.audio.SpeedProvider;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
|
||||
/** Applies a speed change by updating the frame timestamps. */
|
||||
/**
|
||||
* Applies the speed changes specified in a {@link SpeedProvider} change by updating the frame
|
||||
* timestamps.
|
||||
*
|
||||
* <p>Does not support seeking in effects previewing.
|
||||
*/
|
||||
@UnstableApi
|
||||
/* package */ final class SpeedChangeShaderProgram extends PassthroughShaderProgram {
|
||||
|
||||
private final float speed;
|
||||
private final OffsetSpeedProvider speedProvider;
|
||||
|
||||
public SpeedChangeShaderProgram(float speed) {
|
||||
private long lastSpeedChangeInputTimeUs;
|
||||
private long lastSpeedChangeOutputTimeUs;
|
||||
|
||||
public SpeedChangeShaderProgram(SpeedProvider speedProvider) {
|
||||
super();
|
||||
this.speed = speed;
|
||||
this.speedProvider = new OffsetSpeedProvider(speedProvider);
|
||||
lastSpeedChangeInputTimeUs = C.TIME_UNSET;
|
||||
lastSpeedChangeOutputTimeUs = C.TIME_UNSET;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void queueInputFrame(
|
||||
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
super.queueInputFrame(glObjectsProvider, inputTexture, (long) (presentationTimeUs / speed));
|
||||
long outputPresentationTimeUs;
|
||||
if (lastSpeedChangeInputTimeUs == C.TIME_UNSET) {
|
||||
outputPresentationTimeUs = presentationTimeUs;
|
||||
lastSpeedChangeInputTimeUs = presentationTimeUs;
|
||||
lastSpeedChangeOutputTimeUs = outputPresentationTimeUs;
|
||||
speedProvider.setOffset(presentationTimeUs);
|
||||
} else {
|
||||
long nextSpeedChangeInputTimeUs =
|
||||
speedProvider.getNextSpeedChangeTimeUs(lastSpeedChangeInputTimeUs);
|
||||
while (nextSpeedChangeInputTimeUs != C.TIME_UNSET
|
||||
&& nextSpeedChangeInputTimeUs <= presentationTimeUs) {
|
||||
lastSpeedChangeOutputTimeUs =
|
||||
getOutputTimeUs(
|
||||
nextSpeedChangeInputTimeUs, speedProvider.getSpeed(lastSpeedChangeInputTimeUs));
|
||||
lastSpeedChangeInputTimeUs = nextSpeedChangeInputTimeUs;
|
||||
nextSpeedChangeInputTimeUs =
|
||||
speedProvider.getNextSpeedChangeTimeUs(lastSpeedChangeInputTimeUs);
|
||||
}
|
||||
outputPresentationTimeUs =
|
||||
getOutputTimeUs(presentationTimeUs, speedProvider.getSpeed(presentationTimeUs));
|
||||
}
|
||||
super.queueInputFrame(glObjectsProvider, inputTexture, outputPresentationTimeUs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void signalEndOfCurrentInputStream() {
|
||||
super.signalEndOfCurrentInputStream();
|
||||
lastSpeedChangeInputTimeUs = C.TIME_UNSET;
|
||||
lastSpeedChangeOutputTimeUs = C.TIME_UNSET;
|
||||
}
|
||||
|
||||
private long getOutputTimeUs(long inputTimeUs, float speed) {
|
||||
return (long)
|
||||
(lastSpeedChangeOutputTimeUs + (inputTimeUs - lastSpeedChangeInputTimeUs) / speed);
|
||||
}
|
||||
|
||||
private static class OffsetSpeedProvider implements SpeedProvider {
|
||||
|
||||
private final SpeedProvider speedProvider;
|
||||
|
||||
private long offset;
|
||||
|
||||
public OffsetSpeedProvider(SpeedProvider speedProvider) {
|
||||
this.speedProvider = speedProvider;
|
||||
}
|
||||
|
||||
public void setOffset(long offset) {
|
||||
this.offset = offset;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getSpeed(long timeUs) {
|
||||
return speedProvider.getSpeed(timeUs - offset);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNextSpeedChangeTimeUs(long timeUs) {
|
||||
long nextSpeedChangeTimeUs = speedProvider.getNextSpeedChangeTimeUs(timeUs - offset);
|
||||
return nextSpeedChangeTimeUs == C.TIME_UNSET ? C.TIME_UNSET : offset + nextSpeedChangeTimeUs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright 2024 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.test.utils;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.audio.AudioProcessor.AudioFormat;
|
||||
import androidx.media3.common.audio.SpeedProvider;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import androidx.media3.common.util.Util;
|
||||
|
||||
/** {@link SpeedProvider} for tests */
|
||||
@UnstableApi
|
||||
public final class TestSpeedProvider implements SpeedProvider {
|
||||
|
||||
private final long[] startTimesUs;
|
||||
private final float[] speeds;
|
||||
|
||||
/**
|
||||
* Creates a {@code TestSpeedProvider} instance.
|
||||
*
|
||||
* @param startTimesUs The speed change start times, in microseconds. The values must be distinct
|
||||
* and in increasing order.
|
||||
* @param speeds The speeds corresponding to each start time. Consecutive values must be distinct.
|
||||
* @return A {@code TestSpeedProvider}.
|
||||
*/
|
||||
public static TestSpeedProvider createWithStartTimes(long[] startTimesUs, float[] speeds) {
|
||||
return new TestSpeedProvider(startTimesUs, speeds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@code TestSpeedProvider} instance.
|
||||
*
|
||||
* @param audioFormat the {@link AudioFormat}.
|
||||
* @param frameCounts The frame counts for which the same speed should be applied.
|
||||
* @param speeds The speeds corresponding to each frame count. Consecutive values must be
|
||||
* distinct.
|
||||
* @return A {@code TestSpeedProvider}.
|
||||
*/
|
||||
public static TestSpeedProvider createWithFrameCounts(
|
||||
AudioFormat audioFormat, int[] frameCounts, float[] speeds) {
|
||||
long[] startTimesUs = new long[frameCounts.length];
|
||||
int totalFrameCount = 0;
|
||||
for (int i = 0; i < frameCounts.length; i++) {
|
||||
startTimesUs[i] = totalFrameCount * C.MICROS_PER_SECOND / audioFormat.sampleRate;
|
||||
totalFrameCount += frameCounts[i];
|
||||
}
|
||||
return new TestSpeedProvider(startTimesUs, speeds);
|
||||
}
|
||||
|
||||
private TestSpeedProvider(long[] startTimesUs, float[] speeds) {
|
||||
checkArgument(startTimesUs.length == speeds.length);
|
||||
this.startTimesUs = startTimesUs;
|
||||
this.speeds = speeds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getSpeed(long timeUs) {
|
||||
int index =
|
||||
Util.binarySearchFloor(
|
||||
startTimesUs, timeUs, /* inclusive= */ true, /* stayInBounds= */ true);
|
||||
return speeds[index];
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNextSpeedChangeTimeUs(long timeUs) {
|
||||
int index =
|
||||
Util.binarySearchCeil(
|
||||
startTimesUs, timeUs, /* inclusive= */ false, /* stayInBounds= */ false);
|
||||
return index < startTimesUs.length ? startTimesUs[index] : C.TIME_UNSET;
|
||||
}
|
||||
}
|
||||
@ -342,6 +342,8 @@ public final class Composition {
|
||||
/** The {@link VideoCompositorSettings} to apply to the composition. */
|
||||
public final VideoCompositorSettings videoCompositorSettings;
|
||||
|
||||
// TODO: b/302695659 - Ensure composition level effects are only applied consistently between the
|
||||
// different VideoGraphs.
|
||||
/** The {@link Effects} to apply to the composition. */
|
||||
public final Effects effects;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user