loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support segment-specific speed changes in SpeedChangeEffect

Browse Source 
PiperOrigin-RevId: 606204479
This commit is contained in:
tofunmi 2024-02-12 04:00:16 -08:00 committed by Copybara-Service
parent 0b0c419c73
commit 1a5eb4eecd
7 changed files with 331 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -18,6 +18,8 @@
* DRM:
* Effect:
* Improved PQ to SDR tone-mapping by converting color spaces.
* Support multiple speed changes within the same `EditedMediaItem` or
`Composition` in `SpeedChangeEffect`.
* Muxers:
* IMA extension:
* Session:

97
libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java
View File

@ -16,11 +16,10 @@
package androidx.media3.common.audio;
import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER;
import static androidx.media3.common.util.Assertions.checkArgument;
import static com.google.common.truth.Truth.assertThat;
import androidx.media3.common.C;
import androidx.media3.common.util.Util;
import androidx.media3.test.utils.TestSpeedProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@ -39,7 +38,7 @@ public class SpeedChangingAudioProcessorTest {
public void queueInput_noSpeedChange_doesNotOverwriteInput() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -54,7 +53,7 @@ public class SpeedChangingAudioProcessorTest {
public void queueInput_speedChange_doesNotOverwriteInput() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -69,7 +68,7 @@ public class SpeedChangingAudioProcessorTest {
public void queueInput_noSpeedChange_copiesSamples() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -86,7 +85,7 @@ public class SpeedChangingAudioProcessorTest {
public void queueInput_speedChange_modifiesSamples() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -104,7 +103,7 @@ public class SpeedChangingAudioProcessorTest {
public void queueInput_noSpeedChangeAfterSpeedChange_copiesSamples() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -124,7 +123,7 @@ public class SpeedChangingAudioProcessorTest {
throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -137,7 +136,7 @@ public class SpeedChangingAudioProcessorTest {
speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider);
inputBuffer.rewind();
speedChangingAudioProcessor.queueInput(inputBuffer);
@ -152,7 +151,7 @@ public class SpeedChangingAudioProcessorTest {
throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {3, 2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {3, 2});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -165,7 +164,7 @@ public class SpeedChangingAudioProcessorTest {
speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider);
inputBuffer.rewind();
speedChangingAudioProcessor.queueInput(inputBuffer);
@ -180,7 +179,7 @@ public class SpeedChangingAudioProcessorTest {
throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 3});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 3});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -190,7 +189,7 @@ public class SpeedChangingAudioProcessorTest {
speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider);
inputBuffer.rewind();
speedChangingAudioProcessor.queueInput(inputBuffer);
@ -243,7 +242,7 @@ public class SpeedChangingAudioProcessorTest {
throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -262,7 +261,7 @@ public class SpeedChangingAudioProcessorTest {
throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -281,7 +280,7 @@ public class SpeedChangingAudioProcessorTest {
throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -297,7 +296,7 @@ public class SpeedChangingAudioProcessorTest {
throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -312,7 +311,7 @@ public class SpeedChangingAudioProcessorTest {
public void queueEndOfStream_noInputQueued_endsProcessor() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
@ -325,7 +324,7 @@ public class SpeedChangingAudioProcessorTest {
public void isEnded_afterNoSpeedChangeAndOutputRetrieved_isFalse() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -340,7 +339,7 @@ public class SpeedChangingAudioProcessorTest {
public void isEnded_afterSpeedChangeAndOutputRetrieved_isFalse() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithFrameCounts(
/* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2});
SpeedChangingAudioProcessor speedChangingAudioProcessor =
getConfiguredSpeedChangingAudioProcessor(speedProvider);
ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5);
@ -388,62 +387,4 @@ public class SpeedChangingAudioProcessorTest {
}
return concatenatedOutputBuffers;
}
private static final class TestSpeedProvider implements SpeedProvider {
private final long[] startTimesUs;
private final float[] speeds;
/**
* Creates a {@code TestSpeedProvider} instance.
*
* @param startTimesUs The speed change start times, in microseconds. The values must be
* distinct and in increasing order.
* @param speeds The speeds corresponding to each start time. Consecutive values must be
* distinct.
* @return A {@code TestSpeedProvider}.
*/
public static TestSpeedProvider createWithStartTimes(long[] startTimesUs, float[] speeds) {
return new TestSpeedProvider(startTimesUs, speeds);
}
/**
* Creates a {@code TestSpeedProvider} instance.
*
* @param frameCounts The frame counts for which the same speed should be applied.
* @param speeds The speeds corresponding to each frame count. The values must be distinct.
* @return A {@code TestSpeedProvider}.
*/
public static TestSpeedProvider createWithFrameCounts(int[] frameCounts, float[] speeds) {
long[] startTimesUs = new long[frameCounts.length];
int totalFrameCount = 0;
for (int i = 0; i < frameCounts.length; i++) {
startTimesUs[i] = totalFrameCount * C.MICROS_PER_SECOND / AUDIO_FORMAT.sampleRate;
totalFrameCount += frameCounts[i];
}
return new TestSpeedProvider(startTimesUs, speeds);
}
private TestSpeedProvider(long[] startTimesUs, float[] speeds) {
checkArgument(startTimesUs.length == speeds.length);
this.startTimesUs = startTimesUs;
this.speeds = speeds;
}
@Override
public float getSpeed(long timeUs) {
int index =
Util.binarySearchFloor(
startTimesUs, timeUs, /* inclusive= */ true, /* stayInBounds= */ true);
return speeds[index];
}
@Override
public long getNextSpeedChangeTimeUs(long timeUs) {
int index =
Util.binarySearchCeil(
startTimesUs, timeUs, /* inclusive= */ false, /* stayInBounds= */ false);
return index < startTimesUs.length ? startTimesUs[index] : C.TIME_UNSET;
}
}
}

129
libraries/effect/src/androidTest/java/androidx/media3/effect/SpeedChangeEffectTest.java
View File

@ -16,12 +16,19 @@
package androidx.media3.effect;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
import static androidx.media3.test.utils.VideoFrameProcessorTestRunner.createTimestampIterator;
import static com.google.common.truth.Truth.assertThat;
import android.graphics.Bitmap;
import android.util.Pair;
import androidx.media3.common.C;
import androidx.media3.common.Effect;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.audio.SpeedProvider;
import androidx.media3.test.utils.TestSpeedProvider;
import androidx.media3.test.utils.VideoFrameProcessorTestRunner;
import androidx.media3.test.utils.VideoFrameProcessorTestRunner.OnOutputFrameAvailableForRenderingListener;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
@ -41,29 +48,121 @@ public class SpeedChangeEffectTest {
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
@Test
public void changeSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception {
String testId = testName.getMethodName();
VideoFrameProcessor.Factory videoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder().build();
ImmutableList<Effect> effects = ImmutableList.of(new SpeedChangeEffect(2f));
public void increaseSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception {
List<Long> outputPresentationTimesUs = new ArrayList<>();
VideoFrameProcessorTestRunner.OnOutputFrameAvailableForRenderingListener
onOutputFrameAvailableForRenderingListener = outputPresentationTimesUs::add;
VideoFrameProcessorTestRunner videoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(videoFrameProcessorFactory)
.setEffects(effects)
.setOnOutputFrameAvailableForRenderingListener(
onOutputFrameAvailableForRenderingListener)
.build();
getVideoFrameProcessorTestRunner(
testName.getMethodName(), new SpeedChangeEffect(2), outputPresentationTimesUs::add);
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(IMAGE_PATH), C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 5);
readBitmap(IMAGE_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 5);
videoFrameProcessorTestRunner.endFrameProcessing();
assertThat(outputPresentationTimesUs)
.containsExactly(0L, 100_000L, 200_000L, 300_000L, 400_000L)
.inOrder();
}
@Test
public void decreaseSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception {
List<Long> outputPresentationTimesUs = new ArrayList<>();
VideoFrameProcessorTestRunner videoFrameProcessorTestRunner =
getVideoFrameProcessorTestRunner(
testName.getMethodName(), new SpeedChangeEffect(0.5f), outputPresentationTimesUs::add);
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(IMAGE_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 5);
videoFrameProcessorTestRunner.endFrameProcessing();
assertThat(outputPresentationTimesUs)
.containsExactly(0L, 400_000L, 800_000L, 1_200_000L, 1_600_000L)
.inOrder();
}
@Test
public void variableSpeedChange_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithStartTimes(
/* startTimesUs= */ new long[] {0, 1_500_000, 3_000_000},
/* speeds= */ new float[] {1, 2, 1});
List<Long> outputPresentationTimesUs = new ArrayList<>();
VideoFrameProcessorTestRunner videoFrameProcessorTestRunner =
getVideoFrameProcessorTestRunner(
testName.getMethodName(),
new SpeedChangeEffect(speedProvider),
outputPresentationTimesUs::add);
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(IMAGE_PATH),
/* durationUs= */ 5_000_000L,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 1);
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(IMAGE_PATH),
/* durationUs= */ 5_000_000L,
/* offsetToAddUs= */ 5_000_000L,
/* frameRate= */ 1);
videoFrameProcessorTestRunner.endFrameProcessing();
ImmutableList<Long> firstStreamExpectedTimestamps =
ImmutableList.of(0L, 1_000_000L, 1_750_000L, 2_250_000L, 3_250_000L);
ImmutableList<Long> secondStreamExpectedTimestamps =
ImmutableList.of(5_000_000L, 6_000_000L, 6_750_000L, 7_250_000L, 8_250_000L);
ImmutableList<Long> allExpectedTimestamps =
new ImmutableList.Builder<Long>()
.addAll(firstStreamExpectedTimestamps)
.addAll(secondStreamExpectedTimestamps)
.build();
assertThat(outputPresentationTimesUs)
.containsExactlyElementsIn(allExpectedTimestamps)
.inOrder();
}
@Test
public void
variableSpeedChange_multipleSpeedChangesBetweenFrames_outputsFramesAtTheCorrectPresentationTimesUs()
throws Exception {
SpeedProvider speedProvider =
TestSpeedProvider.createWithStartTimes(
/* startTimesUs= */ new long[] {0, 1_000_000, 2_000_000},
/* speeds= */ new float[] {4, 2, 1});
List<Long> outputPresentationTimesUs = new ArrayList<>();
VideoFrameProcessorTestRunner videoFrameProcessorTestRunner =
getVideoFrameProcessorTestRunner(
testName.getMethodName(),
new SpeedChangeEffect(speedProvider),
outputPresentationTimesUs::add);
Bitmap bitmap = readBitmap(IMAGE_PATH);
ImmutableList<Long> inputTimestamps = ImmutableList.of(0L, 4_000_000L, 5_000_000L);
videoFrameProcessorTestRunner.queueInputBitmaps(
bitmap.getWidth(),
bitmap.getHeight(),
Pair.create(bitmap, createTimestampIterator(inputTimestamps)));
videoFrameProcessorTestRunner.endFrameProcessing();
assertThat(outputPresentationTimesUs).containsExactly(0L, 2_750_000L, 3_750_000L).inOrder();
}
private static VideoFrameProcessorTestRunner getVideoFrameProcessorTestRunner(
String testId,
GlEffect speedChangeEffect,
OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableForRenderingListener)
throws VideoFrameProcessingException {
VideoFrameProcessor.Factory videoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder().build();
ImmutableList<Effect> effects = ImmutableList.of(speedChangeEffect);
return new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(videoFrameProcessorFactory)
.setEffects(effects)
.setOnOutputFrameAvailableForRenderingListener(onOutputFrameAvailableForRenderingListener)
.build();
}
}

40
libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeEffect.java
View File

@ -19,33 +19,57 @@ import static androidx.media3.common.util.Assertions.checkArgument;
import android.content.Context;
import androidx.annotation.FloatRange;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.C;
import androidx.media3.common.audio.SpeedProvider;
import androidx.media3.common.util.UnstableApi;
/**
* Applies a speed change by updating the frame timestamps.
*
* <p>This effect doesn't drop any frames.
*
* <p>This effect is not supported for effects previewing.
*/
@UnstableApi
public final class SpeedChangeEffect implements GlEffect {
private final float speed;
private final SpeedProvider speedProvider;
/** Creates an instance. */
/** Creates an instance that applies the same {@code speed} change to all the timestamps. */
public SpeedChangeEffect(@FloatRange(from = 0, fromInclusive = false) float speed) {
checkArgument(speed > 0f);
this.speed = speed;
speedProvider =
new SpeedProvider() {
@Override
public float getSpeed(long timeUs) {
return speed;
}
@Override
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr)
throws VideoFrameProcessingException {
return new SpeedChangeShaderProgram(speed);
public long getNextSpeedChangeTimeUs(long timeUs) {
return C.TIME_UNSET;
}
};
}
/**
* Creates an instance.
*
* @param speedProvider The {@link SpeedProvider} specifying the speed changes. Applied on each
* stream assuming the first frame timestamp of the input media is 0.
*/
public SpeedChangeEffect(SpeedProvider speedProvider) {
this.speedProvider = speedProvider;
}
@Override
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
return new SpeedChangeShaderProgram(speedProvider);
}
@Override
public boolean isNoOp(int inputWidth, int inputHeight) {
return speed == 1f;
return speedProvider.getSpeed(/* timeUs= */ 0) == 1
&& speedProvider.getNextSpeedChangeTimeUs(/* timeUs= */ 0) == C.TIME_UNSET;
}
}

81
libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeShaderProgram.java
View File

@ -15,24 +15,95 @@
*/
package androidx.media3.effect;
import androidx.media3.common.C;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.audio.SpeedProvider;
import androidx.media3.common.util.UnstableApi;
/** Applies a speed change by updating the frame timestamps. */
/**
* Applies the speed changes specified in a {@link SpeedProvider} change by updating the frame
* timestamps.
*
* <p>Does not support seeking in effects previewing.
*/
@UnstableApi
/* package */ final class SpeedChangeShaderProgram extends PassthroughShaderProgram {
private final float speed;
private final OffsetSpeedProvider speedProvider;
public SpeedChangeShaderProgram(float speed) {
private long lastSpeedChangeInputTimeUs;
private long lastSpeedChangeOutputTimeUs;
public SpeedChangeShaderProgram(SpeedProvider speedProvider) {
super();
this.speed = speed;
this.speedProvider = new OffsetSpeedProvider(speedProvider);
lastSpeedChangeInputTimeUs = C.TIME_UNSET;
lastSpeedChangeOutputTimeUs = C.TIME_UNSET;
}
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
super.queueInputFrame(glObjectsProvider, inputTexture, (long) (presentationTimeUs / speed));
long outputPresentationTimeUs;
if (lastSpeedChangeInputTimeUs == C.TIME_UNSET) {
outputPresentationTimeUs = presentationTimeUs;
lastSpeedChangeInputTimeUs = presentationTimeUs;
lastSpeedChangeOutputTimeUs = outputPresentationTimeUs;
speedProvider.setOffset(presentationTimeUs);
} else {
long nextSpeedChangeInputTimeUs =
speedProvider.getNextSpeedChangeTimeUs(lastSpeedChangeInputTimeUs);
while (nextSpeedChangeInputTimeUs != C.TIME_UNSET
&& nextSpeedChangeInputTimeUs <= presentationTimeUs) {
lastSpeedChangeOutputTimeUs =
getOutputTimeUs(
nextSpeedChangeInputTimeUs, speedProvider.getSpeed(lastSpeedChangeInputTimeUs));
lastSpeedChangeInputTimeUs = nextSpeedChangeInputTimeUs;
nextSpeedChangeInputTimeUs =
speedProvider.getNextSpeedChangeTimeUs(lastSpeedChangeInputTimeUs);
}
outputPresentationTimeUs =
getOutputTimeUs(presentationTimeUs, speedProvider.getSpeed(presentationTimeUs));
}
super.queueInputFrame(glObjectsProvider, inputTexture, outputPresentationTimeUs);
}
@Override
public void signalEndOfCurrentInputStream() {
super.signalEndOfCurrentInputStream();
lastSpeedChangeInputTimeUs = C.TIME_UNSET;
lastSpeedChangeOutputTimeUs = C.TIME_UNSET;
}
private long getOutputTimeUs(long inputTimeUs, float speed) {
return (long)
(lastSpeedChangeOutputTimeUs + (inputTimeUs - lastSpeedChangeInputTimeUs) / speed);
}
private static class OffsetSpeedProvider implements SpeedProvider {
private final SpeedProvider speedProvider;
private long offset;
public OffsetSpeedProvider(SpeedProvider speedProvider) {
this.speedProvider = speedProvider;
}
public void setOffset(long offset) {
this.offset = offset;
}
@Override
public float getSpeed(long timeUs) {
return speedProvider.getSpeed(timeUs - offset);
}
@Override
public long getNextSpeedChangeTimeUs(long timeUs) {
long nextSpeedChangeTimeUs = speedProvider.getNextSpeedChangeTimeUs(timeUs - offset);
return nextSpeedChangeTimeUs == C.TIME_UNSET ? C.TIME_UNSET : offset + nextSpeedChangeTimeUs;
}
}
}

86
libraries/test_utils/src/main/java/androidx/media3/test/utils/TestSpeedProvider.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Copyright 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.test.utils;
import static androidx.media3.common.util.Assertions.checkArgument;
import androidx.media3.common.C;
import androidx.media3.common.audio.AudioProcessor.AudioFormat;
import androidx.media3.common.audio.SpeedProvider;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
/** {@link SpeedProvider} for tests */
@UnstableApi
public final class TestSpeedProvider implements SpeedProvider {
private final long[] startTimesUs;
private final float[] speeds;
/**
* Creates a {@code TestSpeedProvider} instance.
*
* @param startTimesUs The speed change start times, in microseconds. The values must be distinct
* and in increasing order.
* @param speeds The speeds corresponding to each start time. Consecutive values must be distinct.
* @return A {@code TestSpeedProvider}.
*/
public static TestSpeedProvider createWithStartTimes(long[] startTimesUs, float[] speeds) {
return new TestSpeedProvider(startTimesUs, speeds);
}
/**
* Creates a {@code TestSpeedProvider} instance.
*
* @param audioFormat the {@link AudioFormat}.
* @param frameCounts The frame counts for which the same speed should be applied.
* @param speeds The speeds corresponding to each frame count. Consecutive values must be
* distinct.
* @return A {@code TestSpeedProvider}.
*/
public static TestSpeedProvider createWithFrameCounts(
AudioFormat audioFormat, int[] frameCounts, float[] speeds) {
long[] startTimesUs = new long[frameCounts.length];
int totalFrameCount = 0;
for (int i = 0; i < frameCounts.length; i++) {
startTimesUs[i] = totalFrameCount * C.MICROS_PER_SECOND / audioFormat.sampleRate;
totalFrameCount += frameCounts[i];
}
return new TestSpeedProvider(startTimesUs, speeds);
}
private TestSpeedProvider(long[] startTimesUs, float[] speeds) {
checkArgument(startTimesUs.length == speeds.length);
this.startTimesUs = startTimesUs;
this.speeds = speeds;
}
@Override
public float getSpeed(long timeUs) {
int index =
Util.binarySearchFloor(
startTimesUs, timeUs, /* inclusive= */ true, /* stayInBounds= */ true);
return speeds[index];
}
@Override
public long getNextSpeedChangeTimeUs(long timeUs) {
int index =
Util.binarySearchCeil(
startTimesUs, timeUs, /* inclusive= */ false, /* stayInBounds= */ false);
return index < startTimesUs.length ? startTimesUs[index] : C.TIME_UNSET;
}
}

2
libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java
View File

@ -342,6 +342,8 @@ public final class Composition {
/** The {@link VideoCompositorSettings} to apply to the composition. */
public final VideoCompositorSettings videoCompositorSettings;
// TODO: b/302695659 - Ensure composition level effects are only applied consistently between the
// different VideoGraphs.
/** The {@link Effects} to apply to the composition. */
public final Effects effects;