diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 82316fa984..2e0ed03fe3 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -18,6 +18,8 @@ * DRM: * Effect: * Improved PQ to SDR tone-mapping by converting color spaces. + * Support multiple speed changes within the same `EditedMediaItem` or + `Composition` in `SpeedChangeEffect`. * Muxers: * IMA extension: * Session: diff --git a/libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java b/libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java index 6e6343d652..9dc725cc9e 100644 --- a/libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/audio/SpeedChangingAudioProcessorTest.java @@ -16,11 +16,10 @@ package androidx.media3.common.audio; import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER; -import static androidx.media3.common.util.Assertions.checkArgument; import static com.google.common.truth.Truth.assertThat; import androidx.media3.common.C; -import androidx.media3.common.util.Util; +import androidx.media3.test.utils.TestSpeedProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -39,7 +38,7 @@ public class SpeedChangingAudioProcessorTest { public void queueInput_noSpeedChange_doesNotOverwriteInput() throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -54,7 +53,7 @@ public class SpeedChangingAudioProcessorTest { public void queueInput_speedChange_doesNotOverwriteInput() throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -69,7 +68,7 @@ public class SpeedChangingAudioProcessorTest { public void queueInput_noSpeedChange_copiesSamples() throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -86,7 +85,7 @@ public class SpeedChangingAudioProcessorTest { public void queueInput_speedChange_modifiesSamples() throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -104,7 +103,7 @@ public class SpeedChangingAudioProcessorTest { public void queueInput_noSpeedChangeAfterSpeedChange_copiesSamples() throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -124,7 +123,7 @@ public class SpeedChangingAudioProcessorTest { throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -137,7 +136,7 @@ public class SpeedChangingAudioProcessorTest { speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); inputBuffer.rewind(); speedChangingAudioProcessor.queueInput(inputBuffer); @@ -152,7 +151,7 @@ public class SpeedChangingAudioProcessorTest { throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {3, 2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {3, 2}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -165,7 +164,7 @@ public class SpeedChangingAudioProcessorTest { speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); inputBuffer.rewind(); speedChangingAudioProcessor.queueInput(inputBuffer); @@ -180,7 +179,7 @@ public class SpeedChangingAudioProcessorTest { throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 3}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 3}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -190,7 +189,7 @@ public class SpeedChangingAudioProcessorTest { speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); inputBuffer.rewind(); speedChangingAudioProcessor.queueInput(inputBuffer); @@ -243,7 +242,7 @@ public class SpeedChangingAudioProcessorTest { throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {2, 1}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -262,7 +261,7 @@ public class SpeedChangingAudioProcessorTest { throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5, 5}, /* speeds= */ new float[] {1, 2}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -281,7 +280,7 @@ public class SpeedChangingAudioProcessorTest { throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -297,7 +296,7 @@ public class SpeedChangingAudioProcessorTest { throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -312,7 +311,7 @@ public class SpeedChangingAudioProcessorTest { public void queueEndOfStream_noInputQueued_endsProcessor() throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); @@ -325,7 +324,7 @@ public class SpeedChangingAudioProcessorTest { public void isEnded_afterNoSpeedChangeAndOutputRetrieved_isFalse() throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {1}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -340,7 +339,7 @@ public class SpeedChangingAudioProcessorTest { public void isEnded_afterSpeedChangeAndOutputRetrieved_isFalse() throws Exception { SpeedProvider speedProvider = TestSpeedProvider.createWithFrameCounts( - /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); + AUDIO_FORMAT, /* frameCounts= */ new int[] {5}, /* speeds= */ new float[] {2}); SpeedChangingAudioProcessor speedChangingAudioProcessor = getConfiguredSpeedChangingAudioProcessor(speedProvider); ByteBuffer inputBuffer = getInputBuffer(/* frameCount= */ 5); @@ -388,62 +387,4 @@ public class SpeedChangingAudioProcessorTest { } return concatenatedOutputBuffers; } - - private static final class TestSpeedProvider implements SpeedProvider { - - private final long[] startTimesUs; - private final float[] speeds; - - /** - * Creates a {@code TestSpeedProvider} instance. - * - * @param startTimesUs The speed change start times, in microseconds. The values must be - * distinct and in increasing order. - * @param speeds The speeds corresponding to each start time. Consecutive values must be - * distinct. - * @return A {@code TestSpeedProvider}. - */ - public static TestSpeedProvider createWithStartTimes(long[] startTimesUs, float[] speeds) { - return new TestSpeedProvider(startTimesUs, speeds); - } - - /** - * Creates a {@code TestSpeedProvider} instance. - * - * @param frameCounts The frame counts for which the same speed should be applied. - * @param speeds The speeds corresponding to each frame count. The values must be distinct. - * @return A {@code TestSpeedProvider}. - */ - public static TestSpeedProvider createWithFrameCounts(int[] frameCounts, float[] speeds) { - long[] startTimesUs = new long[frameCounts.length]; - int totalFrameCount = 0; - for (int i = 0; i < frameCounts.length; i++) { - startTimesUs[i] = totalFrameCount * C.MICROS_PER_SECOND / AUDIO_FORMAT.sampleRate; - totalFrameCount += frameCounts[i]; - } - return new TestSpeedProvider(startTimesUs, speeds); - } - - private TestSpeedProvider(long[] startTimesUs, float[] speeds) { - checkArgument(startTimesUs.length == speeds.length); - this.startTimesUs = startTimesUs; - this.speeds = speeds; - } - - @Override - public float getSpeed(long timeUs) { - int index = - Util.binarySearchFloor( - startTimesUs, timeUs, /* inclusive= */ true, /* stayInBounds= */ true); - return speeds[index]; - } - - @Override - public long getNextSpeedChangeTimeUs(long timeUs) { - int index = - Util.binarySearchCeil( - startTimesUs, timeUs, /* inclusive= */ false, /* stayInBounds= */ false); - return index < startTimesUs.length ? startTimesUs[index] : C.TIME_UNSET; - } - } } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/SpeedChangeEffectTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/SpeedChangeEffectTest.java index 770c77ff05..8dc3cac885 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/SpeedChangeEffectTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/SpeedChangeEffectTest.java @@ -16,12 +16,19 @@ package androidx.media3.effect; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.media3.test.utils.VideoFrameProcessorTestRunner.createTimestampIterator; import static com.google.common.truth.Truth.assertThat; +import android.graphics.Bitmap; +import android.util.Pair; import androidx.media3.common.C; import androidx.media3.common.Effect; +import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.audio.SpeedProvider; +import androidx.media3.test.utils.TestSpeedProvider; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; +import androidx.media3.test.utils.VideoFrameProcessorTestRunner.OnOutputFrameAvailableForRenderingListener; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -41,29 +48,121 @@ public class SpeedChangeEffectTest { "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; @Test - public void changeSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { - String testId = testName.getMethodName(); - VideoFrameProcessor.Factory videoFrameProcessorFactory = - new DefaultVideoFrameProcessor.Factory.Builder().build(); - ImmutableList effects = ImmutableList.of(new SpeedChangeEffect(2f)); + public void increaseSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { List outputPresentationTimesUs = new ArrayList<>(); - VideoFrameProcessorTestRunner.OnOutputFrameAvailableForRenderingListener - onOutputFrameAvailableForRenderingListener = outputPresentationTimesUs::add; VideoFrameProcessorTestRunner videoFrameProcessorTestRunner = - new VideoFrameProcessorTestRunner.Builder() - .setTestId(testId) - .setVideoFrameProcessorFactory(videoFrameProcessorFactory) - .setEffects(effects) - .setOnOutputFrameAvailableForRenderingListener( - onOutputFrameAvailableForRenderingListener) - .build(); + getVideoFrameProcessorTestRunner( + testName.getMethodName(), new SpeedChangeEffect(2), outputPresentationTimesUs::add); videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(IMAGE_PATH), C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 5); + readBitmap(IMAGE_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 5); videoFrameProcessorTestRunner.endFrameProcessing(); assertThat(outputPresentationTimesUs) .containsExactly(0L, 100_000L, 200_000L, 300_000L, 400_000L) .inOrder(); } + + @Test + public void decreaseSpeed_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { + List outputPresentationTimesUs = new ArrayList<>(); + VideoFrameProcessorTestRunner videoFrameProcessorTestRunner = + getVideoFrameProcessorTestRunner( + testName.getMethodName(), new SpeedChangeEffect(0.5f), outputPresentationTimesUs::add); + + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(IMAGE_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 5); + videoFrameProcessorTestRunner.endFrameProcessing(); + + assertThat(outputPresentationTimesUs) + .containsExactly(0L, 400_000L, 800_000L, 1_200_000L, 1_600_000L) + .inOrder(); + } + + @Test + public void variableSpeedChange_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { + SpeedProvider speedProvider = + TestSpeedProvider.createWithStartTimes( + /* startTimesUs= */ new long[] {0, 1_500_000, 3_000_000}, + /* speeds= */ new float[] {1, 2, 1}); + List outputPresentationTimesUs = new ArrayList<>(); + VideoFrameProcessorTestRunner videoFrameProcessorTestRunner = + getVideoFrameProcessorTestRunner( + testName.getMethodName(), + new SpeedChangeEffect(speedProvider), + outputPresentationTimesUs::add); + + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(IMAGE_PATH), + /* durationUs= */ 5_000_000L, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 1); + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(IMAGE_PATH), + /* durationUs= */ 5_000_000L, + /* offsetToAddUs= */ 5_000_000L, + /* frameRate= */ 1); + videoFrameProcessorTestRunner.endFrameProcessing(); + + ImmutableList firstStreamExpectedTimestamps = + ImmutableList.of(0L, 1_000_000L, 1_750_000L, 2_250_000L, 3_250_000L); + ImmutableList secondStreamExpectedTimestamps = + ImmutableList.of(5_000_000L, 6_000_000L, 6_750_000L, 7_250_000L, 8_250_000L); + ImmutableList allExpectedTimestamps = + new ImmutableList.Builder() + .addAll(firstStreamExpectedTimestamps) + .addAll(secondStreamExpectedTimestamps) + .build(); + assertThat(outputPresentationTimesUs) + .containsExactlyElementsIn(allExpectedTimestamps) + .inOrder(); + } + + @Test + public void + variableSpeedChange_multipleSpeedChangesBetweenFrames_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + SpeedProvider speedProvider = + TestSpeedProvider.createWithStartTimes( + /* startTimesUs= */ new long[] {0, 1_000_000, 2_000_000}, + /* speeds= */ new float[] {4, 2, 1}); + List outputPresentationTimesUs = new ArrayList<>(); + VideoFrameProcessorTestRunner videoFrameProcessorTestRunner = + getVideoFrameProcessorTestRunner( + testName.getMethodName(), + new SpeedChangeEffect(speedProvider), + outputPresentationTimesUs::add); + Bitmap bitmap = readBitmap(IMAGE_PATH); + ImmutableList inputTimestamps = ImmutableList.of(0L, 4_000_000L, 5_000_000L); + + videoFrameProcessorTestRunner.queueInputBitmaps( + bitmap.getWidth(), + bitmap.getHeight(), + Pair.create(bitmap, createTimestampIterator(inputTimestamps))); + videoFrameProcessorTestRunner.endFrameProcessing(); + + assertThat(outputPresentationTimesUs).containsExactly(0L, 2_750_000L, 3_750_000L).inOrder(); + } + + private static VideoFrameProcessorTestRunner getVideoFrameProcessorTestRunner( + String testId, + GlEffect speedChangeEffect, + OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableForRenderingListener) + throws VideoFrameProcessingException { + VideoFrameProcessor.Factory videoFrameProcessorFactory = + new DefaultVideoFrameProcessor.Factory.Builder().build(); + ImmutableList effects = ImmutableList.of(speedChangeEffect); + return new VideoFrameProcessorTestRunner.Builder() + .setTestId(testId) + .setVideoFrameProcessorFactory(videoFrameProcessorFactory) + .setEffects(effects) + .setOnOutputFrameAvailableForRenderingListener(onOutputFrameAvailableForRenderingListener) + .build(); + } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeEffect.java index 36b1d7e87f..f8e22c8a59 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeEffect.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeEffect.java @@ -19,33 +19,57 @@ import static androidx.media3.common.util.Assertions.checkArgument; import android.content.Context; import androidx.annotation.FloatRange; -import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.C; +import androidx.media3.common.audio.SpeedProvider; import androidx.media3.common.util.UnstableApi; /** * Applies a speed change by updating the frame timestamps. * *

This effect doesn't drop any frames. + * + *

This effect is not supported for effects previewing. */ @UnstableApi public final class SpeedChangeEffect implements GlEffect { - private final float speed; + private final SpeedProvider speedProvider; - /** Creates an instance. */ + /** Creates an instance that applies the same {@code speed} change to all the timestamps. */ public SpeedChangeEffect(@FloatRange(from = 0, fromInclusive = false) float speed) { checkArgument(speed > 0f); - this.speed = speed; + speedProvider = + new SpeedProvider() { + @Override + public float getSpeed(long timeUs) { + return speed; + } + + @Override + public long getNextSpeedChangeTimeUs(long timeUs) { + return C.TIME_UNSET; + } + }; + } + + /** + * Creates an instance. + * + * @param speedProvider The {@link SpeedProvider} specifying the speed changes. Applied on each + * stream assuming the first frame timestamp of the input media is 0. + */ + public SpeedChangeEffect(SpeedProvider speedProvider) { + this.speedProvider = speedProvider; } @Override - public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) - throws VideoFrameProcessingException { - return new SpeedChangeShaderProgram(speed); + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { + return new SpeedChangeShaderProgram(speedProvider); } @Override public boolean isNoOp(int inputWidth, int inputHeight) { - return speed == 1f; + return speedProvider.getSpeed(/* timeUs= */ 0) == 1 + && speedProvider.getNextSpeedChangeTimeUs(/* timeUs= */ 0) == C.TIME_UNSET; } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeShaderProgram.java index 602cc48de1..c3988e282a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SpeedChangeShaderProgram.java @@ -15,24 +15,95 @@ */ package androidx.media3.effect; +import androidx.media3.common.C; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.audio.SpeedProvider; import androidx.media3.common.util.UnstableApi; -/** Applies a speed change by updating the frame timestamps. */ +/** + * Applies the speed changes specified in a {@link SpeedProvider} change by updating the frame + * timestamps. + * + *

Does not support seeking in effects previewing. + */ @UnstableApi /* package */ final class SpeedChangeShaderProgram extends PassthroughShaderProgram { - private final float speed; + private final OffsetSpeedProvider speedProvider; - public SpeedChangeShaderProgram(float speed) { + private long lastSpeedChangeInputTimeUs; + private long lastSpeedChangeOutputTimeUs; + + public SpeedChangeShaderProgram(SpeedProvider speedProvider) { super(); - this.speed = speed; + this.speedProvider = new OffsetSpeedProvider(speedProvider); + lastSpeedChangeInputTimeUs = C.TIME_UNSET; + lastSpeedChangeOutputTimeUs = C.TIME_UNSET; } @Override public void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { - super.queueInputFrame(glObjectsProvider, inputTexture, (long) (presentationTimeUs / speed)); + long outputPresentationTimeUs; + if (lastSpeedChangeInputTimeUs == C.TIME_UNSET) { + outputPresentationTimeUs = presentationTimeUs; + lastSpeedChangeInputTimeUs = presentationTimeUs; + lastSpeedChangeOutputTimeUs = outputPresentationTimeUs; + speedProvider.setOffset(presentationTimeUs); + } else { + long nextSpeedChangeInputTimeUs = + speedProvider.getNextSpeedChangeTimeUs(lastSpeedChangeInputTimeUs); + while (nextSpeedChangeInputTimeUs != C.TIME_UNSET + && nextSpeedChangeInputTimeUs <= presentationTimeUs) { + lastSpeedChangeOutputTimeUs = + getOutputTimeUs( + nextSpeedChangeInputTimeUs, speedProvider.getSpeed(lastSpeedChangeInputTimeUs)); + lastSpeedChangeInputTimeUs = nextSpeedChangeInputTimeUs; + nextSpeedChangeInputTimeUs = + speedProvider.getNextSpeedChangeTimeUs(lastSpeedChangeInputTimeUs); + } + outputPresentationTimeUs = + getOutputTimeUs(presentationTimeUs, speedProvider.getSpeed(presentationTimeUs)); + } + super.queueInputFrame(glObjectsProvider, inputTexture, outputPresentationTimeUs); + } + + @Override + public void signalEndOfCurrentInputStream() { + super.signalEndOfCurrentInputStream(); + lastSpeedChangeInputTimeUs = C.TIME_UNSET; + lastSpeedChangeOutputTimeUs = C.TIME_UNSET; + } + + private long getOutputTimeUs(long inputTimeUs, float speed) { + return (long) + (lastSpeedChangeOutputTimeUs + (inputTimeUs - lastSpeedChangeInputTimeUs) / speed); + } + + private static class OffsetSpeedProvider implements SpeedProvider { + + private final SpeedProvider speedProvider; + + private long offset; + + public OffsetSpeedProvider(SpeedProvider speedProvider) { + this.speedProvider = speedProvider; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + @Override + public float getSpeed(long timeUs) { + return speedProvider.getSpeed(timeUs - offset); + } + + @Override + public long getNextSpeedChangeTimeUs(long timeUs) { + long nextSpeedChangeTimeUs = speedProvider.getNextSpeedChangeTimeUs(timeUs - offset); + return nextSpeedChangeTimeUs == C.TIME_UNSET ? C.TIME_UNSET : offset + nextSpeedChangeTimeUs; + } } } diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/TestSpeedProvider.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/TestSpeedProvider.java new file mode 100644 index 0000000000..d505235c3a --- /dev/null +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/TestSpeedProvider.java @@ -0,0 +1,86 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.test.utils; + +import static androidx.media3.common.util.Assertions.checkArgument; + +import androidx.media3.common.C; +import androidx.media3.common.audio.AudioProcessor.AudioFormat; +import androidx.media3.common.audio.SpeedProvider; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; + +/** {@link SpeedProvider} for tests */ +@UnstableApi +public final class TestSpeedProvider implements SpeedProvider { + + private final long[] startTimesUs; + private final float[] speeds; + + /** + * Creates a {@code TestSpeedProvider} instance. + * + * @param startTimesUs The speed change start times, in microseconds. The values must be distinct + * and in increasing order. + * @param speeds The speeds corresponding to each start time. Consecutive values must be distinct. + * @return A {@code TestSpeedProvider}. + */ + public static TestSpeedProvider createWithStartTimes(long[] startTimesUs, float[] speeds) { + return new TestSpeedProvider(startTimesUs, speeds); + } + + /** + * Creates a {@code TestSpeedProvider} instance. + * + * @param audioFormat the {@link AudioFormat}. + * @param frameCounts The frame counts for which the same speed should be applied. + * @param speeds The speeds corresponding to each frame count. Consecutive values must be + * distinct. + * @return A {@code TestSpeedProvider}. + */ + public static TestSpeedProvider createWithFrameCounts( + AudioFormat audioFormat, int[] frameCounts, float[] speeds) { + long[] startTimesUs = new long[frameCounts.length]; + int totalFrameCount = 0; + for (int i = 0; i < frameCounts.length; i++) { + startTimesUs[i] = totalFrameCount * C.MICROS_PER_SECOND / audioFormat.sampleRate; + totalFrameCount += frameCounts[i]; + } + return new TestSpeedProvider(startTimesUs, speeds); + } + + private TestSpeedProvider(long[] startTimesUs, float[] speeds) { + checkArgument(startTimesUs.length == speeds.length); + this.startTimesUs = startTimesUs; + this.speeds = speeds; + } + + @Override + public float getSpeed(long timeUs) { + int index = + Util.binarySearchFloor( + startTimesUs, timeUs, /* inclusive= */ true, /* stayInBounds= */ true); + return speeds[index]; + } + + @Override + public long getNextSpeedChangeTimeUs(long timeUs) { + int index = + Util.binarySearchCeil( + startTimesUs, timeUs, /* inclusive= */ false, /* stayInBounds= */ false); + return index < startTimesUs.length ? startTimesUs[index] : C.TIME_UNSET; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java index c2b08d7a41..b6fec3a7a6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java @@ -342,6 +342,8 @@ public final class Composition { /** The {@link VideoCompositorSettings} to apply to the composition. */ public final VideoCompositorSettings videoCompositorSettings; + // TODO: b/302695659 - Ensure composition level effects are only applied consistently between the + // different VideoGraphs. /** The {@link Effects} to apply to the composition. */ public final Effects effects;