diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 97b9c7f788..ab42421743 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -23,12 +23,12 @@ import android.opengl.EGLExt; import android.view.Surface; import androidx.annotation.IntDef; import androidx.annotation.Nullable; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import java.util.Iterator; import java.util.List; import java.util.concurrent.Executor; @@ -184,7 +184,7 @@ public interface VideoFrameProcessor { * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept * {@linkplain #INPUT_TYPE_BITMAP bitmap input}. */ - void queueInputBitmap(Bitmap inputBitmap, Iterator inStreamOffsetsUs); + void queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs); /** * Provides an input texture ID to the {@code VideoFrameProcessor}. diff --git a/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java b/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java new file mode 100644 index 0000000000..fda73e61b4 --- /dev/null +++ b/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java @@ -0,0 +1,65 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.common.util; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkState; +import static java.lang.Math.round; + +import androidx.annotation.FloatRange; +import androidx.annotation.IntRange; +import androidx.media3.common.C; + +/** + * A {@link TimestampIterator} that generates monotonically increasing timestamps (in microseconds) + * distributed evenly over the given {@code durationUs} based on the given {@code frameRate}. + */ +@UnstableApi +public final class ConstantRateTimestampIterator implements TimestampIterator { + + private final double framesDurationUs; + private double currentTimestampUs; + private int framesToAdd; + + /** + * Creates an instance. + * + * @param durationUs The duration the timestamps should span over, in microseconds. + * @param frameRate The frame rate in frames per second. + */ + public ConstantRateTimestampIterator( + @IntRange(from = 1) long durationUs, + @FloatRange(from = 0, fromInclusive = false) float frameRate) { + checkArgument(durationUs > 0); + checkArgument(frameRate > 0); + framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND)); + framesDurationUs = C.MICROS_PER_SECOND / frameRate; + } + + @Override + public boolean hasNext() { + return framesToAdd != 0; + } + + @Override + public long next() { + checkState(hasNext()); + framesToAdd--; + long next = round(currentTimestampUs); + currentTimestampUs += framesDurationUs; + return next; + } +} diff --git a/libraries/common/src/main/java/androidx/media3/common/util/TimestampIterator.java b/libraries/common/src/main/java/androidx/media3/common/util/TimestampIterator.java new file mode 100644 index 0000000000..913dcc9c63 --- /dev/null +++ b/libraries/common/src/main/java/androidx/media3/common/util/TimestampIterator.java @@ -0,0 +1,44 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.common.util; + +import java.util.Iterator; + +/** A primitive long iterator used for generating sequences of timestamps. */ +@UnstableApi +public interface TimestampIterator { + + /** Returns whether there is another element. */ + boolean hasNext(); + + /** Returns the next timestamp. */ + long next(); + + /** Creates TimestampIterator */ + static TimestampIterator createFromLongIterator(Iterator iterator) { + return new TimestampIterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public long next() { + return iterator.next(); + } + }; + } +} diff --git a/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java b/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java new file mode 100644 index 0000000000..0c5f483ba4 --- /dev/null +++ b/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.common.util; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.media3.common.C; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.util.ArrayList; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link ConstantRateTimestampIterator}. */ +@RunWith(AndroidJUnit4.class) +public class ConstantRateTimestampIteratorTest { + + @Test + public void timestampIterator_validArguments_generatesCorrectTimestamps() throws Exception { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator(C.MICROS_PER_SECOND, /* frameRate= */ 2); + + assertThat(generateList(constantRateTimestampIterator)) + .containsExactly(0L, C.MICROS_PER_SECOND / 2); + } + + @Test + public void timestampIterator_realisticArguments_generatesCorrectNumberOfTimestamps() + throws Exception { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator((long) (2.5 * C.MICROS_PER_SECOND), /* frameRate= */ 30); + + assertThat(generateList(constantRateTimestampIterator)).hasSize(75); + } + + @Test + public void timestampIterator_realisticArguments_generatesTimestampsInStrictOrder() + throws Exception { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator((long) (2.5 * C.MICROS_PER_SECOND), /* frameRate= */ 30); + + assertThat(generateList(constantRateTimestampIterator)).isInStrictOrder(); + } + + @Test + public void timestampIterator_realisticArguments_doesNotGenerateDuplicates() throws Exception { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator((long) (2.5 * C.MICROS_PER_SECOND), /* frameRate= */ 30); + + assertThat(generateList(constantRateTimestampIterator)).containsNoDuplicates(); + } + + @Test + public void timestampIterator_smallDuration_generatesEmptyIterator() throws Exception { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator(/* durationUs= */ 1, /* frameRate= */ 2); + + assertThat(generateList(constantRateTimestampIterator)).isEmpty(); + } + + private static List generateList(TimestampIterator iterator) { + ArrayList list = new ArrayList<>(); + + while (iterator.hasNext()) { + list.add(iterator.next()); + } + return list; + } +} diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java index 1f3bc1afba..408eaf62cb 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java @@ -16,6 +16,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.TimestampIterator.createFromLongIterator; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; import static com.google.common.truth.Truth.assertThat; @@ -209,8 +210,9 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { videoFrameProcessorTestRunner.queueInputBitmaps( bitmap1.getWidth(), bitmap1.getHeight(), - Pair.create(bitmap1, ImmutableList.of(offset1).iterator()), - Pair.create(bitmap2, ImmutableList.of(offset2, offset3).iterator())); + Pair.create(bitmap1, createFromLongIterator(ImmutableList.of(offset1).iterator())), + Pair.create( + bitmap2, createFromLongIterator(ImmutableList.of(offset2, offset3).iterator()))); videoFrameProcessorTestRunner.endFrameProcessing(); assertThat(actualPresentationTimesUs).containsExactly(offset1, offset2, offset3).inOrder(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 2c0ff290b7..2d3e87b40d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -52,13 +52,13 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -438,7 +438,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { } @Override - public void queueInputBitmap(Bitmap inputBitmap, Iterator inStreamOffsetsUs) { + public void queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) { FrameInfo frameInfo = checkNotNull(this.nextInputFrameInfo); // TODO(b/262693274): move frame duplication logic out of the texture manager so // textureManager.queueInputBitmap() frame rate and duration parameters be removed. diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index 5c5c354a0b..5f67c2c4e7 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -45,11 +45,11 @@ import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.errorprone.annotations.CanIgnoreReturnValue; -import java.util.Iterator; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -359,14 +359,14 @@ public final class VideoFrameProcessorTestRunner { videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate); } - public void queueInputBitmaps(int width, int height, Pair>... frames) { + public void queueInputBitmaps(int width, int height, Pair... frames) { videoFrameProcessor.registerInputStream( INPUT_TYPE_BITMAP, effects, new FrameInfo.Builder(width, height) .setPixelWidthHeightRatio(pixelWidthHeightRatio) .build()); - for (Pair> frame : frames) { + for (Pair frame : frames) { videoFrameProcessor.queueInputBitmap(frame.first, frame.second); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java index a78b92ed62..c6df629523 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java @@ -20,9 +20,9 @@ import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.ColorInfo; import androidx.media3.common.OnInputFrameProcessedListener; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import androidx.media3.decoder.DecoderInputBuffer; -import java.util.Iterator; /** Consumer of encoded media samples, raw audio or raw video frames. */ @UnstableApi @@ -93,7 +93,7 @@ public interface SampleConsumer { * @param inStreamOffsetsUs The times within the current stream that the bitmap should be * displayed at. The timestamps should be monotonically increasing. */ - default boolean queueInputBitmap(Bitmap inputBitmap, Iterator inStreamOffsetsUs) { + default boolean queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) { throw new UnsupportedOperationException(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java index 5342087c92..e9a7a92824 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java @@ -35,11 +35,11 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.util.Clock; import androidx.media3.common.util.HandlerWrapper; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.decoder.DecoderInputBuffer; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -349,17 +349,16 @@ import java.util.concurrent.atomic.AtomicInteger; } /** - * Given an {@link Iterator}, creates an iterator that includes all the values in the original - * iterator (in the same order) up to and including the first occurrence of the {@code - * clippingValue}. + * Wraps a {@link TimestampIterator}, providing all the values in the original timestamp iterator + * (in the same order) up to and including the first occurrence of the {@code clippingValue}. */ - private static final class ClippingIterator implements Iterator { + private static final class ClippingIterator implements TimestampIterator { - private final Iterator iterator; + private final TimestampIterator iterator; private final long clippingValue; private boolean hasReachedClippingValue; - public ClippingIterator(Iterator iterator, long clippingValue) { + public ClippingIterator(TimestampIterator iterator, long clippingValue) { this.iterator = iterator; this.clippingValue = clippingValue; } @@ -370,9 +369,9 @@ import java.util.concurrent.atomic.AtomicInteger; } @Override - public Long next() { + public long next() { checkState(hasNext()); - Long next = iterator.next(); + long next = iterator.next(); if (clippingValue == next) { hasReachedClippingValue = true; } @@ -450,8 +449,8 @@ import java.util.concurrent.atomic.AtomicInteger; } @Override - public boolean queueInputBitmap(Bitmap inputBitmap, Iterator inStreamOffsetsUs) { - Iterator iteratorToUse = inStreamOffsetsUs; + public boolean queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) { + TimestampIterator iteratorToUse = inStreamOffsetsUs; if (isLooping) { long durationLeftUs = maxSequenceDurationUs - totalDurationUs; if (durationLeftUs <= 0) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java index 8c781df6e6..f21460f74a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java @@ -37,9 +37,9 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.Consumer; import androidx.media3.common.util.Size; +import androidx.media3.common.util.TimestampIterator; import androidx.media3.effect.Presentation; import com.google.common.collect.ImmutableList; -import java.util.Iterator; import java.util.List; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; @@ -175,7 +175,7 @@ import java.util.concurrent.atomic.AtomicLong; } @Override - public boolean queueInputBitmap(Bitmap inputBitmap, Iterator inStreamOffsetsUs) { + public boolean queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) { videoFrameProcessor.queueInputBitmap(inputBitmap, inStreamOffsetsUs); return true; }