From dc4f20ed840f37e397b60435de85b0151209da50 Mon Sep 17 00:00:00 2001 From: claincly Date: Sat, 18 May 2024 07:38:38 -0700 Subject: [PATCH] Fix image seeking Queue image again after position reset, and reset timestamp iterator. PiperOrigin-RevId: 635049953 --- .../util/ConstantRateTimestampIterator.java | 32 ++++++++-- .../ConstantRateTimestampIteratorTest.java | 58 ++++++++++++++++--- .../media3/effect/BitmapTextureManager.java | 16 ++++- .../media3/effect/ExternalTextureManager.java | 2 +- .../media3/effect/TexIdTextureManager.java | 3 +- .../media3/effect/TextureManager.java | 2 +- .../SequencePlayerRenderersWrapper.java | 17 ++++-- 7 files changed, 108 insertions(+), 22 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java b/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java index b02b409643..b2bf6c3c4f 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java @@ -30,10 +30,11 @@ import androidx.media3.common.C; @UnstableApi public final class ConstantRateTimestampIterator implements TimestampIterator { - private final long durationUs; private final float frameRate; private final double framesDurationUs; private final int totalNumberOfFramesToAdd; + private final long startPositionUs; + private final long endPositionUs; private int framesAdded; @@ -46,11 +47,30 @@ public final class ConstantRateTimestampIterator implements TimestampIterator { public ConstantRateTimestampIterator( @IntRange(from = 1) long durationUs, @FloatRange(from = 0, fromInclusive = false) float frameRate) { - checkArgument(durationUs > 0); + this(/* startPositionUs= */ 0, /* endPositionUs= */ durationUs, frameRate); + } + + /** + * Creates an instance that outputs timestamps from {@code startTimeUs}. + * + * @param startPositionUs The start position in microseconds. The first timestamp generated will + * be equal to {@code startPositionUs}. + * @param endPositionUs The end position at which the timestamps finish, in microseconds. The + * generated timestamps are less or equal to the end position. + * @param frameRate The frame rate in frames per second. + */ + public ConstantRateTimestampIterator( + @IntRange(from = 0) long startPositionUs, + @IntRange(from = 1) long endPositionUs, + @FloatRange(from = 0, fromInclusive = false) float frameRate) { + checkArgument(endPositionUs > 0); checkArgument(frameRate > 0); - this.durationUs = durationUs; + checkArgument(0 <= startPositionUs && startPositionUs < endPositionUs); + this.startPositionUs = startPositionUs; + this.endPositionUs = endPositionUs; this.frameRate = frameRate; - this.totalNumberOfFramesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND)); + float durationSecs = (endPositionUs - startPositionUs) / (float) C.MICROS_PER_SECOND; + this.totalNumberOfFramesToAdd = round(frameRate * durationSecs); framesDurationUs = C.MICROS_PER_SECOND / frameRate; } @@ -67,7 +87,7 @@ public final class ConstantRateTimestampIterator implements TimestampIterator { @Override public ConstantRateTimestampIterator copyOf() { - return new ConstantRateTimestampIterator(durationUs, frameRate); + return new ConstantRateTimestampIterator(startPositionUs, endPositionUs, frameRate); } @Override @@ -80,7 +100,7 @@ public final class ConstantRateTimestampIterator implements TimestampIterator { /** Returns the timestamp after {@code numberOfFrames}, in microseconds. */ private long getTimestampUsAfter(int numberOfFrames) { - long timestampUs = round(framesDurationUs * numberOfFrames); + long timestampUs = startPositionUs + round(framesDurationUs * numberOfFrames); // Check for possible overflow. checkState(timestampUs >= 0); return timestampUs; diff --git a/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java b/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java index 122a61a195..1010b20af6 100644 --- a/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java @@ -30,7 +30,7 @@ import org.junit.runner.RunWith; public class ConstantRateTimestampIteratorTest { @Test - public void timestampIterator_validArguments_generatesCorrectTimestamps() throws Exception { + public void timestampIterator_validArguments_generatesCorrectTimestamps() { ConstantRateTimestampIterator constantRateTimestampIterator = new ConstantRateTimestampIterator(C.MICROS_PER_SECOND, /* frameRate= */ 2); @@ -41,8 +41,7 @@ public class ConstantRateTimestampIteratorTest { } @Test - public void timestampIterator_realisticArguments_generatesCorrectNumberOfTimestamps() - throws Exception { + public void timestampIterator_realisticArguments_generatesCorrectNumberOfTimestamps() { ConstantRateTimestampIterator constantRateTimestampIterator = new ConstantRateTimestampIterator((long) (2.5 * C.MICROS_PER_SECOND), /* frameRate= */ 30); @@ -52,8 +51,7 @@ public class ConstantRateTimestampIteratorTest { } @Test - public void timestampIterator_realisticArguments_generatesTimestampsInStrictOrder() - throws Exception { + public void timestampIterator_realisticArguments_generatesTimestampsInStrictOrder() { ConstantRateTimestampIterator constantRateTimestampIterator = new ConstantRateTimestampIterator((long) (2.5 * C.MICROS_PER_SECOND), /* frameRate= */ 30); @@ -61,7 +59,7 @@ public class ConstantRateTimestampIteratorTest { } @Test - public void timestampIterator_realisticArguments_doesNotGenerateDuplicates() throws Exception { + public void timestampIterator_realisticArguments_doesNotGenerateDuplicates() { ConstantRateTimestampIterator constantRateTimestampIterator = new ConstantRateTimestampIterator((long) (2.5 * C.MICROS_PER_SECOND), /* frameRate= */ 30); @@ -69,7 +67,7 @@ public class ConstantRateTimestampIteratorTest { } @Test - public void timestampIterator_smallDuration_generatesEmptyIterator() throws Exception { + public void timestampIterator_smallDuration_generatesEmptyIterator() { ConstantRateTimestampIterator constantRateTimestampIterator = new ConstantRateTimestampIterator(/* durationUs= */ 1, /* frameRate= */ 2); @@ -77,6 +75,52 @@ public class ConstantRateTimestampIteratorTest { assertThat(constantRateTimestampIterator.getLastTimestampUs()).isEqualTo(C.TIME_UNSET); } + @Test + public void timestampIterator_withStartTimeAndEvenFrameRate_generatesCorrectTimestamps() { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator( + /* startPositionUs= */ 500_000L, C.MICROS_PER_SECOND, /* frameRate= */ 2); + + assertThat(generateList(constantRateTimestampIterator)).containsExactly(500_000L); + assertThat(constantRateTimestampIterator.getLastTimestampUs()).isEqualTo(500_000L); + } + + @Test + public void timestampIterator_withStartTimeAndOddFrameRate_generatesCorrectTimestamps() { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator( + /* startPositionUs= */ 500_000L, C.MICROS_PER_SECOND, /* frameRate= */ 3); + + assertThat(generateList(constantRateTimestampIterator)) + .containsExactly(500_000L, 833_333L) + .inOrder(); + assertThat(constantRateTimestampIterator.getLastTimestampUs()).isEqualTo(833_333L); + } + + @Test + public void timestampIterator_withZeroStartTime_generatesCorrectTimestamps() { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator( + /* startPositionUs= */ 0L, + /* endPositionUs= */ C.MICROS_PER_SECOND, + /* frameRate= */ 3); + + assertThat(generateList(constantRateTimestampIterator)).containsExactly(0L, 333_333L, 666_667L); + assertThat(constantRateTimestampIterator.getLastTimestampUs()).isEqualTo(666_667L); + } + + @Test + public void timestampIterator_withNoTimestampsWithinParameters_generatesNoTimestamp() { + ConstantRateTimestampIterator constantRateTimestampIterator = + new ConstantRateTimestampIterator( + /* startPositionUs= */ 900_000L, + /* endPositionUs= */ C.MICROS_PER_SECOND, + /* frameRate= */ 3); + + assertThat(generateList(constantRateTimestampIterator)).isEmpty(); + assertThat(constantRateTimestampIterator.getLastTimestampUs()).isEqualTo(C.TIME_UNSET); + } + private static List generateList(TimestampIterator iterator) { ArrayList list = new ArrayList<>(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java index aa03fb442e..1a8f84fb71 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java @@ -35,6 +35,7 @@ import androidx.media3.common.util.Util; import java.util.Queue; import java.util.concurrent.LinkedBlockingQueue; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.Nullable; /** * Forwards a video frame produced from a {@link Bitmap} to a {@link GlShaderProgram} for @@ -47,7 +48,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final GlObjectsProvider glObjectsProvider; private @MonotonicNonNull GainmapShaderProgram gainmapShaderProgram; - private @MonotonicNonNull GlTextureInfo currentSdrGlTextureInfo; + @Nullable private GlTextureInfo currentSdrGlTextureInfo; private int downstreamShaderProgramCapacity; private boolean currentInputStreamEnded; private boolean isNextFrameInTexture; @@ -180,8 +181,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - protected void flush() { + protected void flush() throws VideoFrameProcessingException { pendingBitmaps.clear(); + isNextFrameInTexture = false; + currentInputStreamEnded = false; + downstreamShaderProgramCapacity = 0; + if (currentSdrGlTextureInfo != null) { + try { + currentSdrGlTextureInfo.release(); + } catch (GlUtil.GlException e) { + throw VideoFrameProcessingException.from(e); + } + currentSdrGlTextureInfo = null; + } super.flush(); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index 7717f82503..758356e357 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -267,7 +267,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - protected void flush() { + protected void flush() throws VideoFrameProcessingException { externalShaderProgramInputCapacity.set(0); currentFrame = null; pendingFrames.clear(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java index e441616f46..031433e8e2 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java @@ -27,6 +27,7 @@ import androidx.media3.common.FrameInfo; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.OnInputFrameProcessedListener; +import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -132,7 +133,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Methods that must be called on the GL thread. @Override - protected synchronized void flush() { + protected synchronized void flush() throws VideoFrameProcessingException { checkNotNull(frameConsumptionManager).onFlush(); super.flush(); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java index 1af0a6c73b..ea884bfff0 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java @@ -157,7 +157,7 @@ import androidx.media3.common.util.TimestampIterator; public abstract void release() throws VideoFrameProcessingException; /** Clears any pending data. Must be called on the GL thread. */ - protected void flush() { + protected void flush() throws VideoFrameProcessingException { synchronized (lock) { if (onFlushCompleteTask != null) { videoFrameProcessingTaskExecutor.submitWithHighPriority(onFlushCompleteTask); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java index d140477782..f59f9592d1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java @@ -15,6 +15,7 @@ */ package androidx.media3.transformer; +import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.exoplayer.DefaultRenderersFactory.DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS; @@ -292,8 +293,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private ImmutableList videoEffects; private @MonotonicNonNull ConstantRateTimestampIterator timestampIterator; - private boolean inputStreamPendingRegistration; + private @MonotonicNonNull EditedMediaItem editedMediaItem; @Nullable private ExoPlaybackException pendingExoPlaybackException; + private boolean inputStreamPendingRegistration; private long streamOffsetUs; private boolean mayRenderStartOfStream; @@ -360,6 +362,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { videoSink.flush(); super.onPositionReset(positionUs, joining); + timestampIterator = + new ConstantRateTimestampIterator( + /* startPositionUs= */ positionUs - streamOffsetUs, + /* endPositionUs= */ checkNotNull(editedMediaItem).getPresentationDurationUs(), + DEFAULT_FRAME_RATE); videoFrameReleaseControl.reset(); if (joining) { videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false); @@ -388,13 +395,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkState(getTimeline().getWindowCount() == 1); super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); streamOffsetUs = offsetUs; - EditedMediaItem editedMediaItem = + editedMediaItem = sequencePlayerRenderersWrapper.sequence.editedMediaItems.get( getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid)); - videoEffects = editedMediaItem.effects.videoEffects; timestampIterator = new ConstantRateTimestampIterator( - editedMediaItem.getPresentationDurationUs(), /* frameRate= */ DEFAULT_FRAME_RATE); + /* startPositionUs= */ startPositionUs - streamOffsetUs, + /* endPositionUs= */ editedMediaItem.getPresentationDurationUs(), + DEFAULT_FRAME_RATE); + videoEffects = editedMediaItem.effects.videoEffects; inputStreamPendingRegistration = true; }