From 219b253731ba648a42a46e930c4a8d4701e03a4d Mon Sep 17 00:00:00 2001 From: sheenachhabra Date: Thu, 31 Aug 2023 10:15:01 -0700 Subject: [PATCH] Pass initial timestamp offset to VideoFrameProcessor For pause and resume feature we will remux the previously processed video in the first export and then in the second export we will resume processing remaining video. For the second export we will have to set initial timestamp offset so that video samples from the second export are in continuation to the previously muxed samples. PiperOrigin-RevId: 561689651 --- .../media3/transformer/SingleInputVideoGraph.java | 14 ++++++++++---- .../androidx/media3/transformer/Transformer.java | 3 ++- .../media3/transformer/TransformerInternal.java | 14 ++++++++++++-- .../transformer/VideoFrameProcessingWrapper.java | 7 +++++-- .../androidx/media3/transformer/VideoGraph.java | 3 ++- .../media3/transformer/VideoSampleExporter.java | 8 ++++++-- 6 files changed, 37 insertions(+), 12 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java index 4dd801041e..c51b5ae5f3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java @@ -53,7 +53,8 @@ import java.util.concurrent.Executor; DebugViewProvider debugViewProvider, Listener listener, Executor listenerExecutor, - List compositionEffects) { + List compositionEffects, + long initialTimestampOffsetUs) { @Nullable Presentation presentation = null; for (int i = 0; i < compositionEffects.size(); i++) { Effect effect = compositionEffects.get(i); @@ -71,7 +72,8 @@ import java.util.concurrent.Executor; debugViewProvider, listenerExecutor, /* renderFramesAutomatically= */ true, - presentation); + presentation, + initialTimestampOffsetUs); } } @@ -84,6 +86,7 @@ import java.util.concurrent.Executor; private final DebugViewProvider debugViewProvider; private final Executor listenerExecutor; private final boolean renderFramesAutomatically; + private final long initialTimestampOffsetUs; @Nullable private final Presentation presentation; @Nullable private VideoFrameProcessingWrapper videoFrameProcessingWrapper; @@ -101,7 +104,8 @@ import java.util.concurrent.Executor; DebugViewProvider debugViewProvider, Executor listenerExecutor, boolean renderFramesAutomatically, - @Nullable Presentation presentation) { + @Nullable Presentation presentation, + long initialTimestampOffsetUs) { this.context = context; this.videoFrameProcessorFactory = videoFrameProcessorFactory; this.inputColorInfo = inputColorInfo; @@ -112,6 +116,7 @@ import java.util.concurrent.Executor; this.listenerExecutor = listenerExecutor; this.renderFramesAutomatically = renderFramesAutomatically; this.presentation = presentation; + this.initialTimestampOffsetUs = initialTimestampOffsetUs; } /** @@ -168,7 +173,8 @@ import java.util.concurrent.Executor; } }, renderFramesAutomatically, - presentation); + presentation, + initialTimestampOffsetUs); } /** Returns the {@link GraphInput}. */ diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 67b38820cf..203f03dca2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -1008,7 +1008,8 @@ public final class Transformer { fallbackListener, applicationHandler, debugViewProvider, - clock); + clock, + /* videoSampleTimestampOffsetUs= */ 0); transformerInternal.start(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index fa1dce1e0c..a1cfb28e56 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -105,6 +105,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Listener listener; private final HandlerWrapper applicationHandler; private final Clock clock; + + /** + * The presentation timestamp offset for all the video samples. It will be set when resuming video + * processing after remuxing previously processed samples. + */ + private final long videoSampleTimestampOffsetUs; + private final HandlerThread internalHandlerThread; private final HandlerWrapper internalHandler; private final List sequenceAssetLoaders; @@ -142,13 +149,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; FallbackListener fallbackListener, HandlerWrapper applicationHandler, DebugViewProvider debugViewProvider, - Clock clock) { + Clock clock, + long videoSampleTimestampOffsetUs) { this.context = context; this.composition = composition; this.encoderFactory = new CapturingEncoderFactory(encoderFactory); this.listener = listener; this.applicationHandler = applicationHandler; this.clock = clock; + this.videoSampleTimestampOffsetUs = videoSampleTimestampOffsetUs; this.muxerWrapper = muxerWrapper; // It's safe to use "this" because we don't mux any data before exiting the constructor. this.muxerWrapper.setListener(this); @@ -601,7 +610,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; muxerWrapper, /* errorConsumer= */ this::onError, fallbackListener, - debugViewProvider)); + debugViewProvider, + videoSampleTimestampOffsetUs)); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java index 29515809a0..f6e5429806 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java @@ -48,6 +48,7 @@ import java.util.concurrent.atomic.AtomicLong; private final VideoFrameProcessor videoFrameProcessor; private final AtomicLong mediaItemOffsetUs; private final ColorInfo inputColorInfo; + private final long initialTimestampOffsetUs; @Nullable final Presentation presentation; public VideoFrameProcessingWrapper( @@ -59,10 +60,12 @@ import java.util.concurrent.atomic.AtomicLong; Executor listenerExecutor, VideoFrameProcessor.Listener listener, boolean renderFramesAutomatically, - @Nullable Presentation presentation) + @Nullable Presentation presentation, + long initialTimestampOffsetUs) throws VideoFrameProcessingException { this.mediaItemOffsetUs = new AtomicLong(); this.inputColorInfo = inputColorInfo; + this.initialTimestampOffsetUs = initialTimestampOffsetUs; this.presentation = presentation; videoFrameProcessor = @@ -89,7 +92,7 @@ import java.util.concurrent.atomic.AtomicLong; createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation), new FrameInfo.Builder(decodedSize.getWidth(), decodedSize.getHeight()) .setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio) - .setOffsetToAddUs(mediaItemOffsetUs.get()) + .setOffsetToAddUs(initialTimestampOffsetUs + mediaItemOffsetUs.get()) .build()); } mediaItemOffsetUs.addAndGet(durationUs); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java index 883d2be788..4a94f12a28 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java @@ -56,7 +56,8 @@ import java.util.concurrent.Executor; DebugViewProvider debugViewProvider, Listener listener, Executor listenerExecutor, - List compositionEffects) + List compositionEffects, + long initialTimestampOffsetUs) throws VideoFrameProcessingException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 46d56101f3..0c098fd372 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -63,6 +63,7 @@ import org.checkerframework.dataflow.qual.Pure; private final VideoGraph videoGraph; private final EncoderWrapper encoderWrapper; private final DecoderInputBuffer encoderOutputBuffer; + private final long initialTimestampOffsetUs; /** * The timestamp of the last buffer processed before {@linkplain @@ -82,11 +83,13 @@ import org.checkerframework.dataflow.qual.Pure; MuxerWrapper muxerWrapper, Consumer errorConsumer, FallbackListener fallbackListener, - DebugViewProvider debugViewProvider) + DebugViewProvider debugViewProvider, + long initialTimestampOffsetUs) throws ExportException { // TODO(b/278259383) Consider delaying configuration of VideoSampleExporter to use the decoder // output format instead of the extractor output format, to match AudioSampleExporter behavior. super(firstInputFormat, muxerWrapper); + this.initialTimestampOffsetUs = initialTimestampOffsetUs; finalFramePresentationTimeUs = C.TIME_UNSET; ColorInfo decoderInputColor; @@ -488,7 +491,8 @@ import org.checkerframework.dataflow.qual.Pure; debugViewProvider, /* listener= */ thisRef, /* listenerExecutor= */ MoreExecutors.directExecutor(), - compositionEffects); + compositionEffects, + initialTimestampOffsetUs); } @Nullable