From 0d12de81344559ffdc4ef17f313d498b6712bd6f Mon Sep 17 00:00:00 2001 From: kimvde Date: Tue, 6 Dec 2022 15:31:32 +0000 Subject: [PATCH] Move slow mo video flattening to AssetLoader PiperOrigin-RevId: 493300556 --- .../AudioTranscodingSamplePipeline.java | 23 +++---- .../transformer/BaseSamplePipeline.java | 63 +------------------ .../transformer/ExoPlayerAssetLoader.java | 19 +++++- .../ExoPlayerAssetLoaderRenderer.java | 40 ++++++++++++ .../PassthroughSamplePipeline.java | 18 ++---- .../transformer/TransformerInternal.java | 2 +- .../VideoTranscodingSamplePipeline.java | 26 +++----- 7 files changed, 83 insertions(+), 108 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java index 79ccd2342a..35c28d0497 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java @@ -66,12 +66,7 @@ import org.checkerframework.dataflow.qual.Pure; MuxerWrapper muxerWrapper, FallbackListener fallbackListener) throws TransformationException { - super( - inputFormat, - streamStartPositionUs, - streamOffsetUs, - transformationRequest.flattenForSlowMotion, - muxerWrapper); + super(inputFormat, streamStartPositionUs, muxerWrapper); if (forceSilentAudioDurationUs != C.TIME_UNSET) { silentAudioGenerator = @@ -153,23 +148,23 @@ import org.checkerframework.dataflow.qual.Pure; return true; } - @Override - public void release() { - audioProcessingPipeline.reset(); - encoder.release(); - } - @Override @Nullable - protected DecoderInputBuffer dequeueInputBufferInternal() { + public DecoderInputBuffer dequeueInputBuffer() { return hasPendingInputBuffer ? null : inputBuffer; } @Override - protected void queueInputBufferInternal() { + public void queueInputBuffer() { hasPendingInputBuffer = true; } + @Override + public void release() { + audioProcessingPipeline.reset(); + encoder.release(); + } + @Override protected boolean processDataUpToMuxer() throws TransformationException { if (!audioProcessingPipeline.isOperational()) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java index c80f677d1c..85d8dbb44f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java @@ -16,7 +16,6 @@ package androidx.media3.transformer; -import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; import androidx.annotation.Nullable; @@ -24,35 +23,20 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.decoder.DecoderInputBuffer; -import java.nio.ByteBuffer; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; /* package */ abstract class BaseSamplePipeline implements SamplePipeline { private final long streamStartPositionUs; - private final long streamOffsetUs; private final MuxerWrapper muxerWrapper; private final @C.TrackType int trackType; - private final @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener; - @Nullable private DecoderInputBuffer inputBuffer; private boolean muxerWrapperTrackAdded; public BaseSamplePipeline( - Format inputFormat, - long streamStartPositionUs, - long streamOffsetUs, - boolean flattenForSlowMotion, - MuxerWrapper muxerWrapper) { + Format inputFormat, long streamStartPositionUs, MuxerWrapper muxerWrapper) { this.streamStartPositionUs = streamStartPositionUs; - this.streamOffsetUs = streamOffsetUs; this.muxerWrapper = muxerWrapper; trackType = MimeTypes.getTrackType(inputFormat.sampleMimeType); - sefVideoSlowMotionFlattener = - flattenForSlowMotion && trackType == C.TRACK_TYPE_VIDEO - ? new SefSlowMotionFlattener(inputFormat) - : null; } protected static TransformationException createNoSupportedMimeTypeException( @@ -66,32 +50,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); } - @Nullable - @Override - public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { - inputBuffer = dequeueInputBufferInternal(); - return inputBuffer; - } - - @Override - public void queueInputBuffer() throws TransformationException { - DecoderInputBuffer inputBuffer = checkNotNull(this.inputBuffer); - checkNotNull(inputBuffer.data); - if (!shouldDropInputBuffer(inputBuffer)) { - queueInputBufferInternal(); - } - } - @Override public boolean processData() throws TransformationException { return feedMuxer() || processDataUpToMuxer(); } - @Nullable - protected abstract DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException; - - protected abstract void queueInputBufferInternal() throws TransformationException; - protected abstract boolean processDataUpToMuxer() throws TransformationException; @Nullable @@ -104,30 +67,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; protected abstract boolean isMuxerInputEnded(); - /** - * Preprocesses an {@linkplain DecoderInputBuffer input buffer} queued to the pipeline and returns - * whether it should be dropped. - */ - @RequiresNonNull("#1.data") - private boolean shouldDropInputBuffer(DecoderInputBuffer inputBuffer) { - ByteBuffer inputBytes = inputBuffer.data; - - if (sefVideoSlowMotionFlattener == null || inputBuffer.isEndOfStream()) { - return false; - } - - long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs; - boolean shouldDropInputBuffer = - sefVideoSlowMotionFlattener.dropOrTransformSample(inputBytes, presentationTimeUs); - if (shouldDropInputBuffer) { - inputBytes.clear(); - } else { - inputBuffer.timeUs = - streamOffsetUs + sefVideoSlowMotionFlattener.getSamplePresentationTimeUs(); - } - return shouldDropInputBuffer; - } - /** * Attempts to pass encoded data to the muxer, and returns whether it may be possible to pass more * data immediately by calling this method again. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java index db2144d89e..a4ba735566 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java @@ -75,6 +75,7 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; MediaItem mediaItem, boolean removeAudio, boolean removeVideo, + boolean flattenForSlowMotion, MediaSource.Factory mediaSourceFactory, Codec.DecoderFactory decoderFactory, Looper looper, @@ -99,7 +100,8 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; ExoPlayer.Builder playerBuilder = new ExoPlayer.Builder( context, - new RenderersFactoryImpl(removeAudio, removeVideo, decoderFactory, listener)) + new RenderersFactoryImpl( + removeAudio, removeVideo, flattenForSlowMotion, decoderFactory, listener)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) .setLoadControl(loadControl) @@ -143,16 +145,19 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; private final TransformerMediaClock mediaClock; private final boolean removeAudio; private final boolean removeVideo; + private final boolean flattenForSlowMotion; private final Codec.DecoderFactory decoderFactory; private final ExoPlayerAssetLoader.Listener assetLoaderListener; public RenderersFactoryImpl( boolean removeAudio, boolean removeVideo, + boolean flattenForSlowMotion, Codec.DecoderFactory decoderFactory, ExoPlayerAssetLoader.Listener assetLoaderListener) { this.removeAudio = removeAudio; this.removeVideo = removeVideo; + this.flattenForSlowMotion = flattenForSlowMotion; this.decoderFactory = decoderFactory; this.assetLoaderListener = assetLoaderListener; mediaClock = new TransformerMediaClock(); @@ -171,13 +176,21 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; if (!removeAudio) { renderers[index] = new ExoPlayerAssetLoaderRenderer( - C.TRACK_TYPE_AUDIO, decoderFactory, mediaClock, assetLoaderListener); + C.TRACK_TYPE_AUDIO, + /* flattenForSlowMotion= */ false, + decoderFactory, + mediaClock, + assetLoaderListener); index++; } if (!removeVideo) { renderers[index] = new ExoPlayerAssetLoaderRenderer( - C.TRACK_TYPE_VIDEO, decoderFactory, mediaClock, assetLoaderListener); + C.TRACK_TYPE_VIDEO, + flattenForSlowMotion, + decoderFactory, + mediaClock, + assetLoaderListener); index++; } return renderers; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java index 0fa0d3e3f7..9727fe765b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java @@ -40,6 +40,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private static final String TAG = "ExoPlayerAssetLoaderRenderer"; + private final boolean flattenForSlowMotion; private final Codec.DecoderFactory decoderFactory; private final TransformerMediaClock mediaClock; private final ExoPlayerAssetLoader.Listener assetLoaderListener; @@ -48,6 +49,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private boolean isTransformationRunning; private long streamStartPositionUs; private long streamOffsetUs; + private @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener; private @MonotonicNonNull Codec decoder; @Nullable private ByteBuffer pendingDecoderOutputBuffer; private SamplePipeline.@MonotonicNonNull Input samplePipelineInput; @@ -55,10 +57,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; public ExoPlayerAssetLoaderRenderer( int trackType, + boolean flattenForSlowMotion, Codec.DecoderFactory decoderFactory, TransformerMediaClock mediaClock, ExoPlayerAssetLoader.Listener assetLoaderListener) { super(trackType); + this.flattenForSlowMotion = flattenForSlowMotion; this.decoderFactory = decoderFactory; this.mediaClock = mediaClock; this.assetLoaderListener = assetLoaderListener; @@ -161,6 +165,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; Format inputFormat = checkNotNull(formatHolder.format); samplePipelineInput = assetLoaderListener.onTrackAdded(inputFormat, streamStartPositionUs, streamOffsetUs); + if (getTrackType() == C.TRACK_TYPE_VIDEO && flattenForSlowMotion) { + sefVideoSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat); + } if (samplePipelineInput.expectsDecodedData()) { decoder = decoderFactory.createForAudioDecoding(inputFormat); } @@ -228,6 +235,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } + if (shouldDropInputBuffer(decoderInputBuffer)) { + return true; + } + decoder.queueInputBuffer(decoderInputBuffer); return true; } @@ -249,6 +260,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } + if (shouldDropInputBuffer(samplePipelineInputBuffer)) { + return true; + } + samplePipelineInput.queueInputBuffer(); if (samplePipelineInputBuffer.isEndOfStream()) { isEnded = true; @@ -279,4 +294,29 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } } + + /** + * Preprocesses an {@linkplain DecoderInputBuffer input buffer} queued to the pipeline and returns + * whether it should be dropped. + * + *

The input buffer is cleared if it should be dropped. + */ + private boolean shouldDropInputBuffer(DecoderInputBuffer inputBuffer) { + ByteBuffer inputBytes = checkNotNull(inputBuffer.data); + + if (sefVideoSlowMotionFlattener == null || inputBuffer.isEndOfStream()) { + return false; + } + + long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs; + boolean shouldDropInputBuffer = + sefVideoSlowMotionFlattener.dropOrTransformSample(inputBytes, presentationTimeUs); + if (shouldDropInputBuffer) { + inputBytes.clear(); + } else { + inputBuffer.timeUs = + streamOffsetUs + sefVideoSlowMotionFlattener.getSamplePresentationTimeUs(); + } + return shouldDropInputBuffer; + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java index b83e65c9a6..722bfc751c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java @@ -31,16 +31,10 @@ import androidx.media3.decoder.DecoderInputBuffer; public PassthroughSamplePipeline( Format format, long streamStartPositionUs, - long streamOffsetUs, TransformationRequest transformationRequest, MuxerWrapper muxerWrapper, FallbackListener fallbackListener) { - super( - format, - streamStartPositionUs, - streamOffsetUs, - transformationRequest.flattenForSlowMotion, - muxerWrapper); + super(format, streamStartPositionUs, muxerWrapper); this.format = format; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); fallbackListener.onTransformationRequestFinalized(transformationRequest); @@ -51,22 +45,22 @@ import androidx.media3.decoder.DecoderInputBuffer; return false; } - @Override - public void release() {} - @Override @Nullable - protected DecoderInputBuffer dequeueInputBufferInternal() { + public DecoderInputBuffer dequeueInputBuffer() { return hasPendingBuffer ? null : buffer; } @Override - protected void queueInputBufferInternal() { + public void queueInputBuffer() { if (buffer.data != null && buffer.data.hasRemaining()) { hasPendingBuffer = true; } } + @Override + public void release() {} + @Override protected boolean processDataUpToMuxer() { return false; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index b8fbec573a..005581944d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -157,6 +157,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; mediaItem, removeAudio, removeVideo, + transformationRequest.flattenForSlowMotion, mediaSourceFactory, decoderFactory, internalLooper, @@ -497,7 +498,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return new PassthroughSamplePipeline( inputFormat, streamStartPositionUs, - streamOffsetUs, transformationRequest, muxerWrapper, fallbackListener); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 90b3ed58c2..703d1b0130 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -87,13 +87,7 @@ import org.checkerframework.dataflow.qual.Pure; FallbackListener fallbackListener, DebugViewProvider debugViewProvider) throws TransformationException { - super( - inputFormat, - streamStartPositionUs, - streamOffsetUs, - transformationRequest.flattenForSlowMotion, - muxerWrapper); - + super(inputFormat, streamStartPositionUs, muxerWrapper); if (ColorInfo.isTransferHdr(inputFormat.colorInfo)) { if (transformationRequest.hdrMode == TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) { @@ -229,27 +223,27 @@ import org.checkerframework.dataflow.qual.Pure; return false; } - @Override - public void release() { - frameProcessor.release(); - decoder.release(); - encoderWrapper.release(); - } - @Override @Nullable - protected DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException { + public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; } @Override - protected void queueInputBufferInternal() throws TransformationException { + public void queueInputBuffer() throws TransformationException { if (decoderInputBuffer.isDecodeOnly()) { decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs); } decoder.queueInputBuffer(decoderInputBuffer); } + @Override + public void release() { + frameProcessor.release(); + decoder.release(); + encoderWrapper.release(); + } + @Override protected boolean processDataUpToMuxer() throws TransformationException { if (decoder.isEnded()) {