From 714e5565053ad48ec23b1f1640851a1e784a6726 Mon Sep 17 00:00:00 2001 From: kimvde Date: Wed, 9 Nov 2022 08:25:53 +0000 Subject: [PATCH] Move slow mo logic to sample pipelines This is to avoid having this logic in TransformerInternal once it is added. PiperOrigin-RevId: 487159941 --- .../AudioTranscodingSamplePipeline.java | 29 ++++---- .../transformer/BaseSamplePipeline.java | 71 ++++++++++++++++++- .../PassthroughSamplePipeline.java | 20 +++--- .../transformer/TransformerAudioRenderer.java | 1 + .../transformer/TransformerBaseRenderer.java | 11 +-- .../transformer/TransformerVideoRenderer.java | 37 +--------- .../VideoTranscodingSamplePipeline.java | 35 +++++---- 7 files changed, 120 insertions(+), 84 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java index eac7200838..17259608fd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java @@ -63,7 +63,12 @@ import org.checkerframework.dataflow.qual.Pure; MuxerWrapper muxerWrapper, FallbackListener fallbackListener) throws TransformationException { - super(C.TRACK_TYPE_AUDIO, streamStartPositionUs, muxerWrapper); + super( + inputFormat, + streamOffsetUs, + streamStartPositionUs, + transformationRequest.flattenForSlowMotion, + muxerWrapper); decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); @@ -118,17 +123,6 @@ import org.checkerframework.dataflow.qual.Pure; nextEncoderInputBufferTimeUs = streamOffsetUs; } - @Override - @Nullable - public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { - return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; - } - - @Override - public void queueInputBuffer() throws TransformationException { - decoder.queueInputBuffer(decoderInputBuffer); - } - @Override public void release() { if (speedChangingAudioProcessor != null) { @@ -138,6 +132,17 @@ import org.checkerframework.dataflow.qual.Pure; encoder.release(); } + @Override + @Nullable + protected DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException { + return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; + } + + @Override + protected void queueInputBufferInternal() throws TransformationException { + decoder.queueInputBuffer(decoderInputBuffer); + } + @Override protected boolean processDataUpToMuxer() throws TransformationException { if (speedChangingAudioProcessor != null) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java index 62b2097f06..ff6e94aae2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java @@ -16,25 +16,60 @@ package androidx.media3.transformer; +import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; import androidx.annotation.Nullable; +import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.decoder.DecoderInputBuffer; +import java.nio.ByteBuffer; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; /* package */ abstract class BaseSamplePipeline implements SamplePipeline { - private final int trackType; + private final long streamOffsetUs; private final long streamStartPositionUs; private final MuxerWrapper muxerWrapper; + private final @C.TrackType int trackType; + private final @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener; + @Nullable private DecoderInputBuffer inputBuffer; private boolean muxerWrapperTrackAdded; private boolean isEnded; - public BaseSamplePipeline(int trackType, long streamStartPositionUs, MuxerWrapper muxerWrapper) { - this.trackType = trackType; + public BaseSamplePipeline( + Format inputFormat, + long streamOffsetUs, + long streamStartPositionUs, + boolean flattenForSlowMotion, + MuxerWrapper muxerWrapper) { + this.streamOffsetUs = streamOffsetUs; this.streamStartPositionUs = streamStartPositionUs; this.muxerWrapper = muxerWrapper; + trackType = MimeTypes.getTrackType(inputFormat.sampleMimeType); + sefVideoSlowMotionFlattener = + flattenForSlowMotion && trackType == C.TRACK_TYPE_VIDEO + ? new SefSlowMotionFlattener(inputFormat) + : null; + } + + @Nullable + @Override + public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { + inputBuffer = dequeueInputBufferInternal(); + return inputBuffer; + } + + @Override + public void queueInputBuffer() throws TransformationException { + checkNotNull(inputBuffer); + checkNotNull(inputBuffer.data); + if (!shouldDropInputBuffer()) { + queueInputBufferInternal(); + } } @Override @@ -47,6 +82,11 @@ import androidx.media3.decoder.DecoderInputBuffer; return isEnded; } + @Nullable + protected abstract DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException; + + protected abstract void queueInputBufferInternal() throws TransformationException; + protected abstract boolean processDataUpToMuxer() throws TransformationException; @Nullable @@ -59,6 +99,31 @@ import androidx.media3.decoder.DecoderInputBuffer; protected abstract boolean isMuxerInputEnded(); + /** + * Preprocesses an {@linkplain DecoderInputBuffer input buffer} queued to the pipeline and returns + * whether it should be dropped. + */ + @RequiresNonNull({"inputBuffer", "inputBuffer.data"}) + private boolean shouldDropInputBuffer() { + ByteBuffer inputBytes = inputBuffer.data; + + if (sefVideoSlowMotionFlattener == null || inputBuffer.isEndOfStream()) { + return false; + } + + long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs; + DecoderInputBuffer inputBuffer = this.inputBuffer; + boolean shouldDropInputBuffer = + sefVideoSlowMotionFlattener.dropOrTransformSample(inputBytes, presentationTimeUs); + if (shouldDropInputBuffer) { + inputBytes.clear(); + } else { + inputBuffer.timeUs = + streamOffsetUs + sefVideoSlowMotionFlattener.getSamplePresentationTimeUs(); + } + return shouldDropInputBuffer; + } + /** * Attempts to pass encoded data to the muxer, and returns whether it may be possible to pass more * data immediately by calling this method again. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java index 00eea6f90a..a49927a929 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java @@ -18,7 +18,6 @@ package androidx.media3.transformer; import androidx.annotation.Nullable; import androidx.media3.common.Format; -import androidx.media3.common.MimeTypes; import androidx.media3.decoder.DecoderInputBuffer; /** Pipeline that passes through the samples without any re-encoding or transformation. */ @@ -31,33 +30,38 @@ import androidx.media3.decoder.DecoderInputBuffer; public PassthroughSamplePipeline( Format format, + long streamOffsetUs, long streamStartPositionUs, TransformationRequest transformationRequest, MuxerWrapper muxerWrapper, FallbackListener fallbackListener) { - super(MimeTypes.getTrackType(format.sampleMimeType), streamStartPositionUs, muxerWrapper); + super( + format, + streamOffsetUs, + streamStartPositionUs, + transformationRequest.flattenForSlowMotion, + muxerWrapper); this.format = format; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); - hasPendingBuffer = false; fallbackListener.onTransformationRequestFinalized(transformationRequest); } + @Override + public void release() {} + @Override @Nullable - public DecoderInputBuffer dequeueInputBuffer() { + protected DecoderInputBuffer dequeueInputBufferInternal() { return hasPendingBuffer ? null : buffer; } @Override - public void queueInputBuffer() { + protected void queueInputBufferInternal() { if (buffer.data != null && buffer.data.hasRemaining()) { hasPendingBuffer = true; } } - @Override - public void release() {} - @Override protected boolean processDataUpToMuxer() { return false; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java index 732c06995f..f998f4a269 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java @@ -79,6 +79,7 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData; samplePipeline = new PassthroughSamplePipeline( inputFormat, + streamOffsetUs, streamStartPositionUs, transformationRequest, muxerWrapper, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java index aa27f48486..0a54b43d77 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java @@ -16,8 +16,6 @@ package androidx.media3.transformer; -import static androidx.media3.common.util.Assertions.checkStateNotNull; - import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; @@ -137,12 +135,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @EnsuresNonNullIf(expression = "samplePipeline", result = true) protected abstract boolean ensureConfigured() throws TransformationException; - @RequiresNonNull({"samplePipeline", "#1.data"}) - protected void maybeQueueSampleToPipeline(DecoderInputBuffer inputBuffer) - throws TransformationException { - samplePipeline.queueInputBuffer(); - } - /** * Attempts to read input data and pass the input data to the sample pipeline. * @@ -166,8 +158,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } mediaClock.updateTimeForTrackType(getTrackType(), samplePipelineInputBuffer.timeUs); - checkStateNotNull(samplePipelineInputBuffer.data); - maybeQueueSampleToPipeline(samplePipelineInputBuffer); + samplePipeline.queueInputBuffer(); return true; case C.RESULT_FORMAT_READ: throw new IllegalStateException("Format changes are not supported."); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index 177730f475..3091cafc37 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -29,9 +29,6 @@ import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; import com.google.common.collect.ImmutableList; -import java.nio.ByteBuffer; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; /* package */ final class TransformerVideoRenderer extends TransformerBaseRenderer { @@ -46,8 +43,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final DebugViewProvider debugViewProvider; private final DecoderInputBuffer decoderInputBuffer; - private @MonotonicNonNull SefSlowMotionFlattener sefSlowMotionFlattener; - public TransformerVideoRenderer( Context context, MuxerWrapper muxerWrapper, @@ -117,14 +112,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; samplePipeline = new PassthroughSamplePipeline( inputFormat, + streamOffsetUs, streamStartPositionUs, transformationRequest, muxerWrapper, fallbackListener); } - if (transformationRequest.flattenForSlowMotion) { - sefSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat); - } return true; } @@ -170,32 +163,4 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } return false; } - - /** - * Queues the input buffer to the sample pipeline unless it should be dropped because of slow - * motion flattening. - * - * @param inputBuffer The {@link DecoderInputBuffer}. - * @throws TransformationException If a {@link SamplePipeline} problem occurs. - */ - @Override - @RequiresNonNull({"samplePipeline", "#1.data"}) - protected void maybeQueueSampleToPipeline(DecoderInputBuffer inputBuffer) - throws TransformationException { - if (sefSlowMotionFlattener == null) { - samplePipeline.queueInputBuffer(); - return; - } - - ByteBuffer data = inputBuffer.data; - long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs; - boolean shouldDropSample = - sefSlowMotionFlattener.dropOrTransformSample(data, presentationTimeUs); - inputBuffer.timeUs = streamOffsetUs + sefSlowMotionFlattener.getSamplePresentationTimeUs(); - if (shouldDropSample) { - data.clear(); - } else { - samplePipeline.queueInputBuffer(); - } - } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index e89b515e77..a791df7d4f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -78,7 +78,12 @@ import org.checkerframework.dataflow.qual.Pure; Transformer.AsyncErrorListener asyncErrorListener, DebugViewProvider debugViewProvider) throws TransformationException { - super(C.TRACK_TYPE_VIDEO, streamStartPositionUs, muxerWrapper); + super( + inputFormat, + streamOffsetUs, + streamStartPositionUs, + transformationRequest.flattenForSlowMotion, + muxerWrapper); if (ColorInfo.isTransferHdr(inputFormat.colorInfo) && (SDK_INT < 31 || deviceNeedsNoToneMappingWorkaround())) { @@ -187,20 +192,6 @@ import org.checkerframework.dataflow.qual.Pure; maxPendingFrameCount = decoder.getMaxPendingFrameCount(); } - @Override - @Nullable - public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { - return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; - } - - @Override - public void queueInputBuffer() throws TransformationException { - if (decoderInputBuffer.isDecodeOnly()) { - decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs); - } - decoder.queueInputBuffer(decoderInputBuffer); - } - @Override public void release() { frameProcessor.release(); @@ -208,6 +199,20 @@ import org.checkerframework.dataflow.qual.Pure; encoderWrapper.release(); } + @Override + @Nullable + protected DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException { + return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; + } + + @Override + protected void queueInputBufferInternal() throws TransformationException { + if (decoderInputBuffer.isDecodeOnly()) { + decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs); + } + decoder.queueInputBuffer(decoderInputBuffer); + } + @Override protected boolean processDataUpToMuxer() throws TransformationException { if (decoder.isEnded()) {