From d8754b6642d0a99e2705f3e21ff8b83d50472bbd Mon Sep 17 00:00:00 2001 From: kimvde Date: Tue, 8 Nov 2022 07:25:42 +0000 Subject: [PATCH] Move muxing inside sample pipelines This logic is currently in the player renderers. With multi-asset, the renderers will go into the AssetLoader, which shouldn't be responsible for muxing. PiperOrigin-RevId: 486860502 --- .../AudioTranscodingSamplePipeline.java | 40 ++++--- .../transformer/BaseSamplePipeline.java | 111 ++++++++++++++++++ .../PassthroughSamplePipeline.java | 26 ++-- .../transformer/SamplePipeline.java | 16 --- .../transformer/TransformerAudioRenderer.java | 10 +- .../transformer/TransformerBaseRenderer.java | 56 +-------- .../transformer/TransformerVideoRenderer.java | 10 +- .../VideoTranscodingSamplePipeline.java | 33 +++--- 8 files changed, 185 insertions(+), 117 deletions(-) create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java index 5f301196f8..2b615d8d27 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java @@ -28,14 +28,13 @@ import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.util.Util; import java.nio.ByteBuffer; -import java.util.List; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them. */ -/* package */ final class AudioTranscodingSamplePipeline implements SamplePipeline { +/* package */ final class AudioTranscodingSamplePipeline extends BaseSamplePipeline { private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; @@ -57,12 +56,15 @@ import org.checkerframework.dataflow.qual.Pure; public AudioTranscodingSamplePipeline( Format inputFormat, long streamOffsetUs, + long streamStartPositionUs, TransformationRequest transformationRequest, Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, - List allowedOutputMimeTypes, + MuxerWrapper muxerWrapper, FallbackListener fallbackListener) throws TransformationException { + super(C.TRACK_TYPE_AUDIO, streamStartPositionUs, muxerWrapper); + decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderInputBuffer = @@ -104,7 +106,9 @@ import org.checkerframework.dataflow.qual.Pure; .setChannelCount(encoderInputAudioFormat.channelCount) .setAverageBitrate(DEFAULT_ENCODER_BITRATE) .build(); - encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat, allowedOutputMimeTypes); + encoder = + encoderFactory.createForAudioEncoding( + requestedOutputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO)); fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( @@ -126,7 +130,16 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public boolean processData() throws TransformationException { + public void release() { + if (speedChangingAudioProcessor != null) { + speedChangingAudioProcessor.reset(); + } + decoder.release(); + encoder.release(); + } + + @Override + protected boolean processDataUpToMuxer() throws TransformationException { if (speedChangingAudioProcessor != null) { return feedEncoderFromProcessor() || feedProcessorFromDecoder(); } else { @@ -136,13 +149,13 @@ import org.checkerframework.dataflow.qual.Pure; @Override @Nullable - public Format getOutputFormat() throws TransformationException { + protected Format getMuxerInputFormat() throws TransformationException { return encoder.getOutputFormat(); } @Override @Nullable - public DecoderInputBuffer getOutputBuffer() throws TransformationException { + protected DecoderInputBuffer getMuxerInputBuffer() throws TransformationException { encoderOutputBuffer.data = encoder.getOutputBuffer(); if (encoderOutputBuffer.data == null) { return null; @@ -153,24 +166,15 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public void releaseOutputBuffer() throws TransformationException { + protected void releaseMuxerInputBuffer() throws TransformationException { encoder.releaseOutputBuffer(/* render= */ false); } @Override - public boolean isEnded() { + protected boolean isMuxerInputEnded() { return encoder.isEnded(); } - @Override - public void release() { - if (speedChangingAudioProcessor != null) { - speedChangingAudioProcessor.reset(); - } - decoder.release(); - encoder.release(); - } - /** * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to * pass more data immediately by calling this method again. diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java new file mode 100644 index 0000000000..ec6d382b68 --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java @@ -0,0 +1,111 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; + +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; + +/* package */ abstract class BaseSamplePipeline implements SamplePipeline { + + private final int trackType; + private final long streamStartPositionUs; + private final MuxerWrapper muxerWrapper; + + private boolean muxerWrapperTrackAdded; + private boolean isEnded; + + public BaseSamplePipeline(int trackType, long streamStartPositionUs, MuxerWrapper muxerWrapper) { + this.trackType = trackType; + this.streamStartPositionUs = streamStartPositionUs; + this.muxerWrapper = muxerWrapper; + } + + @Override + public boolean processData() throws TransformationException { + return feedMuxer() || processDataUpToMuxer(); + } + + @Override + public boolean isEnded() { + return isEnded; + } + + protected abstract boolean processDataUpToMuxer() throws TransformationException; + + @Nullable + protected abstract Format getMuxerInputFormat() throws TransformationException; + + @Nullable + protected abstract DecoderInputBuffer getMuxerInputBuffer() throws TransformationException; + + protected abstract void releaseMuxerInputBuffer() throws TransformationException; + + protected abstract boolean isMuxerInputEnded(); + + /** + * Attempts to pass encoded data to the muxer, and returns whether it may be possible to pass more + * data immediately by calling this method again. + */ + private boolean feedMuxer() throws TransformationException { + if (!muxerWrapperTrackAdded) { + @Nullable Format inputFormat = getMuxerInputFormat(); + if (inputFormat == null) { + return false; + } + try { + muxerWrapper.addTrackFormat(inputFormat); + } catch (Muxer.MuxerException e) { + throw TransformationException.createForMuxer( + e, TransformationException.ERROR_CODE_MUXING_FAILED); + } + muxerWrapperTrackAdded = true; + } + + if (isMuxerInputEnded()) { + muxerWrapper.endTrack(trackType); + isEnded = true; + return false; + } + + @Nullable DecoderInputBuffer muxerInputBuffer = getMuxerInputBuffer(); + if (muxerInputBuffer == null) { + return false; + } + + long samplePresentationTimeUs = muxerInputBuffer.timeUs - streamStartPositionUs; + // TODO(b/204892224): Consider subtracting the first sample timestamp from the sample pipeline + // buffer from all samples so that they are guaranteed to start from zero in the output file. + try { + if (!muxerWrapper.writeSample( + trackType, + checkStateNotNull(muxerInputBuffer.data), + muxerInputBuffer.isKeyFrame(), + samplePresentationTimeUs)) { + return false; + } + } catch (Muxer.MuxerException e) { + throw TransformationException.createForMuxer( + e, TransformationException.ERROR_CODE_MUXING_FAILED); + } + + releaseMuxerInputBuffer(); + return true; + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java index 042819b0fb..8834c7dd43 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java @@ -19,9 +19,10 @@ package com.google.android.exoplayer2.transformer; import androidx.annotation.Nullable; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.util.MimeTypes; /** Pipeline that passes through the samples without any re-encoding or transformation. */ -/* package */ final class PassthroughSamplePipeline implements SamplePipeline { +/* package */ final class PassthroughSamplePipeline extends BaseSamplePipeline { private final DecoderInputBuffer buffer; private final Format format; @@ -30,8 +31,11 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; public PassthroughSamplePipeline( Format format, + long streamStartPositionUs, TransformationRequest transformationRequest, + MuxerWrapper muxerWrapper, FallbackListener fallbackListener) { + super(MimeTypes.getTrackType(format.sampleMimeType), streamStartPositionUs, muxerWrapper); this.format = format; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); hasPendingBuffer = false; @@ -46,36 +50,38 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; @Override public void queueInputBuffer() { - hasPendingBuffer = true; + if (buffer.data != null && buffer.data.hasRemaining()) { + hasPendingBuffer = true; + } } @Override - public boolean processData() { + public void release() {} + + @Override + protected boolean processDataUpToMuxer() { return false; } @Override - public Format getOutputFormat() { + protected Format getMuxerInputFormat() { return format; } @Override @Nullable - public DecoderInputBuffer getOutputBuffer() { + protected DecoderInputBuffer getMuxerInputBuffer() { return hasPendingBuffer ? buffer : null; } @Override - public void releaseOutputBuffer() { + protected void releaseMuxerInputBuffer() { buffer.clear(); hasPendingBuffer = false; } @Override - public boolean isEnded() { + protected boolean isMuxerInputEnded() { return buffer.isEndOfStream(); } - - @Override - public void release() {} } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java index dc835406d8..7d592bbc84 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java @@ -17,7 +17,6 @@ package com.google.android.exoplayer2.transformer; import androidx.annotation.Nullable; -import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; /** @@ -45,21 +44,6 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; */ boolean processData() throws TransformationException; - /** Returns the output format of the pipeline if available, and {@code null} otherwise. */ - @Nullable - Format getOutputFormat() throws TransformationException; - - /** Returns an output buffer if the pipeline has produced output, and {@code null} otherwise */ - @Nullable - DecoderInputBuffer getOutputBuffer() throws TransformationException; - - /** - * Releases the pipeline's output buffer. - * - *

Should be called when the output buffer from {@link #getOutputBuffer()} is no longer needed. - */ - void releaseOutputBuffer() throws TransformationException; - /** Returns whether the pipeline has ended. */ boolean isEnded(); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java index 82e9698c4f..3c50c19993 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java @@ -77,16 +77,22 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; Format inputFormat = checkNotNull(formatHolder.format); if (shouldPassthrough(inputFormat)) { samplePipeline = - new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); + new PassthroughSamplePipeline( + inputFormat, + streamStartPositionUs, + transformationRequest, + muxerWrapper, + fallbackListener); } else { samplePipeline = new AudioTranscodingSamplePipeline( inputFormat, streamOffsetUs, + streamStartPositionUs, transformationRequest, decoderFactory, encoderFactory, - muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), + muxerWrapper, fallbackListener); } return true; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java index 0e05a16f2f..05a1ef267b 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java @@ -41,8 +41,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; protected final FallbackListener fallbackListener; private boolean isTransformationRunning; - private boolean muxerWrapperTrackAdded; - private boolean muxerWrapperTrackEnded; protected long streamOffsetUs; protected long streamStartPositionUs; protected @MonotonicNonNull SamplePipeline samplePipeline; @@ -88,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override public final boolean isEnded() { - return muxerWrapperTrackEnded; + return samplePipeline != null && samplePipeline.isEnded(); } @Override @@ -98,15 +96,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return; } - while (feedMuxerFromPipeline() || samplePipeline.processData() || feedPipelineFromInput()) {} + while (samplePipeline.processData() || feedPipelineFromInput()) {} } catch (TransformationException e) { isTransformationRunning = false; asyncErrorListener.onTransformationException(e); - } catch (Muxer.MuxerException e) { - isTransformationRunning = false; - asyncErrorListener.onTransformationException( - TransformationException.createForMuxer( - e, TransformationException.ERROR_CODE_MUXING_FAILED)); } } @@ -138,8 +131,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; if (samplePipeline != null) { samplePipeline.release(); } - muxerWrapperTrackAdded = false; - muxerWrapperTrackEnded = false; } @ForOverride @@ -152,49 +143,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; samplePipeline.queueInputBuffer(); } - /** - * Attempts to write sample pipeline output data to the muxer. - * - * @return Whether it may be possible to write more data immediately by calling this method again. - * @throws Muxer.MuxerException If a muxing problem occurs. - * @throws TransformationException If a {@link SamplePipeline} problem occurs. - */ - @RequiresNonNull("samplePipeline") - private boolean feedMuxerFromPipeline() throws Muxer.MuxerException, TransformationException { - if (!muxerWrapperTrackAdded) { - @Nullable Format samplePipelineOutputFormat = samplePipeline.getOutputFormat(); - if (samplePipelineOutputFormat == null) { - return false; - } - muxerWrapperTrackAdded = true; - muxerWrapper.addTrackFormat(samplePipelineOutputFormat); - } - - if (samplePipeline.isEnded()) { - muxerWrapper.endTrack(getTrackType()); - muxerWrapperTrackEnded = true; - return false; - } - - @Nullable DecoderInputBuffer samplePipelineOutputBuffer = samplePipeline.getOutputBuffer(); - if (samplePipelineOutputBuffer == null) { - return false; - } - - long samplePresentationTimeUs = samplePipelineOutputBuffer.timeUs - streamStartPositionUs; - // TODO(b/204892224): Consider subtracting the first sample timestamp from the sample pipeline - // buffer from all samples so that they are guaranteed to start from zero in the output file. - if (!muxerWrapper.writeSample( - getTrackType(), - checkStateNotNull(samplePipelineOutputBuffer.data), - samplePipelineOutputBuffer.isKeyFrame(), - samplePresentationTimeUs)) { - return false; - } - samplePipeline.releaseOutputBuffer(); - return true; - } - /** * Attempts to read input data and pass the input data to the sample pipeline. * diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java index 3170d52858..38c33fd32f 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java @@ -103,18 +103,24 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; context, inputFormat, streamOffsetUs, + streamStartPositionUs, transformationRequest, effects, frameProcessorFactory, decoderFactory, encoderFactory, - muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), + muxerWrapper, fallbackListener, asyncErrorListener, debugViewProvider); } else { samplePipeline = - new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); + new PassthroughSamplePipeline( + inputFormat, + streamStartPositionUs, + transformationRequest, + muxerWrapper, + fallbackListener); } if (transformationRequest.flattenForSlowMotion) { sefSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index dd655aefdf..3e8e09821a 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -50,7 +50,7 @@ import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. */ -/* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline { +/* package */ final class VideoTranscodingSamplePipeline extends BaseSamplePipeline { private final int maxPendingFrameCount; @@ -67,16 +67,19 @@ import org.checkerframework.dataflow.qual.Pure; Context context, Format inputFormat, long streamOffsetUs, + long streamStartPositionUs, TransformationRequest transformationRequest, ImmutableList effects, FrameProcessor.Factory frameProcessorFactory, Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, - List allowedOutputMimeTypes, + MuxerWrapper muxerWrapper, FallbackListener fallbackListener, Transformer.AsyncErrorListener asyncErrorListener, DebugViewProvider debugViewProvider) throws TransformationException { + super(C.TRACK_TYPE_VIDEO, streamStartPositionUs, muxerWrapper); + if (ColorInfo.isTransferHdr(inputFormat.colorInfo) && (SDK_INT < 31 || deviceNeedsNoToneMappingWorkaround())) { throw TransformationException.createForCodec( @@ -119,7 +122,7 @@ import org.checkerframework.dataflow.qual.Pure; new EncoderWrapper( encoderFactory, inputFormat, - allowedOutputMimeTypes, + muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_VIDEO), transformationRequest, fallbackListener); @@ -199,7 +202,14 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public boolean processData() throws TransformationException { + public void release() { + frameProcessor.release(); + decoder.release(); + encoderWrapper.release(); + } + + @Override + protected boolean processDataUpToMuxer() throws TransformationException { if (decoder.isEnded()) { return false; } @@ -217,13 +227,13 @@ import org.checkerframework.dataflow.qual.Pure; @Override @Nullable - public Format getOutputFormat() throws TransformationException { + protected Format getMuxerInputFormat() throws TransformationException { return encoderWrapper.getOutputFormat(); } @Override @Nullable - public DecoderInputBuffer getOutputBuffer() throws TransformationException { + protected DecoderInputBuffer getMuxerInputBuffer() throws TransformationException { encoderOutputBuffer.data = encoderWrapper.getOutputBuffer(); if (encoderOutputBuffer.data == null) { return null; @@ -235,22 +245,15 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public void releaseOutputBuffer() throws TransformationException { + protected void releaseMuxerInputBuffer() throws TransformationException { encoderWrapper.releaseOutputBuffer(/* render= */ false); } @Override - public boolean isEnded() { + protected boolean isMuxerInputEnded() { return encoderWrapper.isEnded(); } - @Override - public void release() { - frameProcessor.release(); - decoder.release(); - encoderWrapper.release(); - } - /** * Creates a {@link TransformationRequest}, based on an original {@code TransformationRequest} and * parameters specifying alterations to it that indicate device support.