diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java index 4e2d00bf4a..25793d6234 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java @@ -41,14 +41,14 @@ import org.checkerframework.dataflow.qual.Pure; private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; - private final Codec decoder; - private final DecoderInputBuffer decoderInputBuffer; + private final DecoderInputBuffer inputBuffer; private final AudioProcessingPipeline audioProcessingPipeline; private final Codec encoder; private final AudioFormat encoderInputAudioFormat; private final DecoderInputBuffer encoderInputBuffer; private final DecoderInputBuffer encoderOutputBuffer; + private boolean hasPendingInputBuffer; private long nextEncoderInputBufferTimeUs; private long encoderBufferDurationRemainder; @@ -58,7 +58,6 @@ import org.checkerframework.dataflow.qual.Pure; long streamOffsetUs, TransformationRequest transformationRequest, ImmutableList audioProcessors, - Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, MuxerWrapper muxerWrapper, Listener listener, @@ -72,12 +71,10 @@ import org.checkerframework.dataflow.qual.Pure; muxerWrapper, listener); - decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); + inputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); encoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); encoderOutputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); - decoder = decoderFactory.createForAudioDecoding(inputFormat); - if (transformationRequest.flattenForSlowMotion) { audioProcessors = new ImmutableList.Builder() @@ -139,30 +136,34 @@ import org.checkerframework.dataflow.qual.Pure; nextEncoderInputBufferTimeUs = streamOffsetUs; } + @Override + public boolean expectsDecodedData() { + return true; + } + @Override public void release() { audioProcessingPipeline.reset(); - decoder.release(); encoder.release(); } @Override @Nullable - protected DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException { - return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; + protected DecoderInputBuffer dequeueInputBufferInternal() { + return hasPendingInputBuffer ? null : inputBuffer; } @Override - protected void queueInputBufferInternal() throws TransformationException { - decoder.queueInputBuffer(decoderInputBuffer); + protected void queueInputBufferInternal() { + hasPendingInputBuffer = true; } @Override protected boolean processDataUpToMuxer() throws TransformationException { if (audioProcessingPipeline.isOperational()) { - return feedEncoderFromProcessingPipeline() || feedProcessingPipelineFromDecoder(); + return feedEncoderFromProcessingPipeline() || feedProcessingPipelineFromInput(); } else { - return feedEncoderFromDecoder(); + return feedEncoderFromInput(); } } @@ -195,27 +196,25 @@ import org.checkerframework.dataflow.qual.Pure; } /** - * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to - * pass more data immediately by calling this method again. + * Attempts to pass input data to the encoder. + * + * @return Whether it may be possible to feed more data immediately by calling this method again. */ - private boolean feedEncoderFromDecoder() throws TransformationException { - if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { + private boolean feedEncoderFromInput() throws TransformationException { + if (!hasPendingInputBuffer || !encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { return false; } - if (decoder.isEnded()) { + if (inputBuffer.isEndOfStream()) { queueEndOfStreamToEncoder(); + hasPendingInputBuffer = false; return false; } - @Nullable ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer(); - if (decoderOutputBuffer == null) { - return false; - } - - feedEncoder(decoderOutputBuffer); - if (!decoderOutputBuffer.hasRemaining()) { - decoder.releaseOutputBuffer(/* render= */ false); + ByteBuffer inputData = checkNotNull(inputBuffer.data); + feedEncoder(inputData); + if (!inputData.hasRemaining()) { + hasPendingInputBuffer = false; } return true; } @@ -223,7 +222,7 @@ import org.checkerframework.dataflow.qual.Pure; /** * Attempts to feed audio processor output data to the encoder. * - * @return Whether more data can be fed immediately, by calling this method again. + * @return Whether it may be possible to feed more data immediately by calling this method again. */ private boolean feedEncoderFromProcessingPipeline() throws TransformationException { if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { @@ -244,28 +243,28 @@ import org.checkerframework.dataflow.qual.Pure; } /** - * Attempts to feed decoder output data to the {@link AudioProcessingPipeline}. + * Attempts to feed input data to the {@link AudioProcessingPipeline}. * * @return Whether it may be possible to feed more data immediately by calling this method again. */ - private boolean feedProcessingPipelineFromDecoder() throws TransformationException { - if (decoder.isEnded()) { + private boolean feedProcessingPipelineFromInput() { + if (!hasPendingInputBuffer) { + return false; + } + + if (inputBuffer.isEndOfStream()) { audioProcessingPipeline.queueEndOfStream(); + hasPendingInputBuffer = false; return false; } checkState(!audioProcessingPipeline.isEnded()); - @Nullable ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer(); - if (decoderOutputBuffer == null) { + ByteBuffer inputData = checkNotNull(inputBuffer.data); + audioProcessingPipeline.queueInput(inputData); + if (inputData.hasRemaining()) { return false; } - - audioProcessingPipeline.queueInput(decoderOutputBuffer); - if (decoderOutputBuffer.hasRemaining()) { - return false; - } - // Decoder output buffer was fully consumed by the processing pipeline. - decoder.releaseOutputBuffer(/* render= */ false); + hasPendingInputBuffer = false; return true; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java index e7cd809942..e3f90abcfe 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java @@ -69,6 +69,7 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; boolean removeAudio, boolean removeVideo, MediaSource.Factory mediaSourceFactory, + Codec.DecoderFactory decoderFactory, Looper looper, Listener listener, Clock clock) { @@ -89,7 +90,9 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10) .build(); ExoPlayer.Builder playerBuilder = - new ExoPlayer.Builder(context, new RenderersFactoryImpl(removeAudio, removeVideo, listener)) + new ExoPlayer.Builder( + context, + new RenderersFactoryImpl(removeAudio, removeVideo, decoderFactory, listener)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) .setLoadControl(loadControl) @@ -120,14 +123,17 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; private final TransformerMediaClock mediaClock; private final boolean removeAudio; private final boolean removeVideo; + private final Codec.DecoderFactory decoderFactory; private final ExoPlayerAssetLoader.Listener assetLoaderListener; public RenderersFactoryImpl( boolean removeAudio, boolean removeVideo, + Codec.DecoderFactory decoderFactory, ExoPlayerAssetLoader.Listener assetLoaderListener) { this.removeAudio = removeAudio; this.removeVideo = removeVideo; + this.decoderFactory = decoderFactory; this.assetLoaderListener = assetLoaderListener; mediaClock = new TransformerMediaClock(); } @@ -144,12 +150,14 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; int index = 0; if (!removeAudio) { renderers[index] = - new ExoPlayerAssetLoaderRenderer(C.TRACK_TYPE_AUDIO, mediaClock, assetLoaderListener); + new ExoPlayerAssetLoaderRenderer( + C.TRACK_TYPE_AUDIO, decoderFactory, mediaClock, assetLoaderListener); index++; } if (!removeVideo) { renderers[index] = - new ExoPlayerAssetLoaderRenderer(C.TRACK_TYPE_VIDEO, mediaClock, assetLoaderListener); + new ExoPlayerAssetLoaderRenderer( + C.TRACK_TYPE_VIDEO, decoderFactory, mediaClock, assetLoaderListener); index++; } return renderers; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java index e541ed933f..0fa0d3e3f7 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED; import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; +import android.media.MediaCodec; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; @@ -30,6 +31,7 @@ import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.MediaClock; import androidx.media3.exoplayer.RendererCapabilities; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; +import java.nio.ByteBuffer; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -38,6 +40,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private static final String TAG = "ExoPlayerAssetLoaderRenderer"; + private final Codec.DecoderFactory decoderFactory; private final TransformerMediaClock mediaClock; private final ExoPlayerAssetLoader.Listener assetLoaderListener; private final DecoderInputBuffer decoderInputBuffer; @@ -45,14 +48,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private boolean isTransformationRunning; private long streamStartPositionUs; private long streamOffsetUs; + private @MonotonicNonNull Codec decoder; + @Nullable private ByteBuffer pendingDecoderOutputBuffer; private SamplePipeline.@MonotonicNonNull Input samplePipelineInput; private boolean isEnded; public ExoPlayerAssetLoaderRenderer( int trackType, + Codec.DecoderFactory decoderFactory, TransformerMediaClock mediaClock, ExoPlayerAssetLoader.Listener assetLoaderListener) { super(trackType); + this.decoderFactory = decoderFactory; this.mediaClock = mediaClock; this.assetLoaderListener = assetLoaderListener; decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); @@ -99,7 +106,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return; } - while (feedPipelineFromInput()) {} + if (samplePipelineInput.expectsDecodedData()) { + while (feedPipelineFromDecoder() || feedDecoderFromInput()) {} + } else { + while (feedPipelineFromInput()) {} + } } catch (TransformationException e) { isTransformationRunning = false; assetLoaderListener.onError(e); @@ -128,6 +139,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; isTransformationRunning = false; } + @Override + protected void onReset() { + if (decoder != null) { + decoder.release(); + } + } + @EnsuresNonNullIf(expression = "samplePipelineInput", result = true) private boolean ensureConfigured() throws TransformationException { if (samplePipelineInput != null) { @@ -143,6 +161,74 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; Format inputFormat = checkNotNull(formatHolder.format); samplePipelineInput = assetLoaderListener.onTrackAdded(inputFormat, streamStartPositionUs, streamOffsetUs); + if (samplePipelineInput.expectsDecodedData()) { + decoder = decoderFactory.createForAudioDecoding(inputFormat); + } + return true; + } + + /** + * Attempts to read decoded data and pass it to the sample pipeline. + * + * @return Whether it may be possible to read more data immediately by calling this method again. + * @throws TransformationException If an error occurs in the decoder or in the {@link + * SamplePipeline}. + */ + @RequiresNonNull("samplePipelineInput") + private boolean feedPipelineFromDecoder() throws TransformationException { + @Nullable + DecoderInputBuffer samplePipelineInputBuffer = samplePipelineInput.dequeueInputBuffer(); + if (samplePipelineInputBuffer == null) { + return false; + } + + Codec decoder = checkNotNull(this.decoder); + if (pendingDecoderOutputBuffer != null) { + if (pendingDecoderOutputBuffer.hasRemaining()) { + return false; + } else { + decoder.releaseOutputBuffer(/* render= */ false); + pendingDecoderOutputBuffer = null; + } + } + + if (decoder.isEnded()) { + samplePipelineInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); + samplePipelineInput.queueInputBuffer(); + isEnded = true; + return false; + } + + pendingDecoderOutputBuffer = decoder.getOutputBuffer(); + if (pendingDecoderOutputBuffer == null) { + return false; + } + + samplePipelineInputBuffer.data = pendingDecoderOutputBuffer; + MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo()); + samplePipelineInputBuffer.timeUs = bufferInfo.presentationTimeUs; + samplePipelineInputBuffer.setFlags(bufferInfo.flags); + samplePipelineInput.queueInputBuffer(); + return true; + } + + /** + * Attempts to read input data and pass it to the decoder. + * + * @return Whether it may be possible to read more data immediately by calling this method again. + * @throws TransformationException If an error occurs in the decoder. + */ + private boolean feedDecoderFromInput() throws TransformationException { + Codec decoder = checkNotNull(this.decoder); + if (!decoder.maybeDequeueInputBuffer(decoderInputBuffer)) { + return false; + } + + if (!readInput(decoderInputBuffer)) { + return false; + } + + decoder.queueInputBuffer(decoderInputBuffer); return true; } @@ -159,18 +245,32 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } - @ReadDataResult - int result = readSource(getFormatHolder(), samplePipelineInputBuffer, /* readFlags= */ 0); + if (!readInput(samplePipelineInputBuffer)) { + return false; + } + + samplePipelineInput.queueInputBuffer(); + if (samplePipelineInputBuffer.isEndOfStream()) { + isEnded = true; + return false; + } + return true; + } + + /** + * Attempts to populate {@code buffer} with input data. + * + * @param buffer The buffer to populate. + * @return Whether the {@code buffer} has been populated. + */ + private boolean readInput(DecoderInputBuffer buffer) { + @ReadDataResult int result = readSource(getFormatHolder(), buffer, /* readFlags= */ 0); switch (result) { case C.RESULT_BUFFER_READ: - samplePipelineInputBuffer.flip(); - if (samplePipelineInputBuffer.isEndOfStream()) { - samplePipelineInput.queueInputBuffer(); - isEnded = true; - return false; + buffer.flip(); + if (!buffer.isEndOfStream()) { + mediaClock.updateTimeForTrackType(getTrackType(), buffer.timeUs); } - mediaClock.updateTimeForTrackType(getTrackType(), samplePipelineInputBuffer.timeUs); - samplePipelineInput.queueInputBuffer(); return true; case C.RESULT_FORMAT_READ: throw new IllegalStateException("Format changes are not supported."); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java index 561ec231cd..de49af02dd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java @@ -48,6 +48,11 @@ import androidx.media3.decoder.DecoderInputBuffer; fallbackListener.onTransformationRequestFinalized(transformationRequest); } + @Override + public boolean expectsDecodedData() { + return false; + } + @Override public void release() {} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java index a93a32952f..b2f4823571 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java @@ -29,6 +29,9 @@ import androidx.media3.decoder.DecoderInputBuffer; /** Input of a {@link SamplePipeline}. */ interface Input { + /** See {@link SamplePipeline#expectsDecodedData()}. */ + boolean expectsDecodedData(); + /** See {@link SamplePipeline#dequeueInputBuffer()}. */ @Nullable DecoderInputBuffer dequeueInputBuffer(); @@ -56,6 +59,12 @@ import androidx.media3.decoder.DecoderInputBuffer; void onTransformationError(TransformationException exception); } + /** + * Returns whether the pipeline should be fed with decoded sample data. If false, encoded sample + * data should be queued. + */ + boolean expectsDecodedData(); + /** Returns a buffer if the pipeline is ready to accept input, and {@code null} otherwise. */ @Nullable DecoderInputBuffer dequeueInputBuffer() throws TransformationException; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 4762387c29..72db3d826f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -163,6 +163,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; removeAudio, removeVideo, mediaSourceFactory, + decoderFactory, internalLooper, componentListener, clock); @@ -409,7 +410,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int samplePipelineIndex = tracksAddedCount; tracksAddedCount++; - return new SamplePipelineInput(samplePipelineIndex); + return new SamplePipelineInput(samplePipelineIndex, samplePipeline.expectsDecodedData()); } @Override @@ -458,7 +459,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, transformationRequest, audioProcessors, - decoderFactory, encoderFactory, muxerWrapper, /* listener= */ this, @@ -573,9 +573,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private class SamplePipelineInput implements SamplePipeline.Input { private final int samplePipelineIndex; + private final boolean expectsDecodedData; - public SamplePipelineInput(int samplePipelineIndex) { + public SamplePipelineInput(int samplePipelineIndex, boolean expectsDecodedData) { this.samplePipelineIndex = samplePipelineIndex; + this.expectsDecodedData = expectsDecodedData; + } + + @Override + public boolean expectsDecodedData() { + return expectsDecodedData; } @Nullable diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 21f742f91f..6db43407a3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -207,6 +207,11 @@ import org.checkerframework.dataflow.qual.Pure; maxPendingFrameCount = decoder.getMaxPendingFrameCount(); } + @Override + public boolean expectsDecodedData() { + return false; + } + @Override public void release() { frameProcessor.release();