diff --git a/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java index 5b29c4adc9..4dedaf4685 100644 --- a/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java @@ -120,7 +120,11 @@ public interface FrameProcessor { /** Indicates the frame should be dropped after {@link #releaseOutputFrame(long)} is invoked. */ long DROP_OUTPUT_FRAME = -2; - /** Returns the input {@link Surface}, where {@link FrameProcessor} consumes input frames from. */ + /** + * Returns the input {@link Surface}, where {@link FrameProcessor} consumes input frames from. + * + *

Can be called on any thread. + */ Surface getInputSurface(); /** @@ -142,6 +146,8 @@ public interface FrameProcessor { * *

Must be called before rendering a frame to the frame processor's input surface. * + *

Can be called on any thread. + * * @throws IllegalStateException If called after {@link #signalEndOfInput()} or before {@link * #setInputFrameInfo(FrameInfo)}. */ @@ -150,6 +156,8 @@ public interface FrameProcessor { /** * Returns the number of input frames that have been {@linkplain #registerInputFrame() registered} * but not processed off the {@linkplain #getInputSurface() input surface} yet. + * + *

Can be called on any thread. */ int getPendingInputFrameCount(); @@ -194,6 +202,8 @@ public interface FrameProcessor { /** * Informs the {@code FrameProcessor} that no further input frames should be accepted. * + *

Can be called on any thread. + * * @throws IllegalStateException If called more than once. */ void signalEndOfInput(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java index ca7f16fe88..4c9dc12969 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java @@ -334,14 +334,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { private final FinalMatrixTextureProcessorWrapper finalTextureProcessorWrapper; private final ImmutableList allTextureProcessors; - private @MonotonicNonNull FrameInfo nextInputFrameInfo; - private boolean inputStreamEnded; /** * Offset compared to original media presentation time that has been added to incoming frame * timestamps, in microseconds. */ private long previousStreamOffsetUs; + private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; + private volatile boolean inputStreamEnded; + private GlEffectsFrameProcessor( EGLDisplay eglDisplay, EGLContext eglContext, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java index 87413ed229..82b55cf371 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java @@ -149,12 +149,11 @@ public interface AssetLoader { * streamOffsetUs}), in microseconds. * @param streamOffsetUs The offset that will be added to the timestamps to make sure they are * non-negative, in microseconds. - * @return The {@link SamplePipeline.Input} describing the type of sample data expected, and to - * which to pass this data. - * @throws TransformationException If an error occurs configuring the {@link - * SamplePipeline.Input}. + * @return The {@link SampleConsumer} describing the type of sample data expected, and to which + * to pass this data. + * @throws TransformationException If an error occurs configuring the {@link SampleConsumer}. */ - SamplePipeline.Input onTrackAdded( + SampleConsumer onTrackAdded( Format format, @SupportedOutputTypes int supportedOutputTypes, long streamStartPositionUs, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java index 54dc8ce39c..82608dbe29 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java @@ -35,7 +35,7 @@ import java.util.List; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.dataflow.qual.Pure; -/** Pipeline to apply audio processing to raw audio samples, encode them and mux them. */ +/** Pipeline to process, re-encode and mux raw audio samples. */ /* package */ final class AudioTranscodingSamplePipeline extends BaseSamplePipeline { private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; @@ -137,11 +137,6 @@ import org.checkerframework.dataflow.qual.Pure; nextEncoderInputBufferTimeUs = streamOffsetUs; } - @Override - public boolean expectsDecodedData() { - return true; - } - @Override @Nullable public DecoderInputBuffer dequeueInputBuffer() { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java index 6bb0b226f5..3eb63f6cd9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java @@ -50,6 +50,11 @@ import androidx.media3.decoder.DecoderInputBuffer; TransformationException.ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED); } + @Override + public boolean expectsDecodedData() { + return true; + } + @Override public boolean processData() throws TransformationException { return feedMuxer() || processDataUpToMuxer(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java index c4531d2b23..f63a080474 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java @@ -25,6 +25,7 @@ import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_ENCO import android.media.MediaCodec; import androidx.annotation.Nullable; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.decoder.DecoderInputBuffer; @@ -34,6 +35,8 @@ import androidx.media3.exoplayer.MediaClock; import androidx.media3.exoplayer.RendererCapabilities; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -47,6 +50,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final TransformerMediaClock mediaClock; private final AssetLoader.Listener assetLoaderListener; private final DecoderInputBuffer decoderInputBuffer; + private final List decodeOnlyPresentationTimestamps; private boolean isTransformationRunning; private long streamStartPositionUs; @@ -54,7 +58,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener; private @MonotonicNonNull Codec decoder; @Nullable private ByteBuffer pendingDecoderOutputBuffer; - private SamplePipeline.@MonotonicNonNull Input samplePipelineInput; + private int maxDecoderPendingFrameCount; + private @MonotonicNonNull SampleConsumer sampleConsumer; private boolean isEnded; public ExoPlayerAssetLoaderRenderer( @@ -69,6 +74,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; this.mediaClock = mediaClock; this.assetLoaderListener = assetLoaderListener; decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); + decodeOnlyPresentationTimestamps = new ArrayList<>(); } @Override @@ -112,10 +118,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return; } - if (samplePipelineInput.expectsDecodedData()) { - while (feedPipelineFromDecoder() || feedDecoderFromInput()) {} + if (sampleConsumer.expectsDecodedData()) { + if (getTrackType() == C.TRACK_TYPE_AUDIO) { + while (feedConsumerAudioFromDecoder() || feedDecoderFromInput()) {} + } else if (getTrackType() == C.TRACK_TYPE_VIDEO) { + while (feedConsumerVideoFromDecoder() || feedDecoderFromInput()) {} + } else { + throw new IllegalStateException(); + } } else { - while (feedPipelineFromInput()) {} + while (feedConsumerFromInput()) {} } } catch (TransformationException e) { isTransformationRunning = false; @@ -151,9 +163,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } } - @EnsuresNonNullIf(expression = "samplePipelineInput", result = true) + @EnsuresNonNullIf(expression = "sampleConsumer", result = true) private boolean ensureConfigured() throws TransformationException { - if (samplePipelineInput != null) { + if (sampleConsumer != null) { return true; } @@ -166,30 +178,42 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; Format inputFormat = checkNotNull(formatHolder.format); @AssetLoader.SupportedOutputTypes int supportedOutputTypes = SUPPORTED_OUTPUT_TYPE_ENCODED | SUPPORTED_OUTPUT_TYPE_DECODED; - samplePipelineInput = + sampleConsumer = assetLoaderListener.onTrackAdded( inputFormat, supportedOutputTypes, streamStartPositionUs, streamOffsetUs); if (getTrackType() == C.TRACK_TYPE_VIDEO && flattenForSlowMotion) { sefVideoSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat); } - if (samplePipelineInput.expectsDecodedData()) { - decoder = decoderFactory.createForAudioDecoding(inputFormat); + if (sampleConsumer.expectsDecodedData()) { + if (getTrackType() == C.TRACK_TYPE_AUDIO) { + decoder = decoderFactory.createForAudioDecoding(inputFormat); + } else if (getTrackType() == C.TRACK_TYPE_VIDEO) { + boolean isDecoderToneMappingRequired = + ColorInfo.isTransferHdr(inputFormat.colorInfo) + && !ColorInfo.isTransferHdr(sampleConsumer.getExpectedColorInfo()); + decoder = + decoderFactory.createForVideoDecoding( + inputFormat, + checkNotNull(sampleConsumer.getInputSurface()), + isDecoderToneMappingRequired); + maxDecoderPendingFrameCount = decoder.getMaxPendingFrameCount(); + } else { + throw new IllegalStateException(); + } } return true; } /** - * Attempts to read decoded data and pass it to the sample pipeline. + * Attempts to get decoded audio data and pass it to the sample consumer. * * @return Whether it may be possible to read more data immediately by calling this method again. - * @throws TransformationException If an error occurs in the decoder or in the {@link - * SamplePipeline}. + * @throws TransformationException If an error occurs in the decoder. */ - @RequiresNonNull("samplePipelineInput") - private boolean feedPipelineFromDecoder() throws TransformationException { - @Nullable - DecoderInputBuffer samplePipelineInputBuffer = samplePipelineInput.dequeueInputBuffer(); - if (samplePipelineInputBuffer == null) { + @RequiresNonNull("sampleConsumer") + private boolean feedConsumerAudioFromDecoder() throws TransformationException { + @Nullable DecoderInputBuffer sampleConsumerInputBuffer = sampleConsumer.dequeueInputBuffer(); + if (sampleConsumerInputBuffer == null) { return false; } @@ -204,8 +228,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } if (decoder.isEnded()) { - samplePipelineInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); - samplePipelineInput.queueInputBuffer(); + sampleConsumerInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); + sampleConsumer.queueInputBuffer(); isEnded = true; return false; } @@ -215,11 +239,46 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } - samplePipelineInputBuffer.data = pendingDecoderOutputBuffer; + sampleConsumerInputBuffer.data = pendingDecoderOutputBuffer; MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo()); - samplePipelineInputBuffer.timeUs = bufferInfo.presentationTimeUs; - samplePipelineInputBuffer.setFlags(bufferInfo.flags); - samplePipelineInput.queueInputBuffer(); + sampleConsumerInputBuffer.timeUs = bufferInfo.presentationTimeUs; + sampleConsumerInputBuffer.setFlags(bufferInfo.flags); + sampleConsumer.queueInputBuffer(); + return true; + } + + /** + * Attempts to get decoded video data and pass it to the sample consumer. + * + * @return Whether it may be possible to read more data immediately by calling this method again. + * @throws TransformationException If an error occurs in the decoder. + */ + @RequiresNonNull("sampleConsumer") + private boolean feedConsumerVideoFromDecoder() throws TransformationException { + Codec decoder = checkNotNull(this.decoder); + if (decoder.isEnded()) { + sampleConsumer.signalEndOfVideoInput(); + isEnded = true; + return false; + } + + @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo(); + if (decoderOutputBufferInfo == null) { + return false; + } + + if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) { + decoder.releaseOutputBuffer(/* render= */ false); + return true; + } + + if (maxDecoderPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT + && sampleConsumer.getPendingVideoFrameCount() == maxDecoderPendingFrameCount) { + return false; + } + + sampleConsumer.registerVideoFrame(); + decoder.releaseOutputBuffer(/* render= */ true); return true; } @@ -243,33 +302,35 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return true; } + if (decoderInputBuffer.isDecodeOnly()) { + decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs); + } decoder.queueInputBuffer(decoderInputBuffer); return true; } /** - * Attempts to read input data and pass it to the sample pipeline. + * Attempts to read input data and pass it to the sample consumer. * * @return Whether it may be possible to read more data immediately by calling this method again. */ - @RequiresNonNull("samplePipelineInput") - private boolean feedPipelineFromInput() { - @Nullable - DecoderInputBuffer samplePipelineInputBuffer = samplePipelineInput.dequeueInputBuffer(); - if (samplePipelineInputBuffer == null) { + @RequiresNonNull("sampleConsumer") + private boolean feedConsumerFromInput() { + @Nullable DecoderInputBuffer sampleConsumerInputBuffer = sampleConsumer.dequeueInputBuffer(); + if (sampleConsumerInputBuffer == null) { return false; } - if (!readInput(samplePipelineInputBuffer)) { + if (!readInput(sampleConsumerInputBuffer)) { return false; } - if (shouldDropInputBuffer(samplePipelineInputBuffer)) { + if (shouldDropInputBuffer(sampleConsumerInputBuffer)) { return true; } - samplePipelineInput.queueInputBuffer(); - if (samplePipelineInputBuffer.isEndOfStream()) { + sampleConsumer.queueInputBuffer(); + if (sampleConsumerInputBuffer.isEndOfStream()) { isEnded = true; return false; } @@ -300,8 +361,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } /** - * Preprocesses an {@linkplain DecoderInputBuffer input buffer} queued to the pipeline and returns - * whether it should be dropped. + * Preprocesses an encoded {@linkplain DecoderInputBuffer input buffer} and returns whether it + * should be dropped. * *

The input buffer is cleared if it should be dropped. */ @@ -323,4 +384,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } return shouldDropInputBuffer; } + + private boolean isDecodeOnlyBuffer(long presentationTimeUs) { + // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would + // box presentationTimeUs, creating a Long object that would need to be garbage collected. + int size = decodeOnlyPresentationTimestamps.size(); + for (int i = 0; i < size; i++) { + if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) { + decodeOnlyPresentationTimestamps.remove(i); + return true; + } + } + return false; + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java new file mode 100644 index 0000000000..564a8151a4 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java @@ -0,0 +1,124 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import android.view.Surface; +import androidx.annotation.Nullable; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.decoder.DecoderInputBuffer; + +/** Consumer of encoded media samples, raw audio or raw video frames. */ +@UnstableApi +public interface SampleConsumer { + + /** + * Returns whether the consumer should be fed with decoded sample data. If false, encoded sample + * data should be fed. + * + *

Can be called on any thread. + */ + boolean expectsDecodedData(); + + // Methods to pass compressed input or raw audio input. + + /** + * Returns a buffer if the consumer is ready to accept input, and {@code null} otherwise. + * + *

If the consumer is ready to accept input and this method is called multiple times before + * {@linkplain #queueInputBuffer() queuing} input, the same buffer instance is returned. + * + *

Should only be used for compressed data and raw audio data. + */ + @Nullable + default DecoderInputBuffer dequeueInputBuffer() { + throw new UnsupportedOperationException(); + } + + /** + * Informs the consumer that its input buffer contains new input. + * + *

Should be called after filling the input buffer from {@link #dequeueInputBuffer()} with new + * input. + * + *

Should only be used for compressed data and raw audio data. + */ + default void queueInputBuffer() { + throw new UnsupportedOperationException(); + } + + // Methods to pass raw video input. + + /** + * Returns the input {@link Surface}, where the consumer reads input frames from. + * + *

Should only be used for raw video data. + * + *

Can be called on any thread. + */ + default Surface getInputSurface() { + throw new UnsupportedOperationException(); + } + + /** + * Returns the expected input {@link ColorInfo}. + * + *

Should only be used for raw video data. + * + *

Can be called on any thread. + */ + default ColorInfo getExpectedColorInfo() { + throw new UnsupportedOperationException(); + } + + /** + * Returns the number of input video frames pending in the consumer. Pending input frames are + * frames that have been {@linkplain #registerVideoFrame() registered} but not processed off the + * {@linkplain #getInputSurface() input surface} yet. + * + *

Should only be used for raw video data. + * + *

Can be called on any thread. + */ + default int getPendingVideoFrameCount() { + throw new UnsupportedOperationException(); + } + + /** + * Informs the consumer that a frame will be queued to the {@linkplain #getInputSurface() input + * surface}. + * + *

Must be called before rendering a frame to the input surface. + * + *

Should only be used for raw video data. + * + *

Can be called on any thread. + */ + default void registerVideoFrame() { + throw new UnsupportedOperationException(); + } + + /** + * Informs the consumer that no further input frames will be rendered. + * + *

Should only be used for raw video data. + * + *

Can be called on any thread. + */ + default void signalEndOfVideoInput() { + throw new UnsupportedOperationException(); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java index 2aeb683742..b258e16479 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java @@ -16,49 +16,12 @@ package androidx.media3.transformer; -import androidx.annotation.Nullable; -import androidx.media3.common.util.UnstableApi; -import androidx.media3.decoder.DecoderInputBuffer; - /** * Pipeline for processing media data. * *

This pipeline can be used to implement transformations of audio or video samples. */ -@UnstableApi -public interface SamplePipeline { - - /** Input of a {@link SamplePipeline}. */ - interface Input { - - /** See {@link SamplePipeline#expectsDecodedData()}. */ - boolean expectsDecodedData(); - - /** See {@link SamplePipeline#dequeueInputBuffer()}. */ - @Nullable - DecoderInputBuffer dequeueInputBuffer(); - - /** See {@link SamplePipeline#queueInputBuffer()}. */ - void queueInputBuffer(); - } - - /** - * Returns whether the pipeline should be fed with decoded sample data. If false, encoded sample - * data should be queued. - */ - boolean expectsDecodedData(); - - /** Returns a buffer if the pipeline is ready to accept input, and {@code null} otherwise. */ - @Nullable - DecoderInputBuffer dequeueInputBuffer() throws TransformationException; - - /** - * Informs the pipeline that its input buffer contains new input. - * - *

Should be called after filling the input buffer from {@link #dequeueInputBuffer()} with new - * input. - */ - void queueInputBuffer() throws TransformationException; +/* package */ interface SamplePipeline extends SampleConsumer { /** * Processes the input data and returns whether it may be possible to process more data by calling diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 54106b4501..32e7d71818 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -29,9 +29,11 @@ import android.os.HandlerThread; import android.os.Looper; import android.os.Message; import android.os.ParcelFileDescriptor; +import android.view.Surface; import androidx.annotation.IntDef; import androidx.annotation.Nullable; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.Format; @@ -83,8 +85,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Internal messages. private static final int MSG_START = 0; private static final int MSG_REGISTER_SAMPLE_PIPELINE = 1; - private static final int MSG_DEQUEUE_INPUT = 2; - private static final int MSG_QUEUE_INPUT = 3; + private static final int MSG_DEQUEUE_BUFFER = 2; + private static final int MSG_QUEUE_BUFFER = 3; private static final int MSG_DRAIN_PIPELINES = 4; private static final int MSG_END = 5; private static final int MSG_UPDATE_PROGRESS = 6; @@ -230,11 +232,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; case MSG_REGISTER_SAMPLE_PIPELINE: registerSamplePipelineInternal((SamplePipeline) msg.obj); break; - case MSG_DEQUEUE_INPUT: - dequeueInputInternal(/* samplePipelineIndex= */ msg.arg1); + case MSG_DEQUEUE_BUFFER: + dequeueBufferInternal(/* samplePipelineIndex= */ msg.arg1); break; - case MSG_QUEUE_INPUT: - samplePipelines.get(/* samplePipelineIndex= */ msg.arg1).queueInputBuffer(); + case MSG_QUEUE_BUFFER: + samplePipelines.get(/* index= */ msg.arg1).queueInputBuffer(); break; case MSG_DRAIN_PIPELINES: drainPipelinesInternal(); @@ -271,7 +273,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - private void dequeueInputInternal(int samplePipelineIndex) throws TransformationException { + private void dequeueBufferInternal(int samplePipelineIndex) throws TransformationException { SamplePipeline samplePipeline = samplePipelines.get(samplePipelineIndex); // The sample pipeline is drained before dequeuing input to maximise the chances of having an // input buffer to dequeue. @@ -418,7 +420,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public SamplePipeline.Input onTrackAdded( + public SampleConsumer onTrackAdded( Format format, @AssetLoader.SupportedOutputTypes int supportedOutputTypes, long streamStartPositionUs, @@ -434,7 +436,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; SamplePipeline samplePipeline = getSamplePipeline(format, supportedOutputTypes, streamStartPositionUs, streamOffsetUs); internalHandler.obtainMessage(MSG_REGISTER_SAMPLE_PIPELINE, samplePipeline).sendToTarget(); - int samplePipelineIndex = tracksAddedCount; tracksAddedCount++; @@ -458,7 +459,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; tracksAddedCount++; } - return new SamplePipelineInput(samplePipelineIndex, samplePipeline.expectsDecodedData()); + return new SampleConsumerImpl(samplePipelineIndex, samplePipeline); } // MuxerWrapper.Listener implementation. @@ -523,7 +524,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; transformationRequest, videoEffects, frameProcessorFactory, - decoderFactory, encoderFactory, muxerWrapper, /* errorConsumer= */ this::onTransformationError, @@ -622,19 +622,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return false; } - private class SamplePipelineInput implements SamplePipeline.Input { + private class SampleConsumerImpl implements SampleConsumer { private final int samplePipelineIndex; - private final boolean expectsDecodedData; + private final SamplePipeline samplePipeline; - public SamplePipelineInput(int samplePipelineIndex, boolean expectsDecodedData) { + public SampleConsumerImpl(int samplePipelineIndex, SamplePipeline samplePipeline) { this.samplePipelineIndex = samplePipelineIndex; - this.expectsDecodedData = expectsDecodedData; + this.samplePipeline = samplePipeline; } @Override public boolean expectsDecodedData() { - return expectsDecodedData; + return samplePipeline.expectsDecodedData(); } @Nullable @@ -649,7 +649,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // start of the sample pipelines). Having 2 thread hops per sample (one for dequeuing and // one for queuing) makes transmuxing slower than it used to be. internalHandler - .obtainMessage(MSG_DEQUEUE_INPUT, samplePipelineIndex, /* unused */ 0) + .obtainMessage(MSG_DEQUEUE_BUFFER, samplePipelineIndex, /* unused */ 0) .sendToTarget(); clock.onThreadBlocked(); dequeueBufferConditionVariable.blockUninterruptible(); @@ -660,9 +660,34 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void queueInputBuffer() { internalHandler - .obtainMessage(MSG_QUEUE_INPUT, samplePipelineIndex, /* unused */ 0) + .obtainMessage(MSG_QUEUE_BUFFER, samplePipelineIndex, /* unused */ 0) .sendToTarget(); } + + @Override + public Surface getInputSurface() { + return samplePipeline.getInputSurface(); + } + + @Override + public ColorInfo getExpectedColorInfo() { + return samplePipeline.getExpectedColorInfo(); + } + + @Override + public int getPendingVideoFrameCount() { + return samplePipeline.getPendingVideoFrameCount(); + } + + @Override + public void registerVideoFrame() { + samplePipeline.registerVideoFrame(); + } + + @Override + public void signalEndOfVideoInput() { + samplePipeline.signalEndOfVideoInput(); + } } } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index fbbd63dada..6dee51a3d3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -49,23 +49,15 @@ import androidx.media3.effect.ScaleToFitTransformation; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; -/** - * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. - */ +/** Pipeline to process, re-encode and mux raw video frames. */ /* package */ final class VideoTranscodingSamplePipeline extends BaseSamplePipeline { - private final int maxPendingFrameCount; - - private final DecoderInputBuffer decoderInputBuffer; - private final Codec decoder; - private final ArrayList decodeOnlyPresentationTimestamps; - private final FrameProcessor frameProcessor; + private final ColorInfo frameProcessorInputColor; private final EncoderWrapper encoderWrapper; private final DecoderInputBuffer encoderOutputBuffer; @@ -84,7 +76,6 @@ import org.checkerframework.dataflow.qual.Pure; TransformationRequest transformationRequest, ImmutableList effects, FrameProcessor.Factory frameProcessorFactory, - Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, MuxerWrapper muxerWrapper, Consumer errorConsumer, @@ -131,11 +122,8 @@ import org.checkerframework.dataflow.qual.Pure; finalFramePresentationTimeUs = C.TIME_UNSET; - decoderInputBuffer = - new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - decodeOnlyPresentationTimestamps = new ArrayList<>(); // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. int decodedWidth = @@ -169,7 +157,7 @@ import org.checkerframework.dataflow.qual.Pure; ColorInfo encoderInputColor = encoderWrapper.getSupportedInputColor(); // If not tone mapping using OpenGL, the decoder will output the encoderInputColor, // possibly by tone mapping. - ColorInfo frameProcessorInputColor = + frameProcessorInputColor = isGlToneMapping ? checkNotNull(inputFormat.colorInfo) : encoderInputColor; // For consistency with the Android platform, OpenGL tone mapping outputs colors with // C.COLOR_TRANSFER_GAMMA_2_2 instead of C.COLOR_TRANSFER_SDR, and outputs this as @@ -236,57 +224,42 @@ import org.checkerframework.dataflow.qual.Pure; frameProcessor.setInputFrameInfo( new FrameInfo( decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs)); - - boolean isDecoderToneMappingRequired = - ColorInfo.isTransferHdr(inputFormat.colorInfo) - && !ColorInfo.isTransferHdr(frameProcessorInputColor); - decoder = - decoderFactory.createForVideoDecoding( - inputFormat, frameProcessor.getInputSurface(), isDecoderToneMappingRequired); - maxPendingFrameCount = decoder.getMaxPendingFrameCount(); } @Override - public boolean expectsDecodedData() { - return false; + public Surface getInputSurface() { + return frameProcessor.getInputSurface(); } @Override - @Nullable - public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { - return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; + public ColorInfo getExpectedColorInfo() { + return frameProcessorInputColor; } @Override - public void queueInputBuffer() throws TransformationException { - if (decoderInputBuffer.isDecodeOnly()) { - decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs); - } - decoder.queueInputBuffer(decoderInputBuffer); + public void registerVideoFrame() { + frameProcessor.registerInputFrame(); + } + + @Override + public int getPendingVideoFrameCount() { + return frameProcessor.getPendingInputFrameCount(); + } + + @Override + public void signalEndOfVideoInput() { + frameProcessor.signalEndOfInput(); } @Override public void release() { frameProcessor.release(); - decoder.release(); encoderWrapper.release(); } @Override - protected boolean processDataUpToMuxer() throws TransformationException { - if (decoder.isEnded()) { - return false; - } - - boolean processedData = false; - while (maybeProcessDecoderOutput()) { - processedData = true; - } - if (decoder.isEnded()) { - frameProcessor.signalEndOfInput(); - } - // If the decoder produced output, signal that it may be possible to process data again. - return processedData; + protected boolean processDataUpToMuxer() { + return false; } @Override @@ -377,46 +350,6 @@ import org.checkerframework.dataflow.qual.Pure; || Build.ID.startsWith(/* Pixel Watch */ "rwd9.220429.053")); } - /** - * Feeds at most one decoder output frame to the next step of the pipeline. - * - * @return Whether a frame was processed. - * @throws TransformationException If a problem occurs while processing the frame. - */ - private boolean maybeProcessDecoderOutput() throws TransformationException { - @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo(); - if (decoderOutputBufferInfo == null) { - return false; - } - - if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) { - decoder.releaseOutputBuffer(/* render= */ false); - return true; - } - - if (maxPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT - && frameProcessor.getPendingInputFrameCount() == maxPendingFrameCount) { - return false; - } - - frameProcessor.registerInputFrame(); - decoder.releaseOutputBuffer(/* render= */ true); - return true; - } - - private boolean isDecodeOnlyBuffer(long presentationTimeUs) { - // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would - // box presentationTimeUs, creating a Long object that would need to be garbage collected. - int size = decodeOnlyPresentationTimestamps.size(); - for (int i = 0; i < size; i++) { - if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) { - decodeOnlyPresentationTimestamps.remove(i); - return true; - } - } - return false; - } - /** * Wraps an {@linkplain Codec encoder} and provides its input {@link Surface}. * diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java index 583653a150..fa8a55bd28 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java @@ -68,7 +68,7 @@ public class ExoPlayerAssetLoaderTest { } @Override - public SamplePipeline.Input onTrackAdded( + public SampleConsumer onTrackAdded( Format format, @AssetLoader.SupportedOutputTypes int supportedOutputTypes, long streamStartPositionUs, @@ -81,7 +81,7 @@ public class ExoPlayerAssetLoaderTest { new IllegalStateException("onTrackAdded() called before onTrackCount()")); } isTrackAdded.set(true); - return new FakeSamplePipelineInput(); + return new FakeSampleConsumer(); } @Override @@ -130,7 +130,7 @@ public class ExoPlayerAssetLoaderTest { .createAssetLoader(); } - private static final class FakeSamplePipelineInput implements SamplePipeline.Input { + private static final class FakeSampleConsumer implements SampleConsumer { @Override public boolean expectsDecodedData() { diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java index 73feb48624..69a1cc27ff 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -633,18 +633,18 @@ public final class TransformerEndToEndTest { @Test public void startTransformation_withAssetLoaderAlwaysDecoding_pipelineExpectsDecoded() throws Exception { - AtomicReference samplePipelineInputRef = new AtomicReference<>(); + AtomicReference sampleConsumerRef = new AtomicReference<>(); Transformer transformer = createTransformerBuilder(/* enableFallback= */ false) .setAssetLoaderFactory( - new FakeAssetLoader.Factory(SUPPORTED_OUTPUT_TYPE_DECODED, samplePipelineInputRef)) + new FakeAssetLoader.Factory(SUPPORTED_OUTPUT_TYPE_DECODED, sampleConsumerRef)) .build(); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); transformer.startTransformation(mediaItem, outputPath); - runLooperUntil(transformer.getApplicationLooper(), () -> samplePipelineInputRef.get() != null); + runLooperUntil(transformer.getApplicationLooper(), () -> sampleConsumerRef.get() != null); - assertThat(samplePipelineInputRef.get().expectsDecodedData()).isTrue(); + assertThat(sampleConsumerRef.get().expectsDecodedData()).isTrue(); } @Test @@ -654,7 +654,7 @@ public final class TransformerEndToEndTest { .setAudioProcessors(ImmutableList.of(new SonicAudioProcessor())) .setAssetLoaderFactory( new FakeAssetLoader.Factory( - SUPPORTED_OUTPUT_TYPE_ENCODED, /* samplePipelineInputRef= */ null)) + SUPPORTED_OUTPUT_TYPE_ENCODED, /* sampleConsumerRef= */ null)) .build(); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); @@ -1077,15 +1077,15 @@ public final class TransformerEndToEndTest { public static final class Factory implements AssetLoader.Factory { private final @SupportedOutputTypes int supportedOutputTypes; - @Nullable private final AtomicReference samplePipelineInputRef; + @Nullable private final AtomicReference sampleConsumerRef; @Nullable private AssetLoader.Listener listener; public Factory( @SupportedOutputTypes int supportedOutputTypes, - @Nullable AtomicReference samplePipelineInputRef) { + @Nullable AtomicReference sampleConsumerRef) { this.supportedOutputTypes = supportedOutputTypes; - this.samplePipelineInputRef = samplePipelineInputRef; + this.sampleConsumerRef = sampleConsumerRef; } @Override @@ -1136,22 +1136,21 @@ public final class TransformerEndToEndTest { @Override public AssetLoader createAssetLoader() { - return new FakeAssetLoader( - checkNotNull(listener), supportedOutputTypes, samplePipelineInputRef); + return new FakeAssetLoader(checkNotNull(listener), supportedOutputTypes, sampleConsumerRef); } } private final AssetLoader.Listener listener; private final @SupportedOutputTypes int supportedOutputTypes; - @Nullable private final AtomicReference samplePipelineInputRef; + @Nullable private final AtomicReference sampleConsumerRef; public FakeAssetLoader( Listener listener, @SupportedOutputTypes int supportedOutputTypes, - @Nullable AtomicReference samplePipelineInputRef) { + @Nullable AtomicReference sampleConsumerRef) { this.listener = listener; this.supportedOutputTypes = supportedOutputTypes; - this.samplePipelineInputRef = samplePipelineInputRef; + this.sampleConsumerRef = sampleConsumerRef; } @Override @@ -1165,14 +1164,14 @@ public final class TransformerEndToEndTest { .setChannelCount(2) .build(); try { - SamplePipeline.Input samplePipelineInput = + SampleConsumer sampleConsumer = listener.onTrackAdded( format, supportedOutputTypes, /* streamStartPositionUs= */ 0, /* streamOffsetUs= */ 0); - if (samplePipelineInputRef != null) { - samplePipelineInputRef.set(samplePipelineInput); + if (sampleConsumerRef != null) { + sampleConsumerRef.set(sampleConsumer); } } catch (TransformationException e) { throw new IllegalStateException(e);