diff --git a/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java
index 5b29c4adc9..4dedaf4685 100644
--- a/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java
+++ b/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java
@@ -120,7 +120,11 @@ public interface FrameProcessor {
/** Indicates the frame should be dropped after {@link #releaseOutputFrame(long)} is invoked. */
long DROP_OUTPUT_FRAME = -2;
- /** Returns the input {@link Surface}, where {@link FrameProcessor} consumes input frames from. */
+ /**
+ * Returns the input {@link Surface}, where {@link FrameProcessor} consumes input frames from.
+ *
+ *
Can be called on any thread.
+ */
Surface getInputSurface();
/**
@@ -142,6 +146,8 @@ public interface FrameProcessor {
*
*
Must be called before rendering a frame to the frame processor's input surface.
*
+ *
Can be called on any thread.
+ *
* @throws IllegalStateException If called after {@link #signalEndOfInput()} or before {@link
* #setInputFrameInfo(FrameInfo)}.
*/
@@ -150,6 +156,8 @@ public interface FrameProcessor {
/**
* Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
* but not processed off the {@linkplain #getInputSurface() input surface} yet.
+ *
+ *
Can be called on any thread.
*/
int getPendingInputFrameCount();
@@ -194,6 +202,8 @@ public interface FrameProcessor {
/**
* Informs the {@code FrameProcessor} that no further input frames should be accepted.
*
+ *
Can be called on any thread.
+ *
* @throws IllegalStateException If called more than once.
*/
void signalEndOfInput();
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
index ca7f16fe88..4c9dc12969 100644
--- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
@@ -334,14 +334,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
private final FinalMatrixTextureProcessorWrapper finalTextureProcessorWrapper;
private final ImmutableList allTextureProcessors;
- private @MonotonicNonNull FrameInfo nextInputFrameInfo;
- private boolean inputStreamEnded;
/**
* Offset compared to original media presentation time that has been added to incoming frame
* timestamps, in microseconds.
*/
private long previousStreamOffsetUs;
+ private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
+ private volatile boolean inputStreamEnded;
+
private GlEffectsFrameProcessor(
EGLDisplay eglDisplay,
EGLContext eglContext,
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java
index 87413ed229..82b55cf371 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java
@@ -149,12 +149,11 @@ public interface AssetLoader {
* streamOffsetUs}), in microseconds.
* @param streamOffsetUs The offset that will be added to the timestamps to make sure they are
* non-negative, in microseconds.
- * @return The {@link SamplePipeline.Input} describing the type of sample data expected, and to
- * which to pass this data.
- * @throws TransformationException If an error occurs configuring the {@link
- * SamplePipeline.Input}.
+ * @return The {@link SampleConsumer} describing the type of sample data expected, and to which
+ * to pass this data.
+ * @throws TransformationException If an error occurs configuring the {@link SampleConsumer}.
*/
- SamplePipeline.Input onTrackAdded(
+ SampleConsumer onTrackAdded(
Format format,
@SupportedOutputTypes int supportedOutputTypes,
long streamStartPositionUs,
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
index 54dc8ce39c..82608dbe29 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
@@ -35,7 +35,7 @@ import java.util.List;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.dataflow.qual.Pure;
-/** Pipeline to apply audio processing to raw audio samples, encode them and mux them. */
+/** Pipeline to process, re-encode and mux raw audio samples. */
/* package */ final class AudioTranscodingSamplePipeline extends BaseSamplePipeline {
private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
@@ -137,11 +137,6 @@ import org.checkerframework.dataflow.qual.Pure;
nextEncoderInputBufferTimeUs = streamOffsetUs;
}
- @Override
- public boolean expectsDecodedData() {
- return true;
- }
-
@Override
@Nullable
public DecoderInputBuffer dequeueInputBuffer() {
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java
index 6bb0b226f5..3eb63f6cd9 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java
@@ -50,6 +50,11 @@ import androidx.media3.decoder.DecoderInputBuffer;
TransformationException.ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED);
}
+ @Override
+ public boolean expectsDecodedData() {
+ return true;
+ }
+
@Override
public boolean processData() throws TransformationException {
return feedMuxer() || processDataUpToMuxer();
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java
index c4531d2b23..f63a080474 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java
@@ -25,6 +25,7 @@ import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_ENCO
import android.media.MediaCodec;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
+import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.decoder.DecoderInputBuffer;
@@ -34,6 +35,8 @@ import androidx.media3.exoplayer.MediaClock;
import androidx.media3.exoplayer.RendererCapabilities;
import androidx.media3.exoplayer.source.SampleStream.ReadDataResult;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@@ -47,6 +50,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final TransformerMediaClock mediaClock;
private final AssetLoader.Listener assetLoaderListener;
private final DecoderInputBuffer decoderInputBuffer;
+ private final List decodeOnlyPresentationTimestamps;
private boolean isTransformationRunning;
private long streamStartPositionUs;
@@ -54,7 +58,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener;
private @MonotonicNonNull Codec decoder;
@Nullable private ByteBuffer pendingDecoderOutputBuffer;
- private SamplePipeline.@MonotonicNonNull Input samplePipelineInput;
+ private int maxDecoderPendingFrameCount;
+ private @MonotonicNonNull SampleConsumer sampleConsumer;
private boolean isEnded;
public ExoPlayerAssetLoaderRenderer(
@@ -69,6 +74,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
this.mediaClock = mediaClock;
this.assetLoaderListener = assetLoaderListener;
decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
+ decodeOnlyPresentationTimestamps = new ArrayList<>();
}
@Override
@@ -112,10 +118,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return;
}
- if (samplePipelineInput.expectsDecodedData()) {
- while (feedPipelineFromDecoder() || feedDecoderFromInput()) {}
+ if (sampleConsumer.expectsDecodedData()) {
+ if (getTrackType() == C.TRACK_TYPE_AUDIO) {
+ while (feedConsumerAudioFromDecoder() || feedDecoderFromInput()) {}
+ } else if (getTrackType() == C.TRACK_TYPE_VIDEO) {
+ while (feedConsumerVideoFromDecoder() || feedDecoderFromInput()) {}
+ } else {
+ throw new IllegalStateException();
+ }
} else {
- while (feedPipelineFromInput()) {}
+ while (feedConsumerFromInput()) {}
}
} catch (TransformationException e) {
isTransformationRunning = false;
@@ -151,9 +163,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
}
- @EnsuresNonNullIf(expression = "samplePipelineInput", result = true)
+ @EnsuresNonNullIf(expression = "sampleConsumer", result = true)
private boolean ensureConfigured() throws TransformationException {
- if (samplePipelineInput != null) {
+ if (sampleConsumer != null) {
return true;
}
@@ -166,30 +178,42 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Format inputFormat = checkNotNull(formatHolder.format);
@AssetLoader.SupportedOutputTypes
int supportedOutputTypes = SUPPORTED_OUTPUT_TYPE_ENCODED | SUPPORTED_OUTPUT_TYPE_DECODED;
- samplePipelineInput =
+ sampleConsumer =
assetLoaderListener.onTrackAdded(
inputFormat, supportedOutputTypes, streamStartPositionUs, streamOffsetUs);
if (getTrackType() == C.TRACK_TYPE_VIDEO && flattenForSlowMotion) {
sefVideoSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat);
}
- if (samplePipelineInput.expectsDecodedData()) {
- decoder = decoderFactory.createForAudioDecoding(inputFormat);
+ if (sampleConsumer.expectsDecodedData()) {
+ if (getTrackType() == C.TRACK_TYPE_AUDIO) {
+ decoder = decoderFactory.createForAudioDecoding(inputFormat);
+ } else if (getTrackType() == C.TRACK_TYPE_VIDEO) {
+ boolean isDecoderToneMappingRequired =
+ ColorInfo.isTransferHdr(inputFormat.colorInfo)
+ && !ColorInfo.isTransferHdr(sampleConsumer.getExpectedColorInfo());
+ decoder =
+ decoderFactory.createForVideoDecoding(
+ inputFormat,
+ checkNotNull(sampleConsumer.getInputSurface()),
+ isDecoderToneMappingRequired);
+ maxDecoderPendingFrameCount = decoder.getMaxPendingFrameCount();
+ } else {
+ throw new IllegalStateException();
+ }
}
return true;
}
/**
- * Attempts to read decoded data and pass it to the sample pipeline.
+ * Attempts to get decoded audio data and pass it to the sample consumer.
*
* @return Whether it may be possible to read more data immediately by calling this method again.
- * @throws TransformationException If an error occurs in the decoder or in the {@link
- * SamplePipeline}.
+ * @throws TransformationException If an error occurs in the decoder.
*/
- @RequiresNonNull("samplePipelineInput")
- private boolean feedPipelineFromDecoder() throws TransformationException {
- @Nullable
- DecoderInputBuffer samplePipelineInputBuffer = samplePipelineInput.dequeueInputBuffer();
- if (samplePipelineInputBuffer == null) {
+ @RequiresNonNull("sampleConsumer")
+ private boolean feedConsumerAudioFromDecoder() throws TransformationException {
+ @Nullable DecoderInputBuffer sampleConsumerInputBuffer = sampleConsumer.dequeueInputBuffer();
+ if (sampleConsumerInputBuffer == null) {
return false;
}
@@ -204,8 +228,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
if (decoder.isEnded()) {
- samplePipelineInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM);
- samplePipelineInput.queueInputBuffer();
+ sampleConsumerInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM);
+ sampleConsumer.queueInputBuffer();
isEnded = true;
return false;
}
@@ -215,11 +239,46 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return false;
}
- samplePipelineInputBuffer.data = pendingDecoderOutputBuffer;
+ sampleConsumerInputBuffer.data = pendingDecoderOutputBuffer;
MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo());
- samplePipelineInputBuffer.timeUs = bufferInfo.presentationTimeUs;
- samplePipelineInputBuffer.setFlags(bufferInfo.flags);
- samplePipelineInput.queueInputBuffer();
+ sampleConsumerInputBuffer.timeUs = bufferInfo.presentationTimeUs;
+ sampleConsumerInputBuffer.setFlags(bufferInfo.flags);
+ sampleConsumer.queueInputBuffer();
+ return true;
+ }
+
+ /**
+ * Attempts to get decoded video data and pass it to the sample consumer.
+ *
+ * @return Whether it may be possible to read more data immediately by calling this method again.
+ * @throws TransformationException If an error occurs in the decoder.
+ */
+ @RequiresNonNull("sampleConsumer")
+ private boolean feedConsumerVideoFromDecoder() throws TransformationException {
+ Codec decoder = checkNotNull(this.decoder);
+ if (decoder.isEnded()) {
+ sampleConsumer.signalEndOfVideoInput();
+ isEnded = true;
+ return false;
+ }
+
+ @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo();
+ if (decoderOutputBufferInfo == null) {
+ return false;
+ }
+
+ if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) {
+ decoder.releaseOutputBuffer(/* render= */ false);
+ return true;
+ }
+
+ if (maxDecoderPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT
+ && sampleConsumer.getPendingVideoFrameCount() == maxDecoderPendingFrameCount) {
+ return false;
+ }
+
+ sampleConsumer.registerVideoFrame();
+ decoder.releaseOutputBuffer(/* render= */ true);
return true;
}
@@ -243,33 +302,35 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return true;
}
+ if (decoderInputBuffer.isDecodeOnly()) {
+ decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs);
+ }
decoder.queueInputBuffer(decoderInputBuffer);
return true;
}
/**
- * Attempts to read input data and pass it to the sample pipeline.
+ * Attempts to read input data and pass it to the sample consumer.
*
* @return Whether it may be possible to read more data immediately by calling this method again.
*/
- @RequiresNonNull("samplePipelineInput")
- private boolean feedPipelineFromInput() {
- @Nullable
- DecoderInputBuffer samplePipelineInputBuffer = samplePipelineInput.dequeueInputBuffer();
- if (samplePipelineInputBuffer == null) {
+ @RequiresNonNull("sampleConsumer")
+ private boolean feedConsumerFromInput() {
+ @Nullable DecoderInputBuffer sampleConsumerInputBuffer = sampleConsumer.dequeueInputBuffer();
+ if (sampleConsumerInputBuffer == null) {
return false;
}
- if (!readInput(samplePipelineInputBuffer)) {
+ if (!readInput(sampleConsumerInputBuffer)) {
return false;
}
- if (shouldDropInputBuffer(samplePipelineInputBuffer)) {
+ if (shouldDropInputBuffer(sampleConsumerInputBuffer)) {
return true;
}
- samplePipelineInput.queueInputBuffer();
- if (samplePipelineInputBuffer.isEndOfStream()) {
+ sampleConsumer.queueInputBuffer();
+ if (sampleConsumerInputBuffer.isEndOfStream()) {
isEnded = true;
return false;
}
@@ -300,8 +361,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
/**
- * Preprocesses an {@linkplain DecoderInputBuffer input buffer} queued to the pipeline and returns
- * whether it should be dropped.
+ * Preprocesses an encoded {@linkplain DecoderInputBuffer input buffer} and returns whether it
+ * should be dropped.
*
* The input buffer is cleared if it should be dropped.
*/
@@ -323,4 +384,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
return shouldDropInputBuffer;
}
+
+ private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
+ // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would
+ // box presentationTimeUs, creating a Long object that would need to be garbage collected.
+ int size = decodeOnlyPresentationTimestamps.size();
+ for (int i = 0; i < size; i++) {
+ if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) {
+ decodeOnlyPresentationTimestamps.remove(i);
+ return true;
+ }
+ }
+ return false;
+ }
}
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java
new file mode 100644
index 0000000000..564a8151a4
--- /dev/null
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer;
+
+import android.view.Surface;
+import androidx.annotation.Nullable;
+import androidx.media3.common.ColorInfo;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.DecoderInputBuffer;
+
+/** Consumer of encoded media samples, raw audio or raw video frames. */
+@UnstableApi
+public interface SampleConsumer {
+
+ /**
+ * Returns whether the consumer should be fed with decoded sample data. If false, encoded sample
+ * data should be fed.
+ *
+ *
Can be called on any thread.
+ */
+ boolean expectsDecodedData();
+
+ // Methods to pass compressed input or raw audio input.
+
+ /**
+ * Returns a buffer if the consumer is ready to accept input, and {@code null} otherwise.
+ *
+ *
If the consumer is ready to accept input and this method is called multiple times before
+ * {@linkplain #queueInputBuffer() queuing} input, the same buffer instance is returned.
+ *
+ *
Should only be used for compressed data and raw audio data.
+ */
+ @Nullable
+ default DecoderInputBuffer dequeueInputBuffer() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Informs the consumer that its input buffer contains new input.
+ *
+ *
Should be called after filling the input buffer from {@link #dequeueInputBuffer()} with new
+ * input.
+ *
+ *
Should only be used for compressed data and raw audio data.
+ */
+ default void queueInputBuffer() {
+ throw new UnsupportedOperationException();
+ }
+
+ // Methods to pass raw video input.
+
+ /**
+ * Returns the input {@link Surface}, where the consumer reads input frames from.
+ *
+ *
Should only be used for raw video data.
+ *
+ *
Can be called on any thread.
+ */
+ default Surface getInputSurface() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Returns the expected input {@link ColorInfo}.
+ *
+ *
Should only be used for raw video data.
+ *
+ *
Can be called on any thread.
+ */
+ default ColorInfo getExpectedColorInfo() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Returns the number of input video frames pending in the consumer. Pending input frames are
+ * frames that have been {@linkplain #registerVideoFrame() registered} but not processed off the
+ * {@linkplain #getInputSurface() input surface} yet.
+ *
+ *
Should only be used for raw video data.
+ *
+ *
Can be called on any thread.
+ */
+ default int getPendingVideoFrameCount() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Informs the consumer that a frame will be queued to the {@linkplain #getInputSurface() input
+ * surface}.
+ *
+ *
Must be called before rendering a frame to the input surface.
+ *
+ *
Should only be used for raw video data.
+ *
+ *
Can be called on any thread.
+ */
+ default void registerVideoFrame() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Informs the consumer that no further input frames will be rendered.
+ *
+ *
Should only be used for raw video data.
+ *
+ *
Can be called on any thread.
+ */
+ default void signalEndOfVideoInput() {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java
index 2aeb683742..b258e16479 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java
@@ -16,49 +16,12 @@
package androidx.media3.transformer;
-import androidx.annotation.Nullable;
-import androidx.media3.common.util.UnstableApi;
-import androidx.media3.decoder.DecoderInputBuffer;
-
/**
* Pipeline for processing media data.
*
*
This pipeline can be used to implement transformations of audio or video samples.
*/
-@UnstableApi
-public interface SamplePipeline {
-
- /** Input of a {@link SamplePipeline}. */
- interface Input {
-
- /** See {@link SamplePipeline#expectsDecodedData()}. */
- boolean expectsDecodedData();
-
- /** See {@link SamplePipeline#dequeueInputBuffer()}. */
- @Nullable
- DecoderInputBuffer dequeueInputBuffer();
-
- /** See {@link SamplePipeline#queueInputBuffer()}. */
- void queueInputBuffer();
- }
-
- /**
- * Returns whether the pipeline should be fed with decoded sample data. If false, encoded sample
- * data should be queued.
- */
- boolean expectsDecodedData();
-
- /** Returns a buffer if the pipeline is ready to accept input, and {@code null} otherwise. */
- @Nullable
- DecoderInputBuffer dequeueInputBuffer() throws TransformationException;
-
- /**
- * Informs the pipeline that its input buffer contains new input.
- *
- *
Should be called after filling the input buffer from {@link #dequeueInputBuffer()} with new
- * input.
- */
- void queueInputBuffer() throws TransformationException;
+/* package */ interface SamplePipeline extends SampleConsumer {
/**
* Processes the input data and returns whether it may be possible to process more data by calling
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
index 54106b4501..32e7d71818 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
@@ -29,9 +29,11 @@ import android.os.HandlerThread;
import android.os.Looper;
import android.os.Message;
import android.os.ParcelFileDescriptor;
+import android.view.Surface;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
+import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
@@ -83,8 +85,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Internal messages.
private static final int MSG_START = 0;
private static final int MSG_REGISTER_SAMPLE_PIPELINE = 1;
- private static final int MSG_DEQUEUE_INPUT = 2;
- private static final int MSG_QUEUE_INPUT = 3;
+ private static final int MSG_DEQUEUE_BUFFER = 2;
+ private static final int MSG_QUEUE_BUFFER = 3;
private static final int MSG_DRAIN_PIPELINES = 4;
private static final int MSG_END = 5;
private static final int MSG_UPDATE_PROGRESS = 6;
@@ -230,11 +232,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
case MSG_REGISTER_SAMPLE_PIPELINE:
registerSamplePipelineInternal((SamplePipeline) msg.obj);
break;
- case MSG_DEQUEUE_INPUT:
- dequeueInputInternal(/* samplePipelineIndex= */ msg.arg1);
+ case MSG_DEQUEUE_BUFFER:
+ dequeueBufferInternal(/* samplePipelineIndex= */ msg.arg1);
break;
- case MSG_QUEUE_INPUT:
- samplePipelines.get(/* samplePipelineIndex= */ msg.arg1).queueInputBuffer();
+ case MSG_QUEUE_BUFFER:
+ samplePipelines.get(/* index= */ msg.arg1).queueInputBuffer();
break;
case MSG_DRAIN_PIPELINES:
drainPipelinesInternal();
@@ -271,7 +273,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
- private void dequeueInputInternal(int samplePipelineIndex) throws TransformationException {
+ private void dequeueBufferInternal(int samplePipelineIndex) throws TransformationException {
SamplePipeline samplePipeline = samplePipelines.get(samplePipelineIndex);
// The sample pipeline is drained before dequeuing input to maximise the chances of having an
// input buffer to dequeue.
@@ -418,7 +420,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
- public SamplePipeline.Input onTrackAdded(
+ public SampleConsumer onTrackAdded(
Format format,
@AssetLoader.SupportedOutputTypes int supportedOutputTypes,
long streamStartPositionUs,
@@ -434,7 +436,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
SamplePipeline samplePipeline =
getSamplePipeline(format, supportedOutputTypes, streamStartPositionUs, streamOffsetUs);
internalHandler.obtainMessage(MSG_REGISTER_SAMPLE_PIPELINE, samplePipeline).sendToTarget();
-
int samplePipelineIndex = tracksAddedCount;
tracksAddedCount++;
@@ -458,7 +459,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
tracksAddedCount++;
}
- return new SamplePipelineInput(samplePipelineIndex, samplePipeline.expectsDecodedData());
+ return new SampleConsumerImpl(samplePipelineIndex, samplePipeline);
}
// MuxerWrapper.Listener implementation.
@@ -523,7 +524,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
transformationRequest,
videoEffects,
frameProcessorFactory,
- decoderFactory,
encoderFactory,
muxerWrapper,
/* errorConsumer= */ this::onTransformationError,
@@ -622,19 +622,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return false;
}
- private class SamplePipelineInput implements SamplePipeline.Input {
+ private class SampleConsumerImpl implements SampleConsumer {
private final int samplePipelineIndex;
- private final boolean expectsDecodedData;
+ private final SamplePipeline samplePipeline;
- public SamplePipelineInput(int samplePipelineIndex, boolean expectsDecodedData) {
+ public SampleConsumerImpl(int samplePipelineIndex, SamplePipeline samplePipeline) {
this.samplePipelineIndex = samplePipelineIndex;
- this.expectsDecodedData = expectsDecodedData;
+ this.samplePipeline = samplePipeline;
}
@Override
public boolean expectsDecodedData() {
- return expectsDecodedData;
+ return samplePipeline.expectsDecodedData();
}
@Nullable
@@ -649,7 +649,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// start of the sample pipelines). Having 2 thread hops per sample (one for dequeuing and
// one for queuing) makes transmuxing slower than it used to be.
internalHandler
- .obtainMessage(MSG_DEQUEUE_INPUT, samplePipelineIndex, /* unused */ 0)
+ .obtainMessage(MSG_DEQUEUE_BUFFER, samplePipelineIndex, /* unused */ 0)
.sendToTarget();
clock.onThreadBlocked();
dequeueBufferConditionVariable.blockUninterruptible();
@@ -660,9 +660,34 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void queueInputBuffer() {
internalHandler
- .obtainMessage(MSG_QUEUE_INPUT, samplePipelineIndex, /* unused */ 0)
+ .obtainMessage(MSG_QUEUE_BUFFER, samplePipelineIndex, /* unused */ 0)
.sendToTarget();
}
+
+ @Override
+ public Surface getInputSurface() {
+ return samplePipeline.getInputSurface();
+ }
+
+ @Override
+ public ColorInfo getExpectedColorInfo() {
+ return samplePipeline.getExpectedColorInfo();
+ }
+
+ @Override
+ public int getPendingVideoFrameCount() {
+ return samplePipeline.getPendingVideoFrameCount();
+ }
+
+ @Override
+ public void registerVideoFrame() {
+ samplePipeline.registerVideoFrame();
+ }
+
+ @Override
+ public void signalEndOfVideoInput() {
+ samplePipeline.signalEndOfVideoInput();
+ }
}
}
}
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
index fbbd63dada..6dee51a3d3 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
@@ -49,23 +49,15 @@ import androidx.media3.effect.ScaleToFitTransformation;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.MoreExecutors;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.dataflow.qual.Pure;
-/**
- * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them.
- */
+/** Pipeline to process, re-encode and mux raw video frames. */
/* package */ final class VideoTranscodingSamplePipeline extends BaseSamplePipeline {
- private final int maxPendingFrameCount;
-
- private final DecoderInputBuffer decoderInputBuffer;
- private final Codec decoder;
- private final ArrayList decodeOnlyPresentationTimestamps;
-
private final FrameProcessor frameProcessor;
+ private final ColorInfo frameProcessorInputColor;
private final EncoderWrapper encoderWrapper;
private final DecoderInputBuffer encoderOutputBuffer;
@@ -84,7 +76,6 @@ import org.checkerframework.dataflow.qual.Pure;
TransformationRequest transformationRequest,
ImmutableList effects,
FrameProcessor.Factory frameProcessorFactory,
- Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
MuxerWrapper muxerWrapper,
Consumer errorConsumer,
@@ -131,11 +122,8 @@ import org.checkerframework.dataflow.qual.Pure;
finalFramePresentationTimeUs = C.TIME_UNSET;
- decoderInputBuffer =
- new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
encoderOutputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
- decodeOnlyPresentationTimestamps = new ArrayList<>();
// The decoder rotates encoded frames for display by inputFormat.rotationDegrees.
int decodedWidth =
@@ -169,7 +157,7 @@ import org.checkerframework.dataflow.qual.Pure;
ColorInfo encoderInputColor = encoderWrapper.getSupportedInputColor();
// If not tone mapping using OpenGL, the decoder will output the encoderInputColor,
// possibly by tone mapping.
- ColorInfo frameProcessorInputColor =
+ frameProcessorInputColor =
isGlToneMapping ? checkNotNull(inputFormat.colorInfo) : encoderInputColor;
// For consistency with the Android platform, OpenGL tone mapping outputs colors with
// C.COLOR_TRANSFER_GAMMA_2_2 instead of C.COLOR_TRANSFER_SDR, and outputs this as
@@ -236,57 +224,42 @@ import org.checkerframework.dataflow.qual.Pure;
frameProcessor.setInputFrameInfo(
new FrameInfo(
decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs));
-
- boolean isDecoderToneMappingRequired =
- ColorInfo.isTransferHdr(inputFormat.colorInfo)
- && !ColorInfo.isTransferHdr(frameProcessorInputColor);
- decoder =
- decoderFactory.createForVideoDecoding(
- inputFormat, frameProcessor.getInputSurface(), isDecoderToneMappingRequired);
- maxPendingFrameCount = decoder.getMaxPendingFrameCount();
}
@Override
- public boolean expectsDecodedData() {
- return false;
+ public Surface getInputSurface() {
+ return frameProcessor.getInputSurface();
}
@Override
- @Nullable
- public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {
- return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null;
+ public ColorInfo getExpectedColorInfo() {
+ return frameProcessorInputColor;
}
@Override
- public void queueInputBuffer() throws TransformationException {
- if (decoderInputBuffer.isDecodeOnly()) {
- decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs);
- }
- decoder.queueInputBuffer(decoderInputBuffer);
+ public void registerVideoFrame() {
+ frameProcessor.registerInputFrame();
+ }
+
+ @Override
+ public int getPendingVideoFrameCount() {
+ return frameProcessor.getPendingInputFrameCount();
+ }
+
+ @Override
+ public void signalEndOfVideoInput() {
+ frameProcessor.signalEndOfInput();
}
@Override
public void release() {
frameProcessor.release();
- decoder.release();
encoderWrapper.release();
}
@Override
- protected boolean processDataUpToMuxer() throws TransformationException {
- if (decoder.isEnded()) {
- return false;
- }
-
- boolean processedData = false;
- while (maybeProcessDecoderOutput()) {
- processedData = true;
- }
- if (decoder.isEnded()) {
- frameProcessor.signalEndOfInput();
- }
- // If the decoder produced output, signal that it may be possible to process data again.
- return processedData;
+ protected boolean processDataUpToMuxer() {
+ return false;
}
@Override
@@ -377,46 +350,6 @@ import org.checkerframework.dataflow.qual.Pure;
|| Build.ID.startsWith(/* Pixel Watch */ "rwd9.220429.053"));
}
- /**
- * Feeds at most one decoder output frame to the next step of the pipeline.
- *
- * @return Whether a frame was processed.
- * @throws TransformationException If a problem occurs while processing the frame.
- */
- private boolean maybeProcessDecoderOutput() throws TransformationException {
- @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo();
- if (decoderOutputBufferInfo == null) {
- return false;
- }
-
- if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) {
- decoder.releaseOutputBuffer(/* render= */ false);
- return true;
- }
-
- if (maxPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT
- && frameProcessor.getPendingInputFrameCount() == maxPendingFrameCount) {
- return false;
- }
-
- frameProcessor.registerInputFrame();
- decoder.releaseOutputBuffer(/* render= */ true);
- return true;
- }
-
- private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
- // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would
- // box presentationTimeUs, creating a Long object that would need to be garbage collected.
- int size = decodeOnlyPresentationTimestamps.size();
- for (int i = 0; i < size; i++) {
- if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) {
- decodeOnlyPresentationTimestamps.remove(i);
- return true;
- }
- }
- return false;
- }
-
/**
* Wraps an {@linkplain Codec encoder} and provides its input {@link Surface}.
*
diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java
index 583653a150..fa8a55bd28 100644
--- a/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java
+++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java
@@ -68,7 +68,7 @@ public class ExoPlayerAssetLoaderTest {
}
@Override
- public SamplePipeline.Input onTrackAdded(
+ public SampleConsumer onTrackAdded(
Format format,
@AssetLoader.SupportedOutputTypes int supportedOutputTypes,
long streamStartPositionUs,
@@ -81,7 +81,7 @@ public class ExoPlayerAssetLoaderTest {
new IllegalStateException("onTrackAdded() called before onTrackCount()"));
}
isTrackAdded.set(true);
- return new FakeSamplePipelineInput();
+ return new FakeSampleConsumer();
}
@Override
@@ -130,7 +130,7 @@ public class ExoPlayerAssetLoaderTest {
.createAssetLoader();
}
- private static final class FakeSamplePipelineInput implements SamplePipeline.Input {
+ private static final class FakeSampleConsumer implements SampleConsumer {
@Override
public boolean expectsDecodedData() {
diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java
index 73feb48624..69a1cc27ff 100644
--- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java
+++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java
@@ -633,18 +633,18 @@ public final class TransformerEndToEndTest {
@Test
public void startTransformation_withAssetLoaderAlwaysDecoding_pipelineExpectsDecoded()
throws Exception {
- AtomicReference samplePipelineInputRef = new AtomicReference<>();
+ AtomicReference sampleConsumerRef = new AtomicReference<>();
Transformer transformer =
createTransformerBuilder(/* enableFallback= */ false)
.setAssetLoaderFactory(
- new FakeAssetLoader.Factory(SUPPORTED_OUTPUT_TYPE_DECODED, samplePipelineInputRef))
+ new FakeAssetLoader.Factory(SUPPORTED_OUTPUT_TYPE_DECODED, sampleConsumerRef))
.build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
transformer.startTransformation(mediaItem, outputPath);
- runLooperUntil(transformer.getApplicationLooper(), () -> samplePipelineInputRef.get() != null);
+ runLooperUntil(transformer.getApplicationLooper(), () -> sampleConsumerRef.get() != null);
- assertThat(samplePipelineInputRef.get().expectsDecodedData()).isTrue();
+ assertThat(sampleConsumerRef.get().expectsDecodedData()).isTrue();
}
@Test
@@ -654,7 +654,7 @@ public final class TransformerEndToEndTest {
.setAudioProcessors(ImmutableList.of(new SonicAudioProcessor()))
.setAssetLoaderFactory(
new FakeAssetLoader.Factory(
- SUPPORTED_OUTPUT_TYPE_ENCODED, /* samplePipelineInputRef= */ null))
+ SUPPORTED_OUTPUT_TYPE_ENCODED, /* sampleConsumerRef= */ null))
.build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
@@ -1077,15 +1077,15 @@ public final class TransformerEndToEndTest {
public static final class Factory implements AssetLoader.Factory {
private final @SupportedOutputTypes int supportedOutputTypes;
- @Nullable private final AtomicReference samplePipelineInputRef;
+ @Nullable private final AtomicReference sampleConsumerRef;
@Nullable private AssetLoader.Listener listener;
public Factory(
@SupportedOutputTypes int supportedOutputTypes,
- @Nullable AtomicReference samplePipelineInputRef) {
+ @Nullable AtomicReference sampleConsumerRef) {
this.supportedOutputTypes = supportedOutputTypes;
- this.samplePipelineInputRef = samplePipelineInputRef;
+ this.sampleConsumerRef = sampleConsumerRef;
}
@Override
@@ -1136,22 +1136,21 @@ public final class TransformerEndToEndTest {
@Override
public AssetLoader createAssetLoader() {
- return new FakeAssetLoader(
- checkNotNull(listener), supportedOutputTypes, samplePipelineInputRef);
+ return new FakeAssetLoader(checkNotNull(listener), supportedOutputTypes, sampleConsumerRef);
}
}
private final AssetLoader.Listener listener;
private final @SupportedOutputTypes int supportedOutputTypes;
- @Nullable private final AtomicReference samplePipelineInputRef;
+ @Nullable private final AtomicReference sampleConsumerRef;
public FakeAssetLoader(
Listener listener,
@SupportedOutputTypes int supportedOutputTypes,
- @Nullable AtomicReference samplePipelineInputRef) {
+ @Nullable AtomicReference sampleConsumerRef) {
this.listener = listener;
this.supportedOutputTypes = supportedOutputTypes;
- this.samplePipelineInputRef = samplePipelineInputRef;
+ this.sampleConsumerRef = sampleConsumerRef;
}
@Override
@@ -1165,14 +1164,14 @@ public final class TransformerEndToEndTest {
.setChannelCount(2)
.build();
try {
- SamplePipeline.Input samplePipelineInput =
+ SampleConsumer sampleConsumer =
listener.onTrackAdded(
format,
supportedOutputTypes,
/* streamStartPositionUs= */ 0,
/* streamOffsetUs= */ 0);
- if (samplePipelineInputRef != null) {
- samplePipelineInputRef.set(samplePipelineInput);
+ if (sampleConsumerRef != null) {
+ sampleConsumerRef.set(sampleConsumer);
}
} catch (TransformationException e) {
throw new IllegalStateException(e);