Move audio decoding to AssetLoader

PiperOrigin-RevId: 491933937

libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java

@@ -41,14 +41,14 @@ import org.checkerframework.dataflow.qual.Pure;
 
   private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
 
-  private final Codec decoder;
-  private final DecoderInputBuffer decoderInputBuffer;
+  private final DecoderInputBuffer inputBuffer;
   private final AudioProcessingPipeline audioProcessingPipeline;
   private final Codec encoder;
   private final AudioFormat encoderInputAudioFormat;
   private final DecoderInputBuffer encoderInputBuffer;
   private final DecoderInputBuffer encoderOutputBuffer;
 
+  private boolean hasPendingInputBuffer;
   private long nextEncoderInputBufferTimeUs;
   private long encoderBufferDurationRemainder;
 
@@ -58,7 +58,6 @@ import org.checkerframework.dataflow.qual.Pure;
       long streamOffsetUs,
       TransformationRequest transformationRequest,
       ImmutableList<AudioProcessor> audioProcessors,
-      Codec.DecoderFactory decoderFactory,
       Codec.EncoderFactory encoderFactory,
       MuxerWrapper muxerWrapper,
       Listener listener,
@@ -72,12 +71,10 @@ import org.checkerframework.dataflow.qual.Pure;
         muxerWrapper,
         listener);
 
-    decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
+    inputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
     encoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
     encoderOutputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
 
-    decoder = decoderFactory.createForAudioDecoding(inputFormat);
-
     if (transformationRequest.flattenForSlowMotion) {
       audioProcessors =
           new ImmutableList.Builder<AudioProcessor>()
@@ -139,30 +136,34 @@ import org.checkerframework.dataflow.qual.Pure;
     nextEncoderInputBufferTimeUs = streamOffsetUs;
   }
 
+  @Override
+  public boolean expectsDecodedData() {
+    return true;
+  }
+
   @Override
   public void release() {
     audioProcessingPipeline.reset();
-    decoder.release();
     encoder.release();
   }
 
   @Override
   @Nullable
-  protected DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException {
-    return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null;
+  protected DecoderInputBuffer dequeueInputBufferInternal() {
+    return hasPendingInputBuffer ? null : inputBuffer;
   }
 
   @Override
-  protected void queueInputBufferInternal() throws TransformationException {
-    decoder.queueInputBuffer(decoderInputBuffer);
+  protected void queueInputBufferInternal() {
+    hasPendingInputBuffer = true;
   }
 
   @Override
   protected boolean processDataUpToMuxer() throws TransformationException {
     if (audioProcessingPipeline.isOperational()) {
-      return feedEncoderFromProcessingPipeline() || feedProcessingPipelineFromDecoder();
+      return feedEncoderFromProcessingPipeline() || feedProcessingPipelineFromInput();
     } else {
-      return feedEncoderFromDecoder();
+      return feedEncoderFromInput();
     }
   }
 
@@ -195,27 +196,25 @@ import org.checkerframework.dataflow.qual.Pure;
   }
 
   /**
-   * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to
-   * pass more data immediately by calling this method again.
+   * Attempts to pass input data to the encoder.
+   *
+   * @return Whether it may be possible to feed more data immediately by calling this method again.
    */
-  private boolean feedEncoderFromDecoder() throws TransformationException {
-    if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
+  private boolean feedEncoderFromInput() throws TransformationException {
+    if (!hasPendingInputBuffer || !encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
       return false;
     }
 
-    if (decoder.isEnded()) {
+    if (inputBuffer.isEndOfStream()) {
       queueEndOfStreamToEncoder();
+      hasPendingInputBuffer = false;
       return false;
     }
 
-    @Nullable ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer();
-    if (decoderOutputBuffer == null) {
-      return false;
-    }
-
-    feedEncoder(decoderOutputBuffer);
-    if (!decoderOutputBuffer.hasRemaining()) {
-      decoder.releaseOutputBuffer(/* render= */ false);
+    ByteBuffer inputData = checkNotNull(inputBuffer.data);
+    feedEncoder(inputData);
+    if (!inputData.hasRemaining()) {
+      hasPendingInputBuffer = false;
     }
     return true;
   }
@@ -223,7 +222,7 @@ import org.checkerframework.dataflow.qual.Pure;
   /**
    * Attempts to feed audio processor output data to the encoder.
    *
-   * @return Whether more data can be fed immediately, by calling this method again.
+   * @return Whether it may be possible to feed more data immediately by calling this method again.
    */
   private boolean feedEncoderFromProcessingPipeline() throws TransformationException {
     if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
@@ -244,28 +243,28 @@ import org.checkerframework.dataflow.qual.Pure;
   }
 
   /**
-   * Attempts to feed decoder output data to the {@link AudioProcessingPipeline}.
+   * Attempts to feed input data to the {@link AudioProcessingPipeline}.
    *
    * @return Whether it may be possible to feed more data immediately by calling this method again.
    */
-  private boolean feedProcessingPipelineFromDecoder() throws TransformationException {
-    if (decoder.isEnded()) {
+  private boolean feedProcessingPipelineFromInput() {
+    if (!hasPendingInputBuffer) {
+      return false;
+    }
+
+    if (inputBuffer.isEndOfStream()) {
       audioProcessingPipeline.queueEndOfStream();
+      hasPendingInputBuffer = false;
       return false;
     }
     checkState(!audioProcessingPipeline.isEnded());
 
-    @Nullable ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer();
-    if (decoderOutputBuffer == null) {
+    ByteBuffer inputData = checkNotNull(inputBuffer.data);
+    audioProcessingPipeline.queueInput(inputData);
+    if (inputData.hasRemaining()) {
       return false;
     }
-
-    audioProcessingPipeline.queueInput(decoderOutputBuffer);
-    if (decoderOutputBuffer.hasRemaining()) {
-      return false;
-    }
-    // Decoder output buffer was fully consumed by the processing pipeline.
-    decoder.releaseOutputBuffer(/* render= */ false);
+    hasPendingInputBuffer = false;
     return true;
   }
 

libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java

@@ -69,6 +69,7 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
       boolean removeAudio,
       boolean removeVideo,
       MediaSource.Factory mediaSourceFactory,
+      Codec.DecoderFactory decoderFactory,
       Looper looper,
       Listener listener,
       Clock clock) {
@@ -89,7 +90,9 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
                 DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10)
             .build();
     ExoPlayer.Builder playerBuilder =
-        new ExoPlayer.Builder(context, new RenderersFactoryImpl(removeAudio, removeVideo, listener))
+        new ExoPlayer.Builder(
+                context,
+                new RenderersFactoryImpl(removeAudio, removeVideo, decoderFactory, listener))
             .setMediaSourceFactory(mediaSourceFactory)
             .setTrackSelector(trackSelector)
             .setLoadControl(loadControl)
@@ -120,14 +123,17 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
     private final TransformerMediaClock mediaClock;
     private final boolean removeAudio;
     private final boolean removeVideo;
+    private final Codec.DecoderFactory decoderFactory;
     private final ExoPlayerAssetLoader.Listener assetLoaderListener;
 
     public RenderersFactoryImpl(
         boolean removeAudio,
         boolean removeVideo,
+        Codec.DecoderFactory decoderFactory,
         ExoPlayerAssetLoader.Listener assetLoaderListener) {
       this.removeAudio = removeAudio;
       this.removeVideo = removeVideo;
+      this.decoderFactory = decoderFactory;
       this.assetLoaderListener = assetLoaderListener;
       mediaClock = new TransformerMediaClock();
     }
@@ -144,12 +150,14 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
       int index = 0;
       if (!removeAudio) {
         renderers[index] =
-            new ExoPlayerAssetLoaderRenderer(C.TRACK_TYPE_AUDIO, mediaClock, assetLoaderListener);
+            new ExoPlayerAssetLoaderRenderer(
+                C.TRACK_TYPE_AUDIO, decoderFactory, mediaClock, assetLoaderListener);
         index++;
       }
       if (!removeVideo) {
         renderers[index] =
-            new ExoPlayerAssetLoaderRenderer(C.TRACK_TYPE_VIDEO, mediaClock, assetLoaderListener);
+            new ExoPlayerAssetLoaderRenderer(
+                C.TRACK_TYPE_VIDEO, decoderFactory, mediaClock, assetLoaderListener);
         index++;
       }
       return renderers;

libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java

@@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED;
 import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT;
 
+import android.media.MediaCodec;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
@@ -30,6 +31,7 @@ import androidx.media3.exoplayer.FormatHolder;
 import androidx.media3.exoplayer.MediaClock;
 import androidx.media3.exoplayer.RendererCapabilities;
 import androidx.media3.exoplayer.source.SampleStream.ReadDataResult;
+import java.nio.ByteBuffer;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@@ -38,6 +40,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
   private static final String TAG = "ExoPlayerAssetLoaderRenderer";
 
+  private final Codec.DecoderFactory decoderFactory;
   private final TransformerMediaClock mediaClock;
   private final ExoPlayerAssetLoader.Listener assetLoaderListener;
   private final DecoderInputBuffer decoderInputBuffer;
@@ -45,14 +48,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   private boolean isTransformationRunning;
   private long streamStartPositionUs;
   private long streamOffsetUs;
+  private @MonotonicNonNull Codec decoder;
+  @Nullable private ByteBuffer pendingDecoderOutputBuffer;
   private SamplePipeline.@MonotonicNonNull Input samplePipelineInput;
   private boolean isEnded;
 
   public ExoPlayerAssetLoaderRenderer(
       int trackType,
+      Codec.DecoderFactory decoderFactory,
       TransformerMediaClock mediaClock,
       ExoPlayerAssetLoader.Listener assetLoaderListener) {
     super(trackType);
+    this.decoderFactory = decoderFactory;
     this.mediaClock = mediaClock;
     this.assetLoaderListener = assetLoaderListener;
     decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
@@ -99,7 +106,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
         return;
       }
 
-      while (feedPipelineFromInput()) {}
+      if (samplePipelineInput.expectsDecodedData()) {
+        while (feedPipelineFromDecoder() || feedDecoderFromInput()) {}
+      } else {
+        while (feedPipelineFromInput()) {}
+      }
     } catch (TransformationException e) {
       isTransformationRunning = false;
       assetLoaderListener.onError(e);
@@ -128,6 +139,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     isTransformationRunning = false;
   }
 
+  @Override
+  protected void onReset() {
+    if (decoder != null) {
+      decoder.release();
+    }
+  }
+
   @EnsuresNonNullIf(expression = "samplePipelineInput", result = true)
   private boolean ensureConfigured() throws TransformationException {
     if (samplePipelineInput != null) {
@@ -143,6 +161,74 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     Format inputFormat = checkNotNull(formatHolder.format);
     samplePipelineInput =
         assetLoaderListener.onTrackAdded(inputFormat, streamStartPositionUs, streamOffsetUs);
+    if (samplePipelineInput.expectsDecodedData()) {
+      decoder = decoderFactory.createForAudioDecoding(inputFormat);
+    }
+    return true;
+  }
+
+  /**
+   * Attempts to read decoded data and pass it to the sample pipeline.
+   *
+   * @return Whether it may be possible to read more data immediately by calling this method again.
+   * @throws TransformationException If an error occurs in the decoder or in the {@link
+   *     SamplePipeline}.
+   */
+  @RequiresNonNull("samplePipelineInput")
+  private boolean feedPipelineFromDecoder() throws TransformationException {
+    @Nullable
+    DecoderInputBuffer samplePipelineInputBuffer = samplePipelineInput.dequeueInputBuffer();
+    if (samplePipelineInputBuffer == null) {
+      return false;
+    }
+
+    Codec decoder = checkNotNull(this.decoder);
+    if (pendingDecoderOutputBuffer != null) {
+      if (pendingDecoderOutputBuffer.hasRemaining()) {
+        return false;
+      } else {
+        decoder.releaseOutputBuffer(/* render= */ false);
+        pendingDecoderOutputBuffer = null;
+      }
+    }
+
+    if (decoder.isEnded()) {
+      samplePipelineInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM);
+      samplePipelineInput.queueInputBuffer();
+      isEnded = true;
+      return false;
+    }
+
+    pendingDecoderOutputBuffer = decoder.getOutputBuffer();
+    if (pendingDecoderOutputBuffer == null) {
+      return false;
+    }
+
+    samplePipelineInputBuffer.data = pendingDecoderOutputBuffer;
+    MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo());
+    samplePipelineInputBuffer.timeUs = bufferInfo.presentationTimeUs;
+    samplePipelineInputBuffer.setFlags(bufferInfo.flags);
+    samplePipelineInput.queueInputBuffer();
+    return true;
+  }
+
+  /**
+   * Attempts to read input data and pass it to the decoder.
+   *
+   * @return Whether it may be possible to read more data immediately by calling this method again.
+   * @throws TransformationException If an error occurs in the decoder.
+   */
+  private boolean feedDecoderFromInput() throws TransformationException {
+    Codec decoder = checkNotNull(this.decoder);
+    if (!decoder.maybeDequeueInputBuffer(decoderInputBuffer)) {
+      return false;
+    }
+
+    if (!readInput(decoderInputBuffer)) {
+      return false;
+    }
+
+    decoder.queueInputBuffer(decoderInputBuffer);
     return true;
   }
 
@@ -159,18 +245,32 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       return false;
     }
 
-    @ReadDataResult
-    int result = readSource(getFormatHolder(), samplePipelineInputBuffer, /* readFlags= */ 0);
+    if (!readInput(samplePipelineInputBuffer)) {
+      return false;
+    }
+
+    samplePipelineInput.queueInputBuffer();
+    if (samplePipelineInputBuffer.isEndOfStream()) {
+      isEnded = true;
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Attempts to populate {@code buffer} with input data.
+   *
+   * @param buffer The buffer to populate.
+   * @return Whether the {@code buffer} has been populated.
+   */
+  private boolean readInput(DecoderInputBuffer buffer) {
+    @ReadDataResult int result = readSource(getFormatHolder(), buffer, /* readFlags= */ 0);
     switch (result) {
       case C.RESULT_BUFFER_READ:
-        samplePipelineInputBuffer.flip();
-        if (samplePipelineInputBuffer.isEndOfStream()) {
-          samplePipelineInput.queueInputBuffer();
-          isEnded = true;
-          return false;
+        buffer.flip();
+        if (!buffer.isEndOfStream()) {
+          mediaClock.updateTimeForTrackType(getTrackType(), buffer.timeUs);
         }
-        mediaClock.updateTimeForTrackType(getTrackType(), samplePipelineInputBuffer.timeUs);
-        samplePipelineInput.queueInputBuffer();
         return true;
       case C.RESULT_FORMAT_READ:
         throw new IllegalStateException("Format changes are not supported.");

libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java

@@ -48,6 +48,11 @@ import androidx.media3.decoder.DecoderInputBuffer;
     fallbackListener.onTransformationRequestFinalized(transformationRequest);
   }
 
+  @Override
+  public boolean expectsDecodedData() {
+    return false;
+  }
+
   @Override
   public void release() {}
 

libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java

@@ -29,6 +29,9 @@ import androidx.media3.decoder.DecoderInputBuffer;
   /** Input of a {@link SamplePipeline}. */
   interface Input {
 
+    /** See {@link SamplePipeline#expectsDecodedData()}. */
+    boolean expectsDecodedData();
+
     /** See {@link SamplePipeline#dequeueInputBuffer()}. */
     @Nullable
     DecoderInputBuffer dequeueInputBuffer();
@@ -56,6 +59,12 @@ import androidx.media3.decoder.DecoderInputBuffer;
     void onTransformationError(TransformationException exception);
   }
 
+  /**
+   * Returns whether the pipeline should be fed with decoded sample data. If false, encoded sample
+   * data should be queued.
+   */
+  boolean expectsDecodedData();
+
   /** Returns a buffer if the pipeline is ready to accept input, and {@code null} otherwise. */
   @Nullable
   DecoderInputBuffer dequeueInputBuffer() throws TransformationException;

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java

@@ -163,6 +163,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             removeAudio,
             removeVideo,
             mediaSourceFactory,
+            decoderFactory,
             internalLooper,
             componentListener,
             clock);
@@ -409,7 +410,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
       int samplePipelineIndex = tracksAddedCount;
       tracksAddedCount++;
-      return new SamplePipelineInput(samplePipelineIndex);
+      return new SamplePipelineInput(samplePipelineIndex, samplePipeline.expectsDecodedData());
     }
 
     @Override
@@ -458,7 +459,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             streamOffsetUs,
             transformationRequest,
             audioProcessors,
-            decoderFactory,
             encoderFactory,
             muxerWrapper,
             /* listener= */ this,
@@ -573,9 +573,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     private class SamplePipelineInput implements SamplePipeline.Input {
 
       private final int samplePipelineIndex;
+      private final boolean expectsDecodedData;
 
-      public SamplePipelineInput(int samplePipelineIndex) {
+      public SamplePipelineInput(int samplePipelineIndex, boolean expectsDecodedData) {
         this.samplePipelineIndex = samplePipelineIndex;
+        this.expectsDecodedData = expectsDecodedData;
+      }
+
+      @Override
+      public boolean expectsDecodedData() {
+        return expectsDecodedData;
       }
 
       @Nullable

libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java

@@ -207,6 +207,11 @@ import org.checkerframework.dataflow.qual.Pure;
     maxPendingFrameCount = decoder.getMaxPendingFrameCount();
   }
 
+  @Override
+  public boolean expectsDecodedData() {
+    return false;
+  }
+
   @Override
   public void release() {
     frameProcessor.release();