Move muxing inside sample pipelines

This logic is currently in the player renderers. With multi-asset, the
renderers will go into the AssetLoader, which shouldn't be responsible
for muxing.

PiperOrigin-RevId: 486860502

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java

@@ -28,14 +28,13 @@ import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.util.Util;
 import java.nio.ByteBuffer;
-import java.util.List;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 import org.checkerframework.dataflow.qual.Pure;
 
 /**
  * Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them.
  */
-/* package */ final class AudioTranscodingSamplePipeline implements SamplePipeline {
+/* package */ final class AudioTranscodingSamplePipeline extends BaseSamplePipeline {
 
   private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
 
@@ -57,12 +56,15 @@ import org.checkerframework.dataflow.qual.Pure;
   public AudioTranscodingSamplePipeline(
       Format inputFormat,
       long streamOffsetUs,
+      long streamStartPositionUs,
       TransformationRequest transformationRequest,
       Codec.DecoderFactory decoderFactory,
       Codec.EncoderFactory encoderFactory,
-      List<String> allowedOutputMimeTypes,
+      MuxerWrapper muxerWrapper,
       FallbackListener fallbackListener)
       throws TransformationException {
+    super(C.TRACK_TYPE_AUDIO, streamStartPositionUs, muxerWrapper);
+
     decoderInputBuffer =
         new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
     encoderInputBuffer =
@@ -104,7 +106,9 @@ import org.checkerframework.dataflow.qual.Pure;
             .setChannelCount(encoderInputAudioFormat.channelCount)
             .setAverageBitrate(DEFAULT_ENCODER_BITRATE)
             .build();
-    encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat, allowedOutputMimeTypes);
+    encoder =
+        encoderFactory.createForAudioEncoding(
+            requestedOutputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO));
 
     fallbackListener.onTransformationRequestFinalized(
         createFallbackTransformationRequest(
@@ -126,7 +130,16 @@ import org.checkerframework.dataflow.qual.Pure;
   }
 
   @Override
-  public boolean processData() throws TransformationException {
+  public void release() {
+    if (speedChangingAudioProcessor != null) {
+      speedChangingAudioProcessor.reset();
+    }
+    decoder.release();
+    encoder.release();
+  }
+
+  @Override
+  protected boolean processDataUpToMuxer() throws TransformationException {
     if (speedChangingAudioProcessor != null) {
       return feedEncoderFromProcessor() || feedProcessorFromDecoder();
     } else {
@@ -136,13 +149,13 @@ import org.checkerframework.dataflow.qual.Pure;
 
   @Override
   @Nullable
-  public Format getOutputFormat() throws TransformationException {
+  protected Format getMuxerInputFormat() throws TransformationException {
     return encoder.getOutputFormat();
   }
 
   @Override
   @Nullable
-  public DecoderInputBuffer getOutputBuffer() throws TransformationException {
+  protected DecoderInputBuffer getMuxerInputBuffer() throws TransformationException {
     encoderOutputBuffer.data = encoder.getOutputBuffer();
     if (encoderOutputBuffer.data == null) {
       return null;
@@ -153,24 +166,15 @@ import org.checkerframework.dataflow.qual.Pure;
   }
 
   @Override
-  public void releaseOutputBuffer() throws TransformationException {
+  protected void releaseMuxerInputBuffer() throws TransformationException {
     encoder.releaseOutputBuffer(/* render= */ false);
   }
 
   @Override
-  public boolean isEnded() {
+  protected boolean isMuxerInputEnded() {
     return encoder.isEnded();
   }
 
-  @Override
-  public void release() {
-    if (speedChangingAudioProcessor != null) {
-      speedChangingAudioProcessor.reset();
-    }
-    decoder.release();
-    encoder.release();
-  }
-
   /**
    * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to
    * pass more data immediately by calling this method again.

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
+
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+
+/* package */ abstract class BaseSamplePipeline implements SamplePipeline {
+
+  private final int trackType;
+  private final long streamStartPositionUs;
+  private final MuxerWrapper muxerWrapper;
+
+  private boolean muxerWrapperTrackAdded;
+  private boolean isEnded;
+
+  public BaseSamplePipeline(int trackType, long streamStartPositionUs, MuxerWrapper muxerWrapper) {
+    this.trackType = trackType;
+    this.streamStartPositionUs = streamStartPositionUs;
+    this.muxerWrapper = muxerWrapper;
+  }
+
+  @Override
+  public boolean processData() throws TransformationException {
+    return feedMuxer() || processDataUpToMuxer();
+  }
+
+  @Override
+  public boolean isEnded() {
+    return isEnded;
+  }
+
+  protected abstract boolean processDataUpToMuxer() throws TransformationException;
+
+  @Nullable
+  protected abstract Format getMuxerInputFormat() throws TransformationException;
+
+  @Nullable
+  protected abstract DecoderInputBuffer getMuxerInputBuffer() throws TransformationException;
+
+  protected abstract void releaseMuxerInputBuffer() throws TransformationException;
+
+  protected abstract boolean isMuxerInputEnded();
+
+  /**
+   * Attempts to pass encoded data to the muxer, and returns whether it may be possible to pass more
+   * data immediately by calling this method again.
+   */
+  private boolean feedMuxer() throws TransformationException {
+    if (!muxerWrapperTrackAdded) {
+      @Nullable Format inputFormat = getMuxerInputFormat();
+      if (inputFormat == null) {
+        return false;
+      }
+      try {
+        muxerWrapper.addTrackFormat(inputFormat);
+      } catch (Muxer.MuxerException e) {
+        throw TransformationException.createForMuxer(
+            e, TransformationException.ERROR_CODE_MUXING_FAILED);
+      }
+      muxerWrapperTrackAdded = true;
+    }
+
+    if (isMuxerInputEnded()) {
+      muxerWrapper.endTrack(trackType);
+      isEnded = true;
+      return false;
+    }
+
+    @Nullable DecoderInputBuffer muxerInputBuffer = getMuxerInputBuffer();
+    if (muxerInputBuffer == null) {
+      return false;
+    }
+
+    long samplePresentationTimeUs = muxerInputBuffer.timeUs - streamStartPositionUs;
+    // TODO(b/204892224): Consider subtracting the first sample timestamp from the sample pipeline
+    //  buffer from all samples so that they are guaranteed to start from zero in the output file.
+    try {
+      if (!muxerWrapper.writeSample(
+          trackType,
+          checkStateNotNull(muxerInputBuffer.data),
+          muxerInputBuffer.isKeyFrame(),
+          samplePresentationTimeUs)) {
+        return false;
+      }
+    } catch (Muxer.MuxerException e) {
+      throw TransformationException.createForMuxer(
+          e, TransformationException.ERROR_CODE_MUXING_FAILED);
+    }
+
+    releaseMuxerInputBuffer();
+    return true;
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java

@@ -19,9 +19,10 @@ package com.google.android.exoplayer2.transformer;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.util.MimeTypes;
 
 /** Pipeline that passes through the samples without any re-encoding or transformation. */
-/* package */ final class PassthroughSamplePipeline implements SamplePipeline {
+/* package */ final class PassthroughSamplePipeline extends BaseSamplePipeline {
 
   private final DecoderInputBuffer buffer;
   private final Format format;
@@ -30,8 +31,11 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 
   public PassthroughSamplePipeline(
       Format format,
+      long streamStartPositionUs,
       TransformationRequest transformationRequest,
+      MuxerWrapper muxerWrapper,
       FallbackListener fallbackListener) {
+    super(MimeTypes.getTrackType(format.sampleMimeType), streamStartPositionUs, muxerWrapper);
     this.format = format;
     buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
     hasPendingBuffer = false;
@@ -46,36 +50,38 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 
   @Override
   public void queueInputBuffer() {
+    if (buffer.data != null && buffer.data.hasRemaining()) {
       hasPendingBuffer = true;
     }
+  }
 
   @Override
-  public boolean processData() {
+  public void release() {}
+
+  @Override
+  protected boolean processDataUpToMuxer() {
     return false;
   }
 
   @Override
-  public Format getOutputFormat() {
+  protected Format getMuxerInputFormat() {
     return format;
   }
 
   @Override
   @Nullable
-  public DecoderInputBuffer getOutputBuffer() {
+  protected DecoderInputBuffer getMuxerInputBuffer() {
     return hasPendingBuffer ? buffer : null;
   }
 
   @Override
-  public void releaseOutputBuffer() {
+  protected void releaseMuxerInputBuffer() {
     buffer.clear();
     hasPendingBuffer = false;
   }
 
   @Override
-  public boolean isEnded() {
+  protected boolean isMuxerInputEnded() {
     return buffer.isEndOfStream();
   }
-
-  @Override
-  public void release() {}
 }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java

@@ -17,7 +17,6 @@
 package com.google.android.exoplayer2.transformer;
 
 import androidx.annotation.Nullable;
-import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 
 /**
@@ -45,21 +44,6 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
    */
   boolean processData() throws TransformationException;
 
-  /** Returns the output format of the pipeline if available, and {@code null} otherwise. */
-  @Nullable
-  Format getOutputFormat() throws TransformationException;
-
-  /** Returns an output buffer if the pipeline has produced output, and {@code null} otherwise */
-  @Nullable
-  DecoderInputBuffer getOutputBuffer() throws TransformationException;
-
-  /**
-   * Releases the pipeline's output buffer.
-   *
-   * <p>Should be called when the output buffer from {@link #getOutputBuffer()} is no longer needed.
-   */
-  void releaseOutputBuffer() throws TransformationException;
-
   /** Returns whether the pipeline has ended. */
   boolean isEnded();
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java

@@ -77,16 +77,22 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult;
     Format inputFormat = checkNotNull(formatHolder.format);
     if (shouldPassthrough(inputFormat)) {
       samplePipeline =
-          new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
+          new PassthroughSamplePipeline(
+              inputFormat,
+              streamStartPositionUs,
+              transformationRequest,
+              muxerWrapper,
+              fallbackListener);
     } else {
       samplePipeline =
           new AudioTranscodingSamplePipeline(
               inputFormat,
               streamOffsetUs,
+              streamStartPositionUs,
               transformationRequest,
               decoderFactory,
               encoderFactory,
-              muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
+              muxerWrapper,
               fallbackListener);
     }
     return true;

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java

@@ -41,8 +41,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   protected final FallbackListener fallbackListener;
 
   private boolean isTransformationRunning;
-  private boolean muxerWrapperTrackAdded;
-  private boolean muxerWrapperTrackEnded;
   protected long streamOffsetUs;
   protected long streamStartPositionUs;
   protected @MonotonicNonNull SamplePipeline samplePipeline;
@@ -88,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
   @Override
   public final boolean isEnded() {
-    return muxerWrapperTrackEnded;
+    return samplePipeline != null && samplePipeline.isEnded();
   }
 
   @Override
@@ -98,15 +96,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
         return;
       }
 
-      while (feedMuxerFromPipeline() || samplePipeline.processData() || feedPipelineFromInput()) {}
+      while (samplePipeline.processData() || feedPipelineFromInput()) {}
     } catch (TransformationException e) {
       isTransformationRunning = false;
       asyncErrorListener.onTransformationException(e);
-    } catch (Muxer.MuxerException e) {
-      isTransformationRunning = false;
-      asyncErrorListener.onTransformationException(
-          TransformationException.createForMuxer(
-              e, TransformationException.ERROR_CODE_MUXING_FAILED));
     }
   }
 
@@ -138,8 +131,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     if (samplePipeline != null) {
       samplePipeline.release();
     }
-    muxerWrapperTrackAdded = false;
-    muxerWrapperTrackEnded = false;
   }
 
   @ForOverride
@@ -152,49 +143,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     samplePipeline.queueInputBuffer();
   }
 
-  /**
-   * Attempts to write sample pipeline output data to the muxer.
-   *
-   * @return Whether it may be possible to write more data immediately by calling this method again.
-   * @throws Muxer.MuxerException If a muxing problem occurs.
-   * @throws TransformationException If a {@link SamplePipeline} problem occurs.
-   */
-  @RequiresNonNull("samplePipeline")
-  private boolean feedMuxerFromPipeline() throws Muxer.MuxerException, TransformationException {
-    if (!muxerWrapperTrackAdded) {
-      @Nullable Format samplePipelineOutputFormat = samplePipeline.getOutputFormat();
-      if (samplePipelineOutputFormat == null) {
-        return false;
-      }
-      muxerWrapperTrackAdded = true;
-      muxerWrapper.addTrackFormat(samplePipelineOutputFormat);
-    }
-
-    if (samplePipeline.isEnded()) {
-      muxerWrapper.endTrack(getTrackType());
-      muxerWrapperTrackEnded = true;
-      return false;
-    }
-
-    @Nullable DecoderInputBuffer samplePipelineOutputBuffer = samplePipeline.getOutputBuffer();
-    if (samplePipelineOutputBuffer == null) {
-      return false;
-    }
-
-    long samplePresentationTimeUs = samplePipelineOutputBuffer.timeUs - streamStartPositionUs;
-    // TODO(b/204892224): Consider subtracting the first sample timestamp from the sample pipeline
-    //  buffer from all samples so that they are guaranteed to start from zero in the output file.
-    if (!muxerWrapper.writeSample(
-        getTrackType(),
-        checkStateNotNull(samplePipelineOutputBuffer.data),
-        samplePipelineOutputBuffer.isKeyFrame(),
-        samplePresentationTimeUs)) {
-      return false;
-    }
-    samplePipeline.releaseOutputBuffer();
-    return true;
-  }
-
   /**
    * Attempts to read input data and pass the input data to the sample pipeline.
    *

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java

@@ -103,18 +103,24 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
               context,
               inputFormat,
               streamOffsetUs,
+              streamStartPositionUs,
               transformationRequest,
               effects,
               frameProcessorFactory,
               decoderFactory,
               encoderFactory,
-              muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
+              muxerWrapper,
               fallbackListener,
               asyncErrorListener,
               debugViewProvider);
     } else {
       samplePipeline =
-          new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
+          new PassthroughSamplePipeline(
+              inputFormat,
+              streamStartPositionUs,
+              transformationRequest,
+              muxerWrapper,
+              fallbackListener);
     }
     if (transformationRequest.flattenForSlowMotion) {
       sefSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat);

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -50,7 +50,7 @@ import org.checkerframework.dataflow.qual.Pure;
 /**
  * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them.
  */
-/* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline {
+/* package */ final class VideoTranscodingSamplePipeline extends BaseSamplePipeline {
 
   private final int maxPendingFrameCount;
 
@@ -67,16 +67,19 @@ import org.checkerframework.dataflow.qual.Pure;
       Context context,
       Format inputFormat,
       long streamOffsetUs,
+      long streamStartPositionUs,
       TransformationRequest transformationRequest,
       ImmutableList<Effect> effects,
       FrameProcessor.Factory frameProcessorFactory,
       Codec.DecoderFactory decoderFactory,
       Codec.EncoderFactory encoderFactory,
-      List<String> allowedOutputMimeTypes,
+      MuxerWrapper muxerWrapper,
       FallbackListener fallbackListener,
       Transformer.AsyncErrorListener asyncErrorListener,
       DebugViewProvider debugViewProvider)
       throws TransformationException {
+    super(C.TRACK_TYPE_VIDEO, streamStartPositionUs, muxerWrapper);
+
     if (ColorInfo.isTransferHdr(inputFormat.colorInfo)
         && (SDK_INT < 31 || deviceNeedsNoToneMappingWorkaround())) {
       throw TransformationException.createForCodec(
@@ -119,7 +122,7 @@ import org.checkerframework.dataflow.qual.Pure;
         new EncoderWrapper(
             encoderFactory,
             inputFormat,
-            allowedOutputMimeTypes,
+            muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_VIDEO),
             transformationRequest,
             fallbackListener);
 
@@ -199,7 +202,14 @@ import org.checkerframework.dataflow.qual.Pure;
   }
 
   @Override
-  public boolean processData() throws TransformationException {
+  public void release() {
+    frameProcessor.release();
+    decoder.release();
+    encoderWrapper.release();
+  }
+
+  @Override
+  protected boolean processDataUpToMuxer() throws TransformationException {
     if (decoder.isEnded()) {
       return false;
     }
@@ -217,13 +227,13 @@ import org.checkerframework.dataflow.qual.Pure;
 
   @Override
   @Nullable
-  public Format getOutputFormat() throws TransformationException {
+  protected Format getMuxerInputFormat() throws TransformationException {
     return encoderWrapper.getOutputFormat();
   }
 
   @Override
   @Nullable
-  public DecoderInputBuffer getOutputBuffer() throws TransformationException {
+  protected DecoderInputBuffer getMuxerInputBuffer() throws TransformationException {
     encoderOutputBuffer.data = encoderWrapper.getOutputBuffer();
     if (encoderOutputBuffer.data == null) {
       return null;
@@ -235,22 +245,15 @@ import org.checkerframework.dataflow.qual.Pure;
   }
 
   @Override
-  public void releaseOutputBuffer() throws TransformationException {
+  protected void releaseMuxerInputBuffer() throws TransformationException {
     encoderWrapper.releaseOutputBuffer(/* render= */ false);
   }
 
   @Override
-  public boolean isEnded() {
+  protected boolean isMuxerInputEnded() {
     return encoderWrapper.isEnded();
   }
 
-  @Override
-  public void release() {
-    frameProcessor.release();
-    decoder.release();
-    encoderWrapper.release();
-  }
-
   /**
    * Creates a {@link TransformationRequest}, based on an original {@code TransformationRequest} and
    * parameters specifying alterations to it that indicate device support.