Block secondary audio processing until primary format is configured

PiperOrigin-RevId: 673961235

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java

@@ -236,6 +236,27 @@ public interface AudioSink {
     /** The input {@link Format} of the sink when the error occurs. */
     public final Format format;
 
+    /**
+     * Creates a new instance.
+     *
+     * @param message The detail message for this exception.
+     * @param audioTrackState The underlying {@link AudioTrack}'s state.
+     * @param format The input format of the sink when the error occurs.
+     * @param isRecoverable Whether the exception can be recovered by recreating the sink.
+     * @param cause The {@link Throwable cause} of this exception.
+     */
+    public InitializationException(
+        String message,
+        int audioTrackState,
+        Format format,
+        boolean isRecoverable,
+        @Nullable Throwable cause) {
+      super(message, cause);
+      this.audioTrackState = audioTrackState;
+      this.isRecoverable = isRecoverable;
+      this.format = format;
+    }
+
     /**
      * Creates a new instance.
      *
@@ -255,7 +276,7 @@ public interface AudioSink {
         Format format,
         boolean isRecoverable,
         @Nullable Exception audioTrackException) {
-      super(
+      this(
           "AudioTrack init failed "
               + audioTrackState
               + " "
@@ -263,10 +284,10 @@ public interface AudioSink {
               + " "
               + format
               + (isRecoverable ? " (recoverable)" : ""),
+          audioTrackState,
+          format,
+          isRecoverable,
           audioTrackException);
-      this.audioTrackState = audioTrackState;
-      this.isRecoverable = isRecoverable;
-      this.format = format;
     }
   }
 

libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java

@@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
 
+import android.media.AudioTrack;
 import androidx.annotation.Nullable;
 import androidx.media3.common.AudioAttributes;
 import androidx.media3.common.AuxEffectInfo;
@@ -52,6 +53,8 @@ import java.util.Objects;
      * Returns the {@link AudioGraphInput} instance associated with this {@linkplain
      * AudioGraphInputAudioSink sink}.
      *
+     * <p>If AudioGraphInput is not available, callers should re-try again later.
+     *
      * <p>Data {@linkplain #handleBuffer written} to the sink will be {@linkplain
      * AudioGraphInput#queueInputBuffer() queued} to the {@link AudioGraphInput}.
      *
@@ -59,10 +62,11 @@ import java.util.Objects;
      *     AudioGraphInput}.
      * @param format The {@link Format} used to {@linkplain AudioGraphInputAudioSink#configure
      *     configure} the {@linkplain AudioGraphInputAudioSink sink}.
-     * @return The {@link AudioGraphInput}.
+     * @return The {@link AudioGraphInput}, or {@code null} if the input is not available yet.
      * @throws ExportException If there is a problem initializing the {@linkplain AudioGraphInput
      *     input}.
      */
+    @Nullable
     AudioGraphInput getAudioGraphInput(EditedMediaItem editedMediaItem, Format format)
         throws ExportException;
 
@@ -123,18 +127,13 @@ import java.util.Objects;
     // TODO(b/303029969): Evaluate throwing vs ignoring for null outputChannels.
     checkArgument(outputChannels == null);
     currentInputFormat = inputFormat;
-    if (outputGraphInput == null) {
-      try {
-        outputGraphInput = controller.getAudioGraphInput(editedMediaItem, currentInputFormat);
-      } catch (ExportException e) {
-        throw new ConfigurationException(e, currentInputFormat);
-      }
-    }
 
     // During playback, AudioGraphInput doesn't know the full media duration upfront due to seeking.
     // Pass in C.TIME_UNSET to AudioGraphInput.onMediaItemChanged.
-    outputGraphInput.onMediaItemChanged(
+    if (outputGraphInput != null) {
-        editedMediaItem, C.TIME_UNSET, currentInputFormat, /* isLast= */ false);
+      outputGraphInput.onMediaItemChanged(
+          editedMediaItem, /* durationUs= */ C.TIME_UNSET, currentInputFormat, /* isLast= */ false);
+    }
   }
 
   @Override
@@ -151,8 +150,33 @@ import java.util.Objects;
 
   @Override
   public boolean handleBuffer(
-      ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount) {
+      ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount)
+      throws InitializationException {
     checkState(!inputStreamEnded);
+    EditedMediaItem editedMediaItem = checkStateNotNull(currentEditedMediaItemInfo).editedMediaItem;
+    if (outputGraphInput == null) {
+
+      AudioGraphInput outputGraphInput;
+      try {
+        outputGraphInput =
+            controller.getAudioGraphInput(editedMediaItem, checkStateNotNull(currentInputFormat));
+      } catch (ExportException e) {
+        throw new InitializationException(
+            "Error creating AudioGraphInput",
+            AudioTrack.STATE_UNINITIALIZED,
+            currentInputFormat,
+            /* isRecoverable= */ false,
+            e);
+      }
+      if (outputGraphInput == null) {
+        return false;
+      }
+
+      this.outputGraphInput = outputGraphInput;
+      this.outputGraphInput.onMediaItemChanged(
+          editedMediaItem, /* durationUs= */ C.TIME_UNSET, currentInputFormat, /* isLast= */ false);
+    }
+
     return handleBufferInternal(buffer, presentationTimeUs, /* flags= */ 0);
   }
 

libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java

@@ -689,9 +689,10 @@ public final class CompositionPlayer extends SimpleBasePlayer
                   editedMediaItemSequence,
                   playbackAudioGraphWrapper,
                   playbackVideoGraphWrapper.getSink(),
-                  imageDecoderFactory)
+                  imageDecoderFactory,
+                  /* inputIndex= */ i)
               : SequenceRenderersFactory.createForAudio(
-                  context, editedMediaItemSequence, playbackAudioGraphWrapper);
+                  context, editedMediaItemSequence, playbackAudioGraphWrapper, /* inputIndex= */ i);
       ExoPlayer.Builder playerBuilder =
           new ExoPlayer.Builder(context)
               .setLooper(getApplicationLooper())

libraries/transformer/src/main/java/androidx/media3/transformer/PlaybackAudioGraphWrapper.java

@@ -18,6 +18,7 @@ package androidx.media3.transformer;
 
 import static androidx.media3.common.util.Util.sampleCountToDurationUs;
 
+import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.audio.AudioProcessor;
@@ -31,16 +32,19 @@ import java.util.Objects;
 /**
  * Processes input from {@link AudioGraphInputAudioSink} instances, plumbing the data through an
  * {@link AudioGraph} and writing the output to the provided {@link AudioSink}.
- *
- * <p>Multiple streams of {@linkplain #createInput() input} are not currently supported.
  */
 /* package */ final class PlaybackAudioGraphWrapper {
+
+  // The index number for the primary sequence.
+  private static final int PRIMARY_SEQUENCE_INDEX = 0;
+
   private final AudioSink finalAudioSink;
   private final AudioGraph audioGraph;
 
   private int audioGraphInputsCreated;
   private int inputAudioSinksCreated;
   private int inputAudioSinksPlaying;
+  private boolean hasRegisteredPrimaryFormat;
   private AudioFormat outputAudioFormat;
   private long outputFramesWritten;
   private long seekPositionUs;
@@ -74,8 +78,8 @@ import java.util.Objects;
   }
 
   /** Returns an {@link AudioSink} for a single sequence of non-overlapping raw PCM audio. */
-  public AudioGraphInputAudioSink createInput() {
+  public AudioGraphInputAudioSink createInput(int inputIndex) {
-    return new AudioGraphInputAudioSink(new SinkController());
+    return new AudioGraphInputAudioSink(new SinkController(inputIndex));
   }
 
   /**
@@ -157,17 +161,29 @@ import java.util.Objects;
   }
 
   private final class SinkController implements AudioGraphInputAudioSink.Controller {
+    private final boolean isSequencePrimary;
     private boolean playing;
 
-    public SinkController() {
+    public SinkController(int inputIndex) {
+      this.isSequencePrimary = inputIndex == PRIMARY_SEQUENCE_INDEX;
       inputAudioSinksCreated++;
     }
 
+    @Nullable
     @Override
     public AudioGraphInput getAudioGraphInput(EditedMediaItem editedMediaItem, Format format)
         throws ExportException {
+      if (!isSequencePrimary && !hasRegisteredPrimaryFormat) {
+        // Make sure the format corresponding to the primary sequence is registered first to the
+        // AudioGraph.
+        return null;
+      }
+
       AudioGraphInput audioGraphInput = audioGraph.registerInput(editedMediaItem, format);
       audioGraphInputsCreated++;
+      if (isSequencePrimary) {
+        hasRegisteredPrimaryFormat = true;
+      }
       return audioGraphInput;
     }
 

libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java

@@ -65,6 +65,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final PlaybackAudioGraphWrapper playbackAudioGraphWrapper;
   @Nullable private final VideoSink videoSink;
   @Nullable private final ImageDecoder.Factory imageDecoderFactory;
+  private final int inputIndex;
 
   /** Creates a renderers factory for a player that will play video, image and audio. */
   public static SequenceRenderersFactory create(
@@ -72,22 +73,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       EditedMediaItemSequence sequence,
       PlaybackAudioGraphWrapper playbackAudioGraphWrapper,
       VideoSink videoSink,
-      ImageDecoder.Factory imageDecoderFactory) {
+      ImageDecoder.Factory imageDecoderFactory,
+      int inputIndex) {
     return new SequenceRenderersFactory(
-        context, sequence, playbackAudioGraphWrapper, videoSink, imageDecoderFactory);
+        context, sequence, playbackAudioGraphWrapper, videoSink, imageDecoderFactory, inputIndex);
   }
 
   /** Creates a renderers factory that for a player that will only play audio. */
   public static SequenceRenderersFactory createForAudio(
       Context context,
       EditedMediaItemSequence sequence,
-      PlaybackAudioGraphWrapper playbackAudioGraphWrapper) {
+      PlaybackAudioGraphWrapper playbackAudioGraphWrapper,
+      int inputIndex) {
     return new SequenceRenderersFactory(
         context,
         sequence,
         playbackAudioGraphWrapper,
         /* videoSink= */ null,
-        /* imageDecoderFactory= */ null);
+        /* imageDecoderFactory= */ null,
+        inputIndex);
   }
 
   private SequenceRenderersFactory(
@@ -95,12 +99,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       EditedMediaItemSequence sequence,
       PlaybackAudioGraphWrapper playbackAudioGraphWrapper,
       @Nullable VideoSink videoSink,
-      @Nullable ImageDecoder.Factory imageDecoderFactory) {
+      @Nullable ImageDecoder.Factory imageDecoderFactory,
+      int inputIndex) {
     this.context = context;
     this.sequence = sequence;
     this.playbackAudioGraphWrapper = playbackAudioGraphWrapper;
     this.videoSink = videoSink;
     this.imageDecoderFactory = imageDecoderFactory;
+    this.inputIndex = inputIndex;
   }
 
   @Override
@@ -117,7 +123,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             eventHandler,
             audioRendererEventListener,
             sequence,
-            /* audioSink= */ playbackAudioGraphWrapper.createInput(),
+            /* audioSink= */ playbackAudioGraphWrapper.createInput(inputIndex),
             playbackAudioGraphWrapper));
 
     if (videoSink != null) {

libraries/transformer/src/test/java/androidx/media3/transformer/PlaybackAudioGraphWrapperTest.java

@@ -63,14 +63,15 @@ public class PlaybackAudioGraphWrapperTest {
 
   @Test
   public void processData_audioSinkHasNotConfiguredYet_returnsFalse() throws Exception {
-    AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
 
     assertThat(playbackAudioGraphWrapper.processData()).isFalse();
   }
 
   @Test
   public void inputPlay_withOneInput_playsOutputSink() throws Exception {
-    AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
 
     inputAudioSink.play();
 
@@ -79,7 +80,8 @@ public class PlaybackAudioGraphWrapperTest {
 
   @Test
   public void inputPause_withOneInput_pausesOutputSink() throws Exception {
-    AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
 
     inputAudioSink.play();
     inputAudioSink.pause();
@@ -89,7 +91,8 @@ public class PlaybackAudioGraphWrapperTest {
 
   @Test
   public void inputReset_withOneInput_pausesOutputSink() {
-    AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
 
     inputAudioSink.play();
     inputAudioSink.reset();
@@ -99,7 +102,8 @@ public class PlaybackAudioGraphWrapperTest {
 
   @Test
   public void inputPlay_whenPlaying_doesNotPlayOutputSink() throws Exception {
-    AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
     inputAudioSink.play();
     inputAudioSink.play();
 
@@ -108,7 +112,8 @@ public class PlaybackAudioGraphWrapperTest {
 
   @Test
   public void inputPause_whenNotPlaying_doesNotPauseOutputSink() throws Exception {
-    AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
 
     inputAudioSink.pause();
 
@@ -117,9 +122,11 @@ public class PlaybackAudioGraphWrapperTest {
 
   @Test
   public void someInputPlay_withMultipleInputs_doesNotPlayOutputSink() throws Exception {
-    AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink1 =
-    AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
-    AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink2 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
+    AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
 
     inputAudioSink1.play();
     inputAudioSink2.play();
@@ -128,9 +135,12 @@ public class PlaybackAudioGraphWrapperTest {
 
   @Test
   public void allInputPlay_withMultipleInputs_playsOutputSinkOnce() throws Exception {
-    AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink1 =
-    AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
-    AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink2 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
+    AudioGraphInputAudioSink inputAudioSink3 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
 
     inputAudioSink1.play();
     inputAudioSink2.play();
@@ -142,9 +152,12 @@ public class PlaybackAudioGraphWrapperTest {
   @Test
   public void firstInputPause_withMultipleInputs_pausesOutputSink() throws Exception {
     InOrder inOrder = inOrder(outputAudioSink);
-    AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink1 =
-    AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
-    AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink2 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
+    AudioGraphInputAudioSink inputAudioSink3 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
 
     inputAudioSink1.play();
     inputAudioSink2.play();
@@ -157,9 +170,12 @@ public class PlaybackAudioGraphWrapperTest {
 
   @Test
   public void allInputPause_withMultipleInputs_pausesOutputSinkOnce() throws Exception {
-    AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink1 =
-    AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
-    AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink2 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
+    AudioGraphInputAudioSink inputAudioSink3 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
 
     inputAudioSink1.play();
     inputAudioSink2.play();
@@ -174,9 +190,12 @@ public class PlaybackAudioGraphWrapperTest {
   @Test
   public void inputPlayAfterPause_withMultipleInputs_playsOutputSink() throws Exception {
     InOrder inOrder = inOrder(outputAudioSink);
-    AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink1 =
-    AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
-    AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput();
+    AudioGraphInputAudioSink inputAudioSink2 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
+    AudioGraphInputAudioSink inputAudioSink3 =
+        playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
 
     inputAudioSink1.play();
     inputAudioSink2.play();