If AudioGraphInput is not available, callers should re-try again later. + * *
Data {@linkplain #handleBuffer written} to the sink will be {@linkplain * AudioGraphInput#queueInputBuffer() queued} to the {@link AudioGraphInput}. * @@ -59,10 +62,11 @@ import java.util.Objects; * AudioGraphInput}. * @param format The {@link Format} used to {@linkplain AudioGraphInputAudioSink#configure * configure} the {@linkplain AudioGraphInputAudioSink sink}. - * @return The {@link AudioGraphInput}. + * @return The {@link AudioGraphInput}, or {@code null} if the input is not available yet. * @throws ExportException If there is a problem initializing the {@linkplain AudioGraphInput * input}. */ + @Nullable AudioGraphInput getAudioGraphInput(EditedMediaItem editedMediaItem, Format format) throws ExportException; @@ -123,18 +127,13 @@ import java.util.Objects; // TODO(b/303029969): Evaluate throwing vs ignoring for null outputChannels. checkArgument(outputChannels == null); currentInputFormat = inputFormat; - if (outputGraphInput == null) { - try { - outputGraphInput = controller.getAudioGraphInput(editedMediaItem, currentInputFormat); - } catch (ExportException e) { - throw new ConfigurationException(e, currentInputFormat); - } - } // During playback, AudioGraphInput doesn't know the full media duration upfront due to seeking. // Pass in C.TIME_UNSET to AudioGraphInput.onMediaItemChanged. - outputGraphInput.onMediaItemChanged( - editedMediaItem, C.TIME_UNSET, currentInputFormat, /* isLast= */ false); + if (outputGraphInput != null) { + outputGraphInput.onMediaItemChanged( + editedMediaItem, /* durationUs= */ C.TIME_UNSET, currentInputFormat, /* isLast= */ false); + } } @Override @@ -151,8 +150,33 @@ import java.util.Objects; @Override public boolean handleBuffer( - ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount) { + ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount) + throws InitializationException { checkState(!inputStreamEnded); + EditedMediaItem editedMediaItem = checkStateNotNull(currentEditedMediaItemInfo).editedMediaItem; + if (outputGraphInput == null) { + + AudioGraphInput outputGraphInput; + try { + outputGraphInput = + controller.getAudioGraphInput(editedMediaItem, checkStateNotNull(currentInputFormat)); + } catch (ExportException e) { + throw new InitializationException( + "Error creating AudioGraphInput", + AudioTrack.STATE_UNINITIALIZED, + currentInputFormat, + /* isRecoverable= */ false, + e); + } + if (outputGraphInput == null) { + return false; + } + + this.outputGraphInput = outputGraphInput; + this.outputGraphInput.onMediaItemChanged( + editedMediaItem, /* durationUs= */ C.TIME_UNSET, currentInputFormat, /* isLast= */ false); + } + return handleBufferInternal(buffer, presentationTimeUs, /* flags= */ 0); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java index c9cf864626..a8b1e26337 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java @@ -689,9 +689,10 @@ public final class CompositionPlayer extends SimpleBasePlayer editedMediaItemSequence, playbackAudioGraphWrapper, playbackVideoGraphWrapper.getSink(), - imageDecoderFactory) + imageDecoderFactory, + /* inputIndex= */ i) : SequenceRenderersFactory.createForAudio( - context, editedMediaItemSequence, playbackAudioGraphWrapper); + context, editedMediaItemSequence, playbackAudioGraphWrapper, /* inputIndex= */ i); ExoPlayer.Builder playerBuilder = new ExoPlayer.Builder(context) .setLooper(getApplicationLooper()) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PlaybackAudioGraphWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PlaybackAudioGraphWrapper.java index 9a03ad6f36..60bd9a6f62 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PlaybackAudioGraphWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PlaybackAudioGraphWrapper.java @@ -18,6 +18,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Util.sampleCountToDurationUs; +import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.audio.AudioProcessor; @@ -31,16 +32,19 @@ import java.util.Objects; /** * Processes input from {@link AudioGraphInputAudioSink} instances, plumbing the data through an * {@link AudioGraph} and writing the output to the provided {@link AudioSink}. - * - *
Multiple streams of {@linkplain #createInput() input} are not currently supported. */ /* package */ final class PlaybackAudioGraphWrapper { + + // The index number for the primary sequence. + private static final int PRIMARY_SEQUENCE_INDEX = 0; + private final AudioSink finalAudioSink; private final AudioGraph audioGraph; private int audioGraphInputsCreated; private int inputAudioSinksCreated; private int inputAudioSinksPlaying; + private boolean hasRegisteredPrimaryFormat; private AudioFormat outputAudioFormat; private long outputFramesWritten; private long seekPositionUs; @@ -74,8 +78,8 @@ import java.util.Objects; } /** Returns an {@link AudioSink} for a single sequence of non-overlapping raw PCM audio. */ - public AudioGraphInputAudioSink createInput() { - return new AudioGraphInputAudioSink(new SinkController()); + public AudioGraphInputAudioSink createInput(int inputIndex) { + return new AudioGraphInputAudioSink(new SinkController(inputIndex)); } /** @@ -157,17 +161,29 @@ import java.util.Objects; } private final class SinkController implements AudioGraphInputAudioSink.Controller { + private final boolean isSequencePrimary; private boolean playing; - public SinkController() { + public SinkController(int inputIndex) { + this.isSequencePrimary = inputIndex == PRIMARY_SEQUENCE_INDEX; inputAudioSinksCreated++; } + @Nullable @Override public AudioGraphInput getAudioGraphInput(EditedMediaItem editedMediaItem, Format format) throws ExportException { + if (!isSequencePrimary && !hasRegisteredPrimaryFormat) { + // Make sure the format corresponding to the primary sequence is registered first to the + // AudioGraph. + return null; + } + AudioGraphInput audioGraphInput = audioGraph.registerInput(editedMediaItem, format); audioGraphInputsCreated++; + if (isSequencePrimary) { + hasRegisteredPrimaryFormat = true; + } return audioGraphInput; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java index 341b870d21..c635fd6da1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java @@ -65,6 +65,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final PlaybackAudioGraphWrapper playbackAudioGraphWrapper; @Nullable private final VideoSink videoSink; @Nullable private final ImageDecoder.Factory imageDecoderFactory; + private final int inputIndex; /** Creates a renderers factory for a player that will play video, image and audio. */ public static SequenceRenderersFactory create( @@ -72,22 +73,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EditedMediaItemSequence sequence, PlaybackAudioGraphWrapper playbackAudioGraphWrapper, VideoSink videoSink, - ImageDecoder.Factory imageDecoderFactory) { + ImageDecoder.Factory imageDecoderFactory, + int inputIndex) { return new SequenceRenderersFactory( - context, sequence, playbackAudioGraphWrapper, videoSink, imageDecoderFactory); + context, sequence, playbackAudioGraphWrapper, videoSink, imageDecoderFactory, inputIndex); } /** Creates a renderers factory that for a player that will only play audio. */ public static SequenceRenderersFactory createForAudio( Context context, EditedMediaItemSequence sequence, - PlaybackAudioGraphWrapper playbackAudioGraphWrapper) { + PlaybackAudioGraphWrapper playbackAudioGraphWrapper, + int inputIndex) { return new SequenceRenderersFactory( context, sequence, playbackAudioGraphWrapper, /* videoSink= */ null, - /* imageDecoderFactory= */ null); + /* imageDecoderFactory= */ null, + inputIndex); } private SequenceRenderersFactory( @@ -95,12 +99,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EditedMediaItemSequence sequence, PlaybackAudioGraphWrapper playbackAudioGraphWrapper, @Nullable VideoSink videoSink, - @Nullable ImageDecoder.Factory imageDecoderFactory) { + @Nullable ImageDecoder.Factory imageDecoderFactory, + int inputIndex) { this.context = context; this.sequence = sequence; this.playbackAudioGraphWrapper = playbackAudioGraphWrapper; this.videoSink = videoSink; this.imageDecoderFactory = imageDecoderFactory; + this.inputIndex = inputIndex; } @Override @@ -117,7 +123,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; eventHandler, audioRendererEventListener, sequence, - /* audioSink= */ playbackAudioGraphWrapper.createInput(), + /* audioSink= */ playbackAudioGraphWrapper.createInput(inputIndex), playbackAudioGraphWrapper)); if (videoSink != null) { diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/PlaybackAudioGraphWrapperTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/PlaybackAudioGraphWrapperTest.java index b1fef373aa..0f13d0c1cf 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/PlaybackAudioGraphWrapperTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/PlaybackAudioGraphWrapperTest.java @@ -63,14 +63,15 @@ public class PlaybackAudioGraphWrapperTest { @Test public void processData_audioSinkHasNotConfiguredYet_returnsFalse() throws Exception { - AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); assertThat(playbackAudioGraphWrapper.processData()).isFalse(); } @Test public void inputPlay_withOneInput_playsOutputSink() throws Exception { - AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); inputAudioSink.play(); @@ -79,7 +80,8 @@ public class PlaybackAudioGraphWrapperTest { @Test public void inputPause_withOneInput_pausesOutputSink() throws Exception { - AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); inputAudioSink.play(); inputAudioSink.pause(); @@ -89,7 +91,8 @@ public class PlaybackAudioGraphWrapperTest { @Test public void inputReset_withOneInput_pausesOutputSink() { - AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); inputAudioSink.play(); inputAudioSink.reset(); @@ -99,7 +102,8 @@ public class PlaybackAudioGraphWrapperTest { @Test public void inputPlay_whenPlaying_doesNotPlayOutputSink() throws Exception { - AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); inputAudioSink.play(); inputAudioSink.play(); @@ -108,7 +112,8 @@ public class PlaybackAudioGraphWrapperTest { @Test public void inputPause_whenNotPlaying_doesNotPauseOutputSink() throws Exception { - AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); inputAudioSink.pause(); @@ -117,9 +122,11 @@ public class PlaybackAudioGraphWrapperTest { @Test public void someInputPlay_withMultipleInputs_doesNotPlayOutputSink() throws Exception { - AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink1 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); + AudioGraphInputAudioSink inputAudioSink2 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1); + AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2); inputAudioSink1.play(); inputAudioSink2.play(); @@ -128,9 +135,12 @@ public class PlaybackAudioGraphWrapperTest { @Test public void allInputPlay_withMultipleInputs_playsOutputSinkOnce() throws Exception { - AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink1 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); + AudioGraphInputAudioSink inputAudioSink2 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1); + AudioGraphInputAudioSink inputAudioSink3 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2); inputAudioSink1.play(); inputAudioSink2.play(); @@ -142,9 +152,12 @@ public class PlaybackAudioGraphWrapperTest { @Test public void firstInputPause_withMultipleInputs_pausesOutputSink() throws Exception { InOrder inOrder = inOrder(outputAudioSink); - AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink1 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); + AudioGraphInputAudioSink inputAudioSink2 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1); + AudioGraphInputAudioSink inputAudioSink3 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2); inputAudioSink1.play(); inputAudioSink2.play(); @@ -157,9 +170,12 @@ public class PlaybackAudioGraphWrapperTest { @Test public void allInputPause_withMultipleInputs_pausesOutputSinkOnce() throws Exception { - AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink1 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); + AudioGraphInputAudioSink inputAudioSink2 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1); + AudioGraphInputAudioSink inputAudioSink3 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2); inputAudioSink1.play(); inputAudioSink2.play(); @@ -174,9 +190,12 @@ public class PlaybackAudioGraphWrapperTest { @Test public void inputPlayAfterPause_withMultipleInputs_playsOutputSink() throws Exception { InOrder inOrder = inOrder(outputAudioSink); - AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput(); - AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput(); + AudioGraphInputAudioSink inputAudioSink1 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0); + AudioGraphInputAudioSink inputAudioSink2 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1); + AudioGraphInputAudioSink inputAudioSink3 = + playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2); inputAudioSink1.play(); inputAudioSink2.play();