loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Block secondary audio processing until primary format is configured

Browse Source 
PiperOrigin-RevId: 673961235
This commit is contained in:
claincly 2024-09-12 12:39:44 -07:00 committed by Copybara-Service
parent e88d6fe459
commit bc8d82355f
6 changed files with 136 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java
View File

@ -236,6 +236,27 @@ public interface AudioSink {
/** The input {@link Format} of the sink when the error occurs. */
public final Format format;
/**
* Creates a new instance.
*
* @param message The detail message for this exception.
* @param audioTrackState The underlying {@link AudioTrack}'s state.
* @param format The input format of the sink when the error occurs.
* @param isRecoverable Whether the exception can be recovered by recreating the sink.
* @param cause The {@link Throwable cause} of this exception.
*/
public InitializationException(
String message,
int audioTrackState,
Format format,
boolean isRecoverable,
@Nullable Throwable cause) {
super(message, cause);
this.audioTrackState = audioTrackState;
this.isRecoverable = isRecoverable;
this.format = format;
}
/**
* Creates a new instance.
*
@ -255,7 +276,7 @@ public interface AudioSink {
Format format,
boolean isRecoverable,
@Nullable Exception audioTrackException) {
super(
this(
"AudioTrack init failed "
+ audioTrackState
+ " "
@ -263,10 +284,10 @@ public interface AudioSink {
+ " "
+ format
+ (isRecoverable ? " (recoverable)" : ""),
audioTrackState,
format,
isRecoverable,
audioTrackException);
this.audioTrackState = audioTrackState;
this.isRecoverable = isRecoverable;
this.format = format;
}
}

46
libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java
View File

@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.media.AudioTrack;
import androidx.annotation.Nullable;
import androidx.media3.common.AudioAttributes;
import androidx.media3.common.AuxEffectInfo;
@ -52,6 +53,8 @@ import java.util.Objects;
* Returns the {@link AudioGraphInput} instance associated with this {@linkplain
* AudioGraphInputAudioSink sink}.
*
* <p>If AudioGraphInput is not available, callers should re-try again later.
*
* <p>Data {@linkplain #handleBuffer written} to the sink will be {@linkplain
* AudioGraphInput#queueInputBuffer() queued} to the {@link AudioGraphInput}.
*
@ -59,10 +62,11 @@ import java.util.Objects;
* AudioGraphInput}.
* @param format The {@link Format} used to {@linkplain AudioGraphInputAudioSink#configure
* configure} the {@linkplain AudioGraphInputAudioSink sink}.
* @return The {@link AudioGraphInput}.
* @return The {@link AudioGraphInput}, or {@code null} if the input is not available yet.
* @throws ExportException If there is a problem initializing the {@linkplain AudioGraphInput
* input}.
*/
@Nullable
AudioGraphInput getAudioGraphInput(EditedMediaItem editedMediaItem, Format format)
throws ExportException;
@ -123,18 +127,13 @@ import java.util.Objects;
// TODO(b/303029969): Evaluate throwing vs ignoring for null outputChannels.
checkArgument(outputChannels == null);
currentInputFormat = inputFormat;
if (outputGraphInput == null) {
try {
outputGraphInput = controller.getAudioGraphInput(editedMediaItem, currentInputFormat);
} catch (ExportException e) {
throw new ConfigurationException(e, currentInputFormat);
}
}
// During playback, AudioGraphInput doesn't know the full media duration upfront due to seeking.
// Pass in C.TIME_UNSET to AudioGraphInput.onMediaItemChanged.
outputGraphInput.onMediaItemChanged(
editedMediaItem, C.TIME_UNSET, currentInputFormat, /* isLast= */ false);
if (outputGraphInput != null) {
outputGraphInput.onMediaItemChanged(
editedMediaItem, /* durationUs= */ C.TIME_UNSET, currentInputFormat, /* isLast= */ false);
}
}
@Override
@ -151,8 +150,33 @@ import java.util.Objects;
@Override
public boolean handleBuffer(
ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount) {
ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount)
throws InitializationException {
checkState(!inputStreamEnded);
EditedMediaItem editedMediaItem = checkStateNotNull(currentEditedMediaItemInfo).editedMediaItem;
if (outputGraphInput == null) {
AudioGraphInput outputGraphInput;
try {
outputGraphInput =
controller.getAudioGraphInput(editedMediaItem, checkStateNotNull(currentInputFormat));
} catch (ExportException e) {
throw new InitializationException(
"Error creating AudioGraphInput",
AudioTrack.STATE_UNINITIALIZED,
currentInputFormat,
/* isRecoverable= */ false,
e);
}
if (outputGraphInput == null) {
return false;
}
this.outputGraphInput = outputGraphInput;
this.outputGraphInput.onMediaItemChanged(
editedMediaItem, /* durationUs= */ C.TIME_UNSET, currentInputFormat, /* isLast= */ false);
}
return handleBufferInternal(buffer, presentationTimeUs, /* flags= */ 0);
}

5
libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java
View File

@ -689,9 +689,10 @@ public final class CompositionPlayer extends SimpleBasePlayer
editedMediaItemSequence,
playbackAudioGraphWrapper,
playbackVideoGraphWrapper.getSink(),
imageDecoderFactory)
imageDecoderFactory,
/* inputIndex= */ i)
: SequenceRenderersFactory.createForAudio(
context, editedMediaItemSequence, playbackAudioGraphWrapper);
context, editedMediaItemSequence, playbackAudioGraphWrapper, /* inputIndex= */ i);
ExoPlayer.Builder playerBuilder =
new ExoPlayer.Builder(context)
.setLooper(getApplicationLooper())

26
libraries/transformer/src/main/java/androidx/media3/transformer/PlaybackAudioGraphWrapper.java
View File

@ -18,6 +18,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.util.Util.sampleCountToDurationUs;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.audio.AudioProcessor;
@ -31,16 +32,19 @@ import java.util.Objects;
/**
* Processes input from {@link AudioGraphInputAudioSink} instances, plumbing the data through an
* {@link AudioGraph} and writing the output to the provided {@link AudioSink}.
*
* <p>Multiple streams of {@linkplain #createInput() input} are not currently supported.
*/
/* package */ final class PlaybackAudioGraphWrapper {
// The index number for the primary sequence.
private static final int PRIMARY_SEQUENCE_INDEX = 0;
private final AudioSink finalAudioSink;
private final AudioGraph audioGraph;
private int audioGraphInputsCreated;
private int inputAudioSinksCreated;
private int inputAudioSinksPlaying;
private boolean hasRegisteredPrimaryFormat;
private AudioFormat outputAudioFormat;
private long outputFramesWritten;
private long seekPositionUs;
@ -74,8 +78,8 @@ import java.util.Objects;
}
/** Returns an {@link AudioSink} for a single sequence of non-overlapping raw PCM audio. */
public AudioGraphInputAudioSink createInput() {
return new AudioGraphInputAudioSink(new SinkController());
public AudioGraphInputAudioSink createInput(int inputIndex) {
return new AudioGraphInputAudioSink(new SinkController(inputIndex));
}
/**
@ -157,17 +161,29 @@ import java.util.Objects;
}
private final class SinkController implements AudioGraphInputAudioSink.Controller {
private final boolean isSequencePrimary;
private boolean playing;
public SinkController() {
public SinkController(int inputIndex) {
this.isSequencePrimary = inputIndex == PRIMARY_SEQUENCE_INDEX;
inputAudioSinksCreated++;
}
@Nullable
@Override
public AudioGraphInput getAudioGraphInput(EditedMediaItem editedMediaItem, Format format)
throws ExportException {
if (!isSequencePrimary && !hasRegisteredPrimaryFormat) {
// Make sure the format corresponding to the primary sequence is registered first to the
// AudioGraph.
return null;
}
AudioGraphInput audioGraphInput = audioGraph.registerInput(editedMediaItem, format);
audioGraphInputsCreated++;
if (isSequencePrimary) {
hasRegisteredPrimaryFormat = true;
}
return audioGraphInput;
}

18
libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java
View File

@ -65,6 +65,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final PlaybackAudioGraphWrapper playbackAudioGraphWrapper;
@Nullable private final VideoSink videoSink;
@Nullable private final ImageDecoder.Factory imageDecoderFactory;
private final int inputIndex;
/** Creates a renderers factory for a player that will play video, image and audio. */
public static SequenceRenderersFactory create(
@ -72,22 +73,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EditedMediaItemSequence sequence,
PlaybackAudioGraphWrapper playbackAudioGraphWrapper,
VideoSink videoSink,
ImageDecoder.Factory imageDecoderFactory) {
ImageDecoder.Factory imageDecoderFactory,
int inputIndex) {
return new SequenceRenderersFactory(
context, sequence, playbackAudioGraphWrapper, videoSink, imageDecoderFactory);
context, sequence, playbackAudioGraphWrapper, videoSink, imageDecoderFactory, inputIndex);
}
/** Creates a renderers factory that for a player that will only play audio. */
public static SequenceRenderersFactory createForAudio(
Context context,
EditedMediaItemSequence sequence,
PlaybackAudioGraphWrapper playbackAudioGraphWrapper) {
PlaybackAudioGraphWrapper playbackAudioGraphWrapper,
int inputIndex) {
return new SequenceRenderersFactory(
context,
sequence,
playbackAudioGraphWrapper,
/* videoSink= */ null,
/* imageDecoderFactory= */ null);
/* imageDecoderFactory= */ null,
inputIndex);
}
private SequenceRenderersFactory(
@ -95,12 +99,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EditedMediaItemSequence sequence,
PlaybackAudioGraphWrapper playbackAudioGraphWrapper,
@Nullable VideoSink videoSink,
@Nullable ImageDecoder.Factory imageDecoderFactory) {
@Nullable ImageDecoder.Factory imageDecoderFactory,
int inputIndex) {
this.context = context;
this.sequence = sequence;
this.playbackAudioGraphWrapper = playbackAudioGraphWrapper;
this.videoSink = videoSink;
this.imageDecoderFactory = imageDecoderFactory;
this.inputIndex = inputIndex;
}
@Override
@ -117,7 +123,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
eventHandler,
audioRendererEventListener,
sequence,
/* audioSink= */ playbackAudioGraphWrapper.createInput(),
/* audioSink= */ playbackAudioGraphWrapper.createInput(inputIndex),
playbackAudioGraphWrapper));
if (videoSink != null) {

61
libraries/transformer/src/test/java/androidx/media3/transformer/PlaybackAudioGraphWrapperTest.java
View File

@ -63,14 +63,15 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void processData_audioSinkHasNotConfiguredYet_returnsFalse() throws Exception {
AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
assertThat(playbackAudioGraphWrapper.processData()).isFalse();
}
@Test
public void inputPlay_withOneInput_playsOutputSink() throws Exception {
AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
inputAudioSink.play();
@ -79,7 +80,8 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void inputPause_withOneInput_pausesOutputSink() throws Exception {
AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
inputAudioSink.play();
inputAudioSink.pause();
@ -89,7 +91,8 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void inputReset_withOneInput_pausesOutputSink() {
AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
inputAudioSink.play();
inputAudioSink.reset();
@ -99,7 +102,8 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void inputPlay_whenPlaying_doesNotPlayOutputSink() throws Exception {
AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
inputAudioSink.play();
inputAudioSink.play();
@ -108,7 +112,8 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void inputPause_whenNotPlaying_doesNotPauseOutputSink() throws Exception {
AudioGraphInputAudioSink inputAudioSink = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
inputAudioSink.pause();
@ -117,9 +122,11 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void someInputPlay_withMultipleInputs_doesNotPlayOutputSink() throws Exception {
AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink1 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
AudioGraphInputAudioSink inputAudioSink2 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
AudioGraphInputAudioSink unused = playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
inputAudioSink1.play();
inputAudioSink2.play();
@ -128,9 +135,12 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void allInputPlay_withMultipleInputs_playsOutputSinkOnce() throws Exception {
AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink1 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
AudioGraphInputAudioSink inputAudioSink2 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
AudioGraphInputAudioSink inputAudioSink3 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
inputAudioSink1.play();
inputAudioSink2.play();
@ -142,9 +152,12 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void firstInputPause_withMultipleInputs_pausesOutputSink() throws Exception {
InOrder inOrder = inOrder(outputAudioSink);
AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink1 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
AudioGraphInputAudioSink inputAudioSink2 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
AudioGraphInputAudioSink inputAudioSink3 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
inputAudioSink1.play();
inputAudioSink2.play();
@ -157,9 +170,12 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void allInputPause_withMultipleInputs_pausesOutputSinkOnce() throws Exception {
AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink1 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
AudioGraphInputAudioSink inputAudioSink2 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
AudioGraphInputAudioSink inputAudioSink3 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
inputAudioSink1.play();
inputAudioSink2.play();
@ -174,9 +190,12 @@ public class PlaybackAudioGraphWrapperTest {
@Test
public void inputPlayAfterPause_withMultipleInputs_playsOutputSink() throws Exception {
InOrder inOrder = inOrder(outputAudioSink);
AudioGraphInputAudioSink inputAudioSink1 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink2 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink3 = playbackAudioGraphWrapper.createInput();
AudioGraphInputAudioSink inputAudioSink1 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 0);
AudioGraphInputAudioSink inputAudioSink2 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 1);
AudioGraphInputAudioSink inputAudioSink3 =
playbackAudioGraphWrapper.createInput(/* inputIndex= */ 2);
inputAudioSink1.play();
inputAudioSink2.play();