From 2b367759cf15f9d73058d918e1fa2cf8b86c9ea6 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Thu, 19 Apr 2018 08:25:49 -0700 Subject: [PATCH] Allow overriding skipping/scaling with custom AudioProcessors Issue: #3142 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=193519029 --- RELEASENOTES.md | 2 + .../exoplayer2/audio/DefaultAudioSink.java | 196 +++++++++++++----- .../audio/SilenceSkippingAudioProcessor.java | 2 +- 3 files changed, 151 insertions(+), 49 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index ff723c943c..0fd6f3c8ee 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -53,6 +53,8 @@ with encoder padding when the decoder returns a non-empty final buffer. * Allow trimming more than one sample when applying an elst audio edit via gapless playback info. + * Allow overriding skipping/scaling with custom `AudioProcessor`s + ((#3142)[https://github.com/google/ExoPlayer/issues/3142]). * Caching: * Add release method to Cache interface. * Prevent multiple instances of SimpleCache in the same folder. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java index a206d2c1ec..1025cb953b 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java @@ -35,6 +35,8 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; /** * Plays audio data. The implementation delegates to an {@link AudioTrack} and handles playback @@ -64,6 +66,92 @@ public final class DefaultAudioSink implements AudioSink { } + /** + * Provides a chain of audio processors, which are used for any user-defined processing and + * applying playback parameters (if supported). Because applying playback parameters can skip and + * stretch/compress audio, the sink will query the chain for information on how to transform its + * output position to map it onto a media position, via {@link #getMediaDuration(long)} and {@link + * #getSkippedOutputFrameCount()}. + */ + public interface AudioProcessorChain { + + /** + * Returns the fixed chain of audio processors that will process audio. This method is called + * once during initialization, but audio processors may change state to become active/inactive + * during playback. + */ + AudioProcessor[] getAudioProcessors(); + + /** + * Configures audio processors to apply the specified playback parameters immediately, returning + * the new parameters, which may differ from those passed in. Only called when processors have + * no input pending. + * + * @param playbackParameters The playback parameters to try to apply. + * @return The playback parameters that were actually applied. + */ + PlaybackParameters applyPlaybackParameters(PlaybackParameters playbackParameters); + + /** + * Scales the specified playout duration to take into account speedup due to audio processing, + * returning an input media duration, in arbitrary units. + */ + long getMediaDuration(long playoutDuration); + + /** + * Returns the number of output audio frames skipped since the audio processors were last + * flushed. + */ + long getSkippedOutputFrameCount(); + } + + /** + * The default audio processor chain, which applies a (possibly empty) chain of user-defined audio + * processors followed by {@link SilenceSkippingAudioProcessor} and {@link SonicAudioProcessor}. + */ + public static class DefaultAudioProcessorChain implements AudioProcessorChain { + + private final AudioProcessor[] audioProcessors; + private final SilenceSkippingAudioProcessor silenceSkippingAudioProcessor; + private final SonicAudioProcessor sonicAudioProcessor; + + /** + * Creates a new default chain of audio processors, with the user-defined {@code + * audioProcessors} applied before silence skipping and playback parameters. + */ + public DefaultAudioProcessorChain(AudioProcessor... audioProcessors) { + this.audioProcessors = Arrays.copyOf(audioProcessors, audioProcessors.length + 2); + silenceSkippingAudioProcessor = new SilenceSkippingAudioProcessor(); + sonicAudioProcessor = new SonicAudioProcessor(); + this.audioProcessors[audioProcessors.length] = silenceSkippingAudioProcessor; + this.audioProcessors[audioProcessors.length + 1] = sonicAudioProcessor; + } + + @Override + public AudioProcessor[] getAudioProcessors() { + return audioProcessors; + } + + @Override + public PlaybackParameters applyPlaybackParameters(PlaybackParameters playbackParameters) { + silenceSkippingAudioProcessor.setEnabled(playbackParameters.skipSilence); + return new PlaybackParameters( + sonicAudioProcessor.setSpeed(playbackParameters.speed), + sonicAudioProcessor.setPitch(playbackParameters.pitch), + playbackParameters.skipSilence); + } + + @Override + public long getMediaDuration(long playoutDuration) { + return sonicAudioProcessor.scaleDurationForSpeedup(playoutDuration); + } + + @Override + public long getSkippedOutputFrameCount() { + return silenceSkippingAudioProcessor.getSkippedFrames(); + } + } + /** * A minimum length for the {@link AudioTrack} buffer, in microseconds. */ @@ -135,11 +223,10 @@ public final class DefaultAudioSink implements AudioSink { public static boolean failOnSpuriousAudioTimestamp = false; @Nullable private final AudioCapabilities audioCapabilities; + private final AudioProcessorChain audioProcessorChain; private final boolean enableConvertHighResIntPcmToFloat; private final ChannelMappingAudioProcessor channelMappingAudioProcessor; private final TrimmingAudioProcessor trimmingAudioProcessor; - private final SilenceSkippingAudioProcessor silenceSkippingAudioProcessor; - private final SonicAudioProcessor sonicAudioProcessor; private final AudioProcessor[] toIntPcmAvailableAudioProcessors; private final AudioProcessor[] toFloatPcmAvailableAudioProcessors; private final ConditionVariable releasingConditionVariable; @@ -181,7 +268,7 @@ public final class DefaultAudioSink implements AudioSink { private long startMediaTimeUs; private float volume; - private AudioProcessor[] audioProcessors; + private AudioProcessor[] activeAudioProcessors; private ByteBuffer[] outputBuffers; @Nullable private ByteBuffer inputBuffer; @Nullable private ByteBuffer outputBuffer; @@ -196,17 +283,21 @@ public final class DefaultAudioSink implements AudioSink { private long lastFeedElapsedRealtimeMs; /** + * Creates a new default audio sink. + * * @param audioCapabilities The audio capabilities for playback on this device. May be null if the * default capabilities (no encoded audio passthrough support) should be assumed. * @param audioProcessors An array of {@link AudioProcessor}s that will process PCM audio before * output. May be empty. */ - public DefaultAudioSink(@Nullable AudioCapabilities audioCapabilities, - AudioProcessor[] audioProcessors) { + public DefaultAudioSink( + @Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors) { this(audioCapabilities, audioProcessors, /* enableConvertHighResIntPcmToFloat= */ false); } /** + * Creates a new default audio sink, optionally using float output for high resolution PCM. + * * @param audioCapabilities The audio capabilities for playback on this device. May be null if the * default capabilities (no encoded audio passthrough support) should be assumed. * @param audioProcessors An array of {@link AudioProcessor}s that will process PCM audio before @@ -220,22 +311,45 @@ public final class DefaultAudioSink implements AudioSink { @Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors, boolean enableConvertHighResIntPcmToFloat) { + this( + audioCapabilities, + new DefaultAudioProcessorChain(audioProcessors), + enableConvertHighResIntPcmToFloat); + } + + /** + * Creates a new default audio sink, optionally using float output for high resolution PCM and + * with the specified {@code audioProcessorChain}. + * + * @param audioCapabilities The audio capabilities for playback on this device. May be null if the + * default capabilities (no encoded audio passthrough support) should be assumed. + * @param audioProcessorChain An {@link AudioProcessorChain} which is used to apply playback + * parameters adjustments. The instance passed in must not be reused in other sinks. + * @param enableConvertHighResIntPcmToFloat Whether to enable conversion of high resolution + * integer PCM to 32-bit float for output, if possible. Functionality that uses 16-bit integer + * audio processing (for example, speed and pitch adjustment) will not be available when float + * output is in use. + */ + public DefaultAudioSink( + @Nullable AudioCapabilities audioCapabilities, + AudioProcessorChain audioProcessorChain, + boolean enableConvertHighResIntPcmToFloat) { this.audioCapabilities = audioCapabilities; + this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain); this.enableConvertHighResIntPcmToFloat = enableConvertHighResIntPcmToFloat; releasingConditionVariable = new ConditionVariable(true); audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener()); channelMappingAudioProcessor = new ChannelMappingAudioProcessor(); trimmingAudioProcessor = new TrimmingAudioProcessor(); - silenceSkippingAudioProcessor = new SilenceSkippingAudioProcessor(); - sonicAudioProcessor = new SonicAudioProcessor(); - toIntPcmAvailableAudioProcessors = new AudioProcessor[5 + audioProcessors.length]; - toIntPcmAvailableAudioProcessors[0] = new ResamplingAudioProcessor(); - toIntPcmAvailableAudioProcessors[1] = channelMappingAudioProcessor; - toIntPcmAvailableAudioProcessors[2] = trimmingAudioProcessor; - System.arraycopy( - audioProcessors, 0, toIntPcmAvailableAudioProcessors, 3, audioProcessors.length); - toIntPcmAvailableAudioProcessors[3 + audioProcessors.length] = silenceSkippingAudioProcessor; - toIntPcmAvailableAudioProcessors[4 + audioProcessors.length] = sonicAudioProcessor; + ArrayList toIntPcmAudioProcessors = new ArrayList<>(); + Collections.addAll( + toIntPcmAudioProcessors, + new ResamplingAudioProcessor(), + channelMappingAudioProcessor, + trimmingAudioProcessor); + Collections.addAll(toIntPcmAudioProcessors, audioProcessorChain.getAudioProcessors()); + toIntPcmAvailableAudioProcessors = + toIntPcmAudioProcessors.toArray(new AudioProcessor[toIntPcmAudioProcessors.size()]); toFloatPcmAvailableAudioProcessors = new AudioProcessor[] {new FloatResamplingAudioProcessor()}; volume = 1.0f; startMediaTimeState = START_NOT_SET; @@ -243,7 +357,7 @@ public final class DefaultAudioSink implements AudioSink { audioSessionId = C.AUDIO_SESSION_ID_UNSET; playbackParameters = PlaybackParameters.DEFAULT; drainingAudioProcessorIndex = C.INDEX_UNSET; - this.audioProcessors = new AudioProcessor[0]; + activeAudioProcessors = new AudioProcessor[0]; outputBuffers = new ByteBuffer[0]; playbackParametersCheckpoints = new ArrayDeque<>(); } @@ -423,14 +537,14 @@ public final class DefaultAudioSink implements AudioSink { } } int count = newAudioProcessors.size(); - audioProcessors = newAudioProcessors.toArray(new AudioProcessor[count]); + activeAudioProcessors = newAudioProcessors.toArray(new AudioProcessor[count]); outputBuffers = new ByteBuffer[count]; flushAudioProcessors(); } private void flushAudioProcessors() { - for (int i = 0; i < audioProcessors.length; i++) { - AudioProcessor audioProcessor = audioProcessors[i]; + for (int i = 0; i < activeAudioProcessors.length; i++) { + AudioProcessor audioProcessor = activeAudioProcessors[i]; audioProcessor.flush(); outputBuffers[i] = audioProcessor.getOutput(); } @@ -468,7 +582,7 @@ public final class DefaultAudioSink implements AudioSink { playbackParameters = canApplyPlaybackParameters - ? applyPlaybackParameters(playbackParameters) + ? audioProcessorChain.applyPlaybackParameters(playbackParameters) : PlaybackParameters.DEFAULT; setupAudioProcessors(); @@ -536,7 +650,7 @@ public final class DefaultAudioSink implements AudioSink { } PlaybackParameters newPlaybackParameters = afterDrainPlaybackParameters; afterDrainPlaybackParameters = null; - newPlaybackParameters = applyPlaybackParameters(newPlaybackParameters); + newPlaybackParameters = audioProcessorChain.applyPlaybackParameters(newPlaybackParameters); // Store the position and corresponding media time from which the parameters will apply. playbackParametersCheckpoints.add( new PlaybackParametersCheckpoint( @@ -601,7 +715,7 @@ public final class DefaultAudioSink implements AudioSink { } private void processBuffers(long avSyncPresentationTimeUs) throws WriteException { - int count = audioProcessors.length; + int count = activeAudioProcessors.length; int index = count; while (index >= 0) { ByteBuffer input = index > 0 ? outputBuffers[index - 1] @@ -609,7 +723,7 @@ public final class DefaultAudioSink implements AudioSink { if (index == count) { writeBuffer(input, avSyncPresentationTimeUs); } else { - AudioProcessor audioProcessor = audioProcessors[index]; + AudioProcessor audioProcessor = activeAudioProcessors[index]; audioProcessor.queueInput(input); ByteBuffer output = audioProcessor.getOutput(); outputBuffers[index] = output; @@ -706,11 +820,11 @@ public final class DefaultAudioSink implements AudioSink { private boolean drainAudioProcessorsToEndOfStream() throws WriteException { boolean audioProcessorNeedsEndOfStream = false; if (drainingAudioProcessorIndex == C.INDEX_UNSET) { - drainingAudioProcessorIndex = processingEnabled ? 0 : audioProcessors.length; + drainingAudioProcessorIndex = processingEnabled ? 0 : activeAudioProcessors.length; audioProcessorNeedsEndOfStream = true; } - while (drainingAudioProcessorIndex < audioProcessors.length) { - AudioProcessor audioProcessor = audioProcessors[drainingAudioProcessorIndex]; + while (drainingAudioProcessorIndex < activeAudioProcessors.length) { + AudioProcessor audioProcessor = activeAudioProcessors[drainingAudioProcessorIndex]; if (audioProcessorNeedsEndOfStream) { audioProcessor.queueEndOfStream(); } @@ -762,7 +876,7 @@ public final class DefaultAudioSink implements AudioSink { afterDrainPlaybackParameters = playbackParameters; } else { // Update the playback parameters now. - this.playbackParameters = applyPlaybackParameters(playbackParameters); + this.playbackParameters = audioProcessorChain.applyPlaybackParameters(playbackParameters); } } return this.playbackParameters; @@ -920,29 +1034,14 @@ public final class DefaultAudioSink implements AudioSink { }.start(); } - /** - * Configures audio processors to apply the specified playback parameters, returning the new - * parameters, which may differ from those passed in. - * - * @param playbackParameters The playback parameters to try to apply. - * @return The playback parameters that were actually applied. - */ - private PlaybackParameters applyPlaybackParameters(PlaybackParameters playbackParameters) { - silenceSkippingAudioProcessor.setEnabled(playbackParameters.skipSilence); - return new PlaybackParameters( - sonicAudioProcessor.setSpeed(playbackParameters.speed), - sonicAudioProcessor.setPitch(playbackParameters.pitch), - playbackParameters.skipSilence); - } - - /** - * Returns the underlying audio track {@code positionUs} with any applicable speedup applied. - */ private long applySpeedup(long positionUs) { + @Nullable PlaybackParametersCheckpoint checkpoint = null; while (!playbackParametersCheckpoints.isEmpty() && positionUs >= playbackParametersCheckpoints.getFirst().positionUs) { + checkpoint = playbackParametersCheckpoints.remove(); + } + if (checkpoint != null) { // We are playing (or about to play) media with the new playback parameters, so update them. - PlaybackParametersCheckpoint checkpoint = playbackParametersCheckpoints.remove(); playbackParameters = checkpoint.playbackParameters; playbackParametersPositionUs = checkpoint.positionUs; playbackParametersOffsetUs = checkpoint.mediaTimeUs - startMediaTimeUs; @@ -954,8 +1053,9 @@ public final class DefaultAudioSink implements AudioSink { if (playbackParametersCheckpoints.isEmpty()) { return playbackParametersOffsetUs - + sonicAudioProcessor.scaleDurationForSpeedup(positionUs - playbackParametersPositionUs); + + audioProcessorChain.getMediaDuration(positionUs - playbackParametersPositionUs); } + // We are playing data at a previous playback speed, so fall back to multiplying by the speed. return playbackParametersOffsetUs + Util.getMediaDurationForPlayoutDuration( @@ -963,7 +1063,7 @@ public final class DefaultAudioSink implements AudioSink { } private long applySkipping(long positionUs) { - return positionUs + framesToDurationUs(silenceSkippingAudioProcessor.getSkippedFrames()); + return positionUs + framesToDurationUs(audioProcessorChain.getSkippedOutputFrameCount()); } private boolean isInitialized() { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/SilenceSkippingAudioProcessor.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/SilenceSkippingAudioProcessor.java index b748dbab5d..a289ced128 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/SilenceSkippingAudioProcessor.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/SilenceSkippingAudioProcessor.java @@ -27,7 +27,7 @@ import java.nio.ByteOrder; * An {@link AudioProcessor} that skips silence in the input stream. Input and output are 16-bit * PCM. */ -/* package */ final class SilenceSkippingAudioProcessor implements AudioProcessor { +public final class SilenceSkippingAudioProcessor implements AudioProcessor { /** * The minimum duration of audio that must be below {@link #SILENCE_THRESHOLD_LEVEL} to classify