loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resample float audio to 16-bit by default to enable audio processing

Browse Source 
This is less confusing than having audio processing functionality (e.g., playback
speed adjustment) just "not work" for some pieces of media.

If this change is merged, I will update #6749 to also track making DefaultAudioSink
intelligently enable/disable float output depending on how the audio processors are
configured.

Issue: #7134
PiperOrigin-RevId: 302871568
This commit is contained in:
olly 2020-03-25 12:50:03 +00:00 committed by Oliver Woodman
parent f794a79852
commit 2555fb36b4
4 changed files with 76 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
RELEASENOTES.md
View File

@ -60,6 +60,12 @@
* `SimpleDecoderVideoRenderer` and `SimpleDecoderAudioRenderer` renamed to
`DecoderVideoRenderer` and `DecoderAudioRenderer` respectively, and
generalized to work with `Decoder` rather than `SimpleDecoder`.
* Audio:
* Enable playback speed adjustment and silence skipping for floating point
PCM audio, via resampling to 16-bit integer PCM. To output the original
floating point audio without adjustment, pass `enableFloatOutput=true`
to the `DefaultAudioSink` constructor
([#7134](https://github.com/google/ExoPlayer/issues/7134)).
* Text:
* Parse `<ruby>` and `<rt>` tags in WebVTT subtitles (rendering is coming
later).

10
library/common/src/main/java/com/google/android/exoplayer2/util/Util.java
View File

@ -1385,13 +1385,15 @@ public final class Util {
}
/**
* Returns whether {@code encoding} is high resolution (&gt; 16-bit) integer PCM.
* Returns whether {@code encoding} is high resolution (&gt; 16-bit) PCM.
*
* @param encoding The encoding of the audio data.
* @return Whether the encoding is high resolution integer PCM.
* @return Whether the encoding is high resolution PCM.
*/
public static boolean isEncodingHighResolutionIntegerPcm(@C.PcmEncoding int encoding) {
return encoding == C.ENCODING_PCM_24BIT || encoding == C.ENCODING_PCM_32BIT;
public static boolean isEncodingHighResolutionPcm(@C.PcmEncoding int encoding) {
return encoding == C.ENCODING_PCM_24BIT
|| encoding == C.ENCODING_PCM_32BIT
|| encoding == C.ENCODING_PCM_FLOAT;
}
/**

45
library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java
View File

@ -256,7 +256,7 @@ public final class DefaultAudioSink implements AudioSink {
@Nullable private final AudioCapabilities audioCapabilities;
private final AudioProcessorChain audioProcessorChain;
private final boolean enableConvertHighResIntPcmToFloat;
private final boolean enableFloatOutput;
private final ChannelMappingAudioProcessor channelMappingAudioProcessor;
private final TrimmingAudioProcessor trimmingAudioProcessor;
private final AudioProcessor[] toIntPcmAvailableAudioProcessors;
@ -316,7 +316,7 @@ public final class DefaultAudioSink implements AudioSink {
*/
public DefaultAudioSink(
@Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors) {
this(audioCapabilities, audioProcessors, /* enableConvertHighResIntPcmToFloat= */ false);
this(audioCapabilities, audioProcessors, /* enableFloatOutput= */ false);
}
/**
@ -326,19 +326,16 @@ public final class DefaultAudioSink implements AudioSink {
* default capabilities (no encoded audio passthrough support) should be assumed.
* @param audioProcessors An array of {@link AudioProcessor}s that will process PCM audio before
* output. May be empty.
* @param enableConvertHighResIntPcmToFloat Whether to enable conversion of high resolution
* integer PCM to 32-bit float for output, if possible. Functionality that uses 16-bit integer
* audio processing (for example, speed adjustment) will not be available when float output is
* in use.
* @param enableFloatOutput Whether to enable 32-bit float output. Where possible, 32-bit float
* output will be used if the input is 32-bit float, and also if the input is high resolution
* (24-bit or 32-bit) integer PCM. Audio processing (for example, speed adjustment) will not
* be available when float output is in use.
*/
public DefaultAudioSink(
@Nullable AudioCapabilities audioCapabilities,
AudioProcessor[] audioProcessors,
boolean enableConvertHighResIntPcmToFloat) {
this(
audioCapabilities,
new DefaultAudioProcessorChain(audioProcessors),
enableConvertHighResIntPcmToFloat);
boolean enableFloatOutput) {
this(audioCapabilities, new DefaultAudioProcessorChain(audioProcessors), enableFloatOutput);
}
/**
@ -349,18 +346,18 @@ public final class DefaultAudioSink implements AudioSink {
* default capabilities (no encoded audio passthrough support) should be assumed.
* @param audioProcessorChain An {@link AudioProcessorChain} which is used to apply playback
* parameters adjustments. The instance passed in must not be reused in other sinks.
* @param enableConvertHighResIntPcmToFloat Whether to enable conversion of high resolution
* integer PCM to 32-bit float for output, if possible. Functionality that uses 16-bit integer
* audio processing (for example, speed adjustment) will not be available when float output is
* in use.
* @param enableFloatOutput Whether to enable 32-bit float output. Where possible, 32-bit float
* output will be used if the input is 32-bit float, and also if the input is high resolution
* (24-bit or 32-bit) integer PCM. Audio processing (for example, speed adjustment) will not
* be available when float output is in use.
*/
public DefaultAudioSink(
@Nullable AudioCapabilities audioCapabilities,
AudioProcessorChain audioProcessorChain,
boolean enableConvertHighResIntPcmToFloat) {
boolean enableFloatOutput) {
this.audioCapabilities = audioCapabilities;
this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain);
this.enableConvertHighResIntPcmToFloat = enableConvertHighResIntPcmToFloat;
this.enableFloatOutput = enableFloatOutput;
releasingConditionVariable = new ConditionVariable(true);
audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener());
channelMappingAudioProcessor = new ChannelMappingAudioProcessor();
@ -443,18 +440,16 @@ public final class DefaultAudioSink implements AudioSink {
}
boolean isInputPcm = Util.isEncodingLinearPcm(inputEncoding);
boolean processingEnabled = isInputPcm && inputEncoding != C.ENCODING_PCM_FLOAT;
boolean processingEnabled = isInputPcm;
int sampleRate = inputSampleRate;
int channelCount = inputChannelCount;
@C.Encoding int encoding = inputEncoding;
boolean shouldConvertHighResIntPcmToFloat =
enableConvertHighResIntPcmToFloat
boolean useFloatOutput =
enableFloatOutput
&& supportsOutput(inputChannelCount, C.ENCODING_PCM_FLOAT)
&& Util.isEncodingHighResolutionIntegerPcm(inputEncoding);
&& Util.isEncodingHighResolutionPcm(inputEncoding);
AudioProcessor[] availableAudioProcessors =
shouldConvertHighResIntPcmToFloat
? toFloatPcmAvailableAudioProcessors
: toIntPcmAvailableAudioProcessors;
useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors;
if (processingEnabled) {
trimmingAudioProcessor.setTrimFrameCount(trimStartFrames, trimEndFrames);
channelMappingAudioProcessor.setChannelMap(outputChannels);
@ -484,7 +479,7 @@ public final class DefaultAudioSink implements AudioSink {
isInputPcm ? Util.getPcmFrameSize(inputEncoding, inputChannelCount) : C.LENGTH_UNSET;
int outputPcmFrameSize =
isInputPcm ? Util.getPcmFrameSize(encoding, channelCount) : C.LENGTH_UNSET;
boolean canApplyPlaybackParameters = processingEnabled && !shouldConvertHighResIntPcmToFloat;
boolean canApplyPlaybackParameters = processingEnabled && !useFloatOutput;
Configuration pendingConfiguration =
new Configuration(
isInputPcm,

70
library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java
View File

@ -16,13 +16,19 @@
package com.google.android.exoplayer2.audio;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.util.Util;
import java.nio.ByteBuffer;
/**
* An {@link AudioProcessor} that converts 24-bit and 32-bit integer PCM audio to 32-bit float PCM
* audio.
* An {@link AudioProcessor} that converts high resolution PCM audio to 32-bit float. The following
* encodings are supported as input:
*
* <ul>
* <li>{@link C#ENCODING_PCM_24BIT}
* <li>{@link C#ENCODING_PCM_32BIT}
* <li>{@link C#ENCODING_PCM_FLOAT} ({@link #isActive()} will return {@code false})
* </ul>
*/
/* package */ final class FloatResamplingAudioProcessor extends BaseAudioProcessor {
@ -32,10 +38,11 @@ import java.nio.ByteBuffer;
@Override
public AudioFormat onConfigure(AudioFormat inputAudioFormat)
throws UnhandledAudioFormatException {
if (!Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding)) {
@C.PcmEncoding int encoding = inputAudioFormat.encoding;
if (!Util.isEncodingHighResolutionPcm(encoding)) {
throw new UnhandledAudioFormatException(inputAudioFormat);
}
return Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding)
return encoding != C.ENCODING_PCM_FLOAT
? new AudioFormat(
inputAudioFormat.sampleRate, inputAudioFormat.channelCount, C.ENCODING_PCM_FLOAT)
: AudioFormat.NOT_SET;
@ -43,31 +50,42 @@ import java.nio.ByteBuffer;
@Override
public void queueInput(ByteBuffer inputBuffer) {
Assertions.checkState(Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding));
boolean isInput32Bit = inputAudioFormat.encoding == C.ENCODING_PCM_32BIT;
int position = inputBuffer.position();
int limit = inputBuffer.limit();
int size = limit - position;
int resampledSize = isInput32Bit ? size : (size / 3) * 4;
ByteBuffer buffer = replaceOutputBuffer(resampledSize);
if (isInput32Bit) {
for (int i = position; i < limit; i += 4) {
int pcm32BitInteger =
(inputBuffer.get(i) & 0xFF)
| ((inputBuffer.get(i + 1) & 0xFF) << 8)
| ((inputBuffer.get(i + 2) & 0xFF) << 16)
| ((inputBuffer.get(i + 3) & 0xFF) << 24);
writePcm32BitFloat(pcm32BitInteger, buffer);
}
} else { // Input is 24-bit PCM.
for (int i = position; i < limit; i += 3) {
int pcm32BitInteger =
((inputBuffer.get(i) & 0xFF) << 8)
| ((inputBuffer.get(i + 1) & 0xFF) << 16)
| ((inputBuffer.get(i + 2) & 0xFF) << 24);
writePcm32BitFloat(pcm32BitInteger, buffer);
}
ByteBuffer buffer;
switch (inputAudioFormat.encoding) {
case C.ENCODING_PCM_24BIT:
buffer = replaceOutputBuffer((size / 3) * 4);
for (int i = position; i < limit; i += 3) {
int pcm32BitInteger =
((inputBuffer.get(i) & 0xFF) << 8)
| ((inputBuffer.get(i + 1) & 0xFF) << 16)
| ((inputBuffer.get(i + 2) & 0xFF) << 24);
writePcm32BitFloat(pcm32BitInteger, buffer);
}
break;
case C.ENCODING_PCM_32BIT:
buffer = replaceOutputBuffer(size);
for (int i = position; i < limit; i += 4) {
int pcm32BitInteger =
(inputBuffer.get(i) & 0xFF)
| ((inputBuffer.get(i + 1) & 0xFF) << 8)
| ((inputBuffer.get(i + 2) & 0xFF) << 16)
| ((inputBuffer.get(i + 3) & 0xFF) << 24);
writePcm32BitFloat(pcm32BitInteger, buffer);
}
break;
case C.ENCODING_PCM_8BIT:
case C.ENCODING_PCM_16BIT:
case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
case C.ENCODING_PCM_FLOAT:
case C.ENCODING_INVALID:
case Format.NO_VALUE:
default:
// Never happens.
throw new IllegalStateException();
}
inputBuffer.position(inputBuffer.limit());