loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Libopus Support For WebM DiscardPadding

Browse Source 
PiperOrigin-RevId: 429364728
This commit is contained in:
olly 2022-02-17 19:49:23 +00:00 committed by Ian Baker
parent cf85d1bdc8
commit f1e59f8001
4 changed files with 162 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
libraries/decoder_opus/src/main/java/androidx/media3/decoder/opus/LibopusAudioRenderer.java
View File

@ -115,6 +115,7 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer<OpusDecoder> {
format.initializationData,
cryptoConfig,
outputFloat);
decoder.experimentalSetDiscardPaddingEnabled(experimentalGetDiscardPaddingEnabled());
TraceUtil.endSection();
return decoder;
@ -126,4 +127,14 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer<OpusDecoder> {
int pcmEncoding = decoder.outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT;
return Util.getPcmFormat(pcmEncoding, decoder.channelCount, OpusDecoder.SAMPLE_RATE);
}
/**
* Returns true if support for padding removal from the end of decoder output buffer should be
* enabled.
*
* <p>This method is experimental, and will be renamed or removed in a future release.
*/
protected boolean experimentalGetDiscardPaddingEnabled() {
return false;
}
}

38
libraries/decoder_opus/src/main/java/androidx/media3/decoder/opus/OpusDecoder.java
View File

@ -56,6 +56,7 @@ public final class OpusDecoder
private final int preSkipSamples;
private final int seekPreRollSamples;
private final long nativeDecoderContext;
private boolean experimentalDiscardPaddingEnabled;
private int skipSamples;
@ -145,6 +146,16 @@ public final class OpusDecoder
}
}
/**
* Sets whether discard padding is enabled. When enabled, discard padding samples (provided as
* supplemental data on the input buffer) will be removed from the end of the decoder output.
*
* <p>This method is experimental, and will be renamed or removed in a future release.
*/
public void experimentalSetDiscardPaddingEnabled(boolean enabled) {
this.experimentalDiscardPaddingEnabled = enabled;
}
@Override
public String getName() {
return "libopus" + OpusLibrary.getVersion();
@ -224,6 +235,14 @@ public final class OpusDecoder
skipSamples = 0;
outputData.position(skipBytes);
}
} else if (experimentalDiscardPaddingEnabled && inputBuffer.hasSupplementalData()) {
int discardPaddingSamples = getDiscardPaddingSamples(inputBuffer.supplementalData);
if (discardPaddingSamples > 0) {
int discardBytes = samplesToBytes(discardPaddingSamples, channelCount, outputFloat);
if (result >= discardBytes) {
outputData.limit(result - discardBytes);
}
}
}
return null;
}
@ -281,6 +300,25 @@ public final class OpusDecoder
return DEFAULT_SEEK_PRE_ROLL_SAMPLES;
}
/**
* Returns the number of discard padding samples specified by the supplemental data attached to an
* input buffer.
*
* @param supplementalData Supplemental data related to the an input buffer.
* @return The number of discard padding samples to remove from the decoder output.
*/
@VisibleForTesting
/* package */ static int getDiscardPaddingSamples(@Nullable ByteBuffer supplementalData) {
if (supplementalData == null || supplementalData.remaining() != 8) {
return 0;
}
long discardPaddingNs = supplementalData.order(ByteOrder.LITTLE_ENDIAN).getLong();
if (discardPaddingNs < 0) {
return 0;
}
return (int) ((discardPaddingNs * SAMPLE_RATE) / C.NANOS_PER_SECOND);
}
/** Returns number of bytes to represent {@code samples}. */
private static int samplesToBytes(int samples, int channelCount, boolean outputFloat) {
int bytesPerChannel = outputFloat ? 4 : 2;

63
libraries/decoder_opus/src/test/java/androidx/media3/decoder/opus/OpusDecoderTest.java
View File

@ -52,6 +52,8 @@ public final class OpusDecoderTest {
private static final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840;
private static final int DISCARD_PADDING_NANOS = 166667;
private static final ImmutableList<byte[]> HEADER_ONLY_INITIALIZATION_DATA =
ImmutableList.of(HEADER);
@ -102,6 +104,20 @@ public final class OpusDecoderTest {
assertThat(seekPreRollSamples).isEqualTo(DEFAULT_SEEK_PRE_ROLL_SAMPLES);
}
@Test
public void getDiscardPaddingSamples_positiveSampleLength_returnSampleLength() {
int discardPaddingSamples =
OpusDecoder.getDiscardPaddingSamples(createSupplementalData(DISCARD_PADDING_NANOS));
assertThat(discardPaddingSamples).isEqualTo(nanosecondsToSampleCount(DISCARD_PADDING_NANOS));
}
@Test
public void getDiscardPaddingSamples_negativeSampleLength_returnZero() {
int discardPaddingSamples =
OpusDecoder.getDiscardPaddingSamples(createSupplementalData(-DISCARD_PADDING_NANOS));
assertThat(discardPaddingSamples).isEqualTo(0);
}
@Test
public void decode_removesPreSkipFromOutput() throws OpusDecoderException {
OpusDecoder decoder =
@ -120,6 +136,49 @@ public final class OpusDecoderTest {
.isEqualTo(DECODED_DATA_SIZE - nanosecondsToBytes(PRE_SKIP_NANOS));
}
@Test
public void decode_whenDiscardPaddingDisabled_returnsDiscardPadding()
throws OpusDecoderException {
OpusDecoder decoder =
new OpusDecoder(
/* numInputBuffers= */ 0,
/* numOutputBuffers= */ 0,
/* initialInputBufferSize= */ 0,
createInitializationData(/* preSkipNanos= */ 0),
/* cryptoConfig= */ null,
/* outputFloat= */ false);
DecoderInputBuffer input =
createInputBuffer(
decoder,
ENCODED_DATA,
/* supplementalData= */ buildNativeOrderByteArray(DISCARD_PADDING_NANOS));
SimpleDecoderOutputBuffer output = decoder.createOutputBuffer();
assertThat(decoder.decode(input, output, false)).isNull();
assertThat(output.data.remaining()).isEqualTo(DECODED_DATA_SIZE);
}
@Test
public void decode_whenDiscardPaddingEnabled_removesDiscardPadding() throws OpusDecoderException {
OpusDecoder decoder =
new OpusDecoder(
/* numInputBuffers= */ 0,
/* numOutputBuffers= */ 0,
/* initialInputBufferSize= */ 0,
createInitializationData(/* preSkipNanos= */ 0),
/* cryptoConfig= */ null,
/* outputFloat= */ false);
decoder.experimentalSetDiscardPaddingEnabled(true);
DecoderInputBuffer input =
createInputBuffer(
decoder,
ENCODED_DATA,
/* supplementalData= */ buildNativeOrderByteArray(DISCARD_PADDING_NANOS));
SimpleDecoderOutputBuffer output = decoder.createOutputBuffer();
assertThat(decoder.decode(input, output, false)).isNull();
assertThat(output.data.limit())
.isEqualTo(DECODED_DATA_SIZE - nanosecondsToBytes(DISCARD_PADDING_NANOS));
}
private static long sampleCountToNanoseconds(long sampleCount) {
return (sampleCount * C.NANOS_PER_SECOND) / OpusDecoder.SAMPLE_RATE;
}
@ -141,6 +200,10 @@ public final class OpusDecoderTest {
return ImmutableList.of(HEADER, preSkip, CUSTOM_SEEK_PRE_ROLL_BYTES);
}
private static ByteBuffer createSupplementalData(long value) {
return ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN).putLong(value).rewind();
}
private static DecoderInputBuffer createInputBuffer(
OpusDecoder decoder, byte[] data, @Nullable byte[] supplementalData) {
DecoderInputBuffer input = decoder.createInputBuffer();

64
libraries/extractor/src/main/java/androidx/media3/extractor/mkv/MatroskaExtractor.java
View File

@ -193,6 +193,7 @@ public class MatroskaExtractor implements Extractor {
private static final int ID_CODEC_PRIVATE = 0x63A2;
private static final int ID_CODEC_DELAY = 0x56AA;
private static final int ID_SEEK_PRE_ROLL = 0x56BB;
private static final int ID_DISCARD_PADDING = 0x75A2;
private static final int ID_VIDEO = 0xE0;
private static final int ID_PIXEL_WIDTH = 0xB0;
private static final int ID_PIXEL_HEIGHT = 0xBA;
@ -391,7 +392,7 @@ public class MatroskaExtractor implements Extractor {
private final ParsableByteArray subtitleSample;
private final ParsableByteArray encryptionInitializationVector;
private final ParsableByteArray encryptionSubsampleData;
private final ParsableByteArray blockAdditionalData;
private final ParsableByteArray supplementalData;
private @MonotonicNonNull ByteBuffer encryptionSubsampleDataBuffer;
private long segmentContentSize;
@ -434,6 +435,7 @@ public class MatroskaExtractor implements Extractor {
private @C.BufferFlags int blockFlags;
private int blockAdditionalId;
private boolean blockHasReferenceBlock;
private long blockGroupDiscardPaddingNs;
// Sample writing state.
private int sampleBytesRead;
@ -472,7 +474,7 @@ public class MatroskaExtractor implements Extractor {
subtitleSample = new ParsableByteArray();
encryptionInitializationVector = new ParsableByteArray(ENCRYPTION_IV_SIZE);
encryptionSubsampleData = new ParsableByteArray();
blockAdditionalData = new ParsableByteArray();
supplementalData = new ParsableByteArray();
blockSampleSizes = new int[1];
}
@ -579,6 +581,7 @@ public class MatroskaExtractor implements Extractor {
case ID_BLOCK_ADD_ID_TYPE:
case ID_CODEC_DELAY:
case ID_SEEK_PRE_ROLL:
case ID_DISCARD_PADDING:
case ID_CHANNELS:
case ID_AUDIO_BIT_DEPTH:
case ID_CONTENT_ENCODING_ORDER:
@ -690,6 +693,7 @@ public class MatroskaExtractor implements Extractor {
break;
case ID_BLOCK_GROUP:
blockHasReferenceBlock = false;
blockGroupDiscardPaddingNs = 0L;
break;
case ID_CONTENT_ENCODING:
// TODO: check and fail if more than one content encoding is present.
@ -750,13 +754,22 @@ public class MatroskaExtractor implements Extractor {
// We've skipped this block (due to incompatible track number).
return;
}
Track track = tracks.get(blockTrackNumber);
track.assertOutputInitialized();
if (blockGroupDiscardPaddingNs > 0L && CODEC_ID_OPUS.equals(track.codecId)) {
// For Opus, attach DiscardPadding to the block group samples as supplemental data.
supplementalData.reset(
ByteBuffer.allocate(8)
.order(ByteOrder.LITTLE_ENDIAN)
.putLong(blockGroupDiscardPaddingNs)
.array());
}
// Commit sample metadata.
int sampleOffset = 0;
for (int i = 0; i < blockSampleCount; i++) {
sampleOffset += blockSampleSizes[i];
}
Track track = tracks.get(blockTrackNumber);
track.assertOutputInitialized();
for (int i = 0; i < blockSampleCount; i++) {
long sampleTimeUs = blockTimeUs + (i * track.defaultSampleDurationNs) / 1000;
int sampleFlags = blockFlags;
@ -888,6 +901,9 @@ public class MatroskaExtractor implements Extractor {
case ID_SEEK_PRE_ROLL:
getCurrentTrack(id).seekPreRollNs = value;
break;
case ID_DISCARD_PADDING:
blockGroupDiscardPaddingNs = value;
break;
case ID_CHANNELS:
getCurrentTrack(id).channelCount = (int) value;
break;
@ -1281,7 +1297,9 @@ public class MatroskaExtractor implements Extractor {
// For SimpleBlock, we can write sample data and immediately commit the corresponding
// sample metadata.
while (blockSampleIndex < blockSampleCount) {
int sampleSize = writeSampleData(input, track, blockSampleSizes[blockSampleIndex]);
int sampleSize =
writeSampleData(
input, track, blockSampleSizes[blockSampleIndex], /* isBlockGroup= */ false);
long sampleTimeUs =
blockTimeUs + (blockSampleIndex * track.defaultSampleDurationNs) / 1000;
commitSampleToOutput(track, sampleTimeUs, blockFlags, sampleSize, /* offset= */ 0);
@ -1296,7 +1314,8 @@ public class MatroskaExtractor implements Extractor {
// the sample data, storing the final sample sizes for when we commit the metadata.
while (blockSampleIndex < blockSampleCount) {
blockSampleSizes[blockSampleIndex] =
writeSampleData(input, track, blockSampleSizes[blockSampleIndex]);
writeSampleData(
input, track, blockSampleSizes[blockSampleIndex], /* isBlockGroup= */ true);
blockSampleIndex++;
}
}
@ -1332,8 +1351,8 @@ public class MatroskaExtractor implements Extractor {
throws IOException {
if (blockAdditionalId == BLOCK_ADDITIONAL_ID_VP9_ITU_T_35
&& CODEC_ID_VP9.equals(track.codecId)) {
blockAdditionalData.reset(contentSize);
input.readFully(blockAdditionalData.getData(), 0, contentSize);
supplementalData.reset(contentSize);
input.readFully(supplementalData.getData(), 0, contentSize);
} else {
// Unhandled block additional data.
input.skipFully(contentSize);
@ -1405,10 +1424,10 @@ public class MatroskaExtractor implements Extractor {
flags &= ~C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA;
} else {
// Append supplemental data.
int blockAdditionalSize = blockAdditionalData.limit();
int supplementalDataSize = supplementalData.limit();
track.output.sampleData(
blockAdditionalData, blockAdditionalSize, TrackOutput.SAMPLE_DATA_PART_SUPPLEMENTAL);
size += blockAdditionalSize;
supplementalData, supplementalDataSize, TrackOutput.SAMPLE_DATA_PART_SUPPLEMENTAL);
size += supplementalDataSize;
}
}
track.output.sampleMetadata(timeUs, flags, size, offset, track.cryptoData);
@ -1437,11 +1456,13 @@ public class MatroskaExtractor implements Extractor {
* @param input The input from which to read sample data.
* @param track The track to output the sample to.
* @param size The size of the sample data on the input side.
* @param isBlockGroup Whether the samples are from a BlockGroup.
* @return The final size of the written sample.
* @throws IOException If an error occurs reading from the input.
*/
@RequiresNonNull("#2.output")
private int writeSampleData(ExtractorInput input, Track track, int size) throws IOException {
private int writeSampleData(ExtractorInput input, Track track, int size, boolean isBlockGroup)
throws IOException {
if (CODEC_ID_SUBRIP.equals(track.codecId)) {
writeSubtitleSampleData(input, SUBRIP_PREFIX, size);
return finishWriteSampleData();
@ -1548,9 +1569,9 @@ public class MatroskaExtractor implements Extractor {
sampleStrippedBytes.reset(track.sampleStrippedBytes, track.sampleStrippedBytes.length);
}
if (track.maxBlockAdditionId > 0) {
if (track.samplesHaveSupplementalData(isBlockGroup)) {
blockFlags |= C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA;
blockAdditionalData.reset(/* limit= */ 0);
supplementalData.reset(/* limit= */ 0);
// If there is supplemental data, the structure of the sample data is:
// encryption data (if any) || sample size (4 bytes) || sample data || supplemental data
int sampleSize = size + sampleStrippedBytes.limit() - sampleBytesRead;
@ -2337,6 +2358,21 @@ public class MatroskaExtractor implements Extractor {
}
}
/**
* Returns true if supplemental data will be attached to the samples.
*
* @param isBlockGroup Whether the samples are from a BlockGroup.
*/
private boolean samplesHaveSupplementalData(boolean isBlockGroup) {
if (CODEC_ID_OPUS.equals(codecId)) {
// At the end of a BlockGroup, a positive DiscardPadding value will be written out as
// supplemental data for Opus codec. Otherwise (i.e. DiscardPadding <= 0) supplemental data
// size will be 0.
return isBlockGroup;
}
return maxBlockAdditionId > 0;
}
/** Returns the HDR Static Info as defined in CTA-861.3. */
@Nullable
private byte[] getHdrStaticInfo() {