Libopus Support For WebM DiscardPadding

PiperOrigin-RevId: 429364728

RELEASENOTES.md

@@ -83,6 +83,7 @@
         ([#9719](https://github.com/google/ExoPlayer/issues/9719)).
     *   Parse Vorbis Comments (including `METADATA_BLOCK_PICTURE`) in Ogg Opus
         and Vorbis files.
+    *   Opus DiscardPadding in Matroska files.
 *   Text:
     *   Add a `MediaItem.SubtitleConfiguration#id` field which is propagated to
         the `Format#id` field of the subtitle track created from the

extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java

@@ -113,6 +113,7 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer<OpusDecoder> {
             format.initializationData,
             cryptoConfig,
             outputFloat);
+    decoder.experimentalSetDiscardPaddingEnabled(experimentalGetDiscardPaddingEnabled());
 
     TraceUtil.endSection();
     return decoder;
@@ -124,4 +125,14 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer<OpusDecoder> {
     int pcmEncoding = decoder.outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT;
     return Util.getPcmFormat(pcmEncoding, decoder.channelCount, OpusDecoder.SAMPLE_RATE);
   }
+
+  /**
+   * Returns true if support for padding removal from the end of decoder output buffer should be
+   * enabled.
+   *
+   * <p>This method is experimental, and will be renamed or removed in a future release.
+   */
+  protected boolean experimentalGetDiscardPaddingEnabled() {
+    return false;
+  }
 }

extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java

@@ -54,6 +54,7 @@ public final class OpusDecoder
   private final int preSkipSamples;
   private final int seekPreRollSamples;
   private final long nativeDecoderContext;
+  private boolean experimentalDiscardPaddingEnabled;
 
   private int skipSamples;
 
@@ -143,6 +144,16 @@ public final class OpusDecoder
     }
   }
 
+  /**
+   * Sets whether discard padding is enabled. When enabled, discard padding samples (provided as
+   * supplemental data on the input buffer) will be removed from the end of the decoder output.
+   *
+   * <p>This method is experimental, and will be renamed or removed in a future release.
+   */
+  public void experimentalSetDiscardPaddingEnabled(boolean enabled) {
+    this.experimentalDiscardPaddingEnabled = enabled;
+  }
+
   @Override
   public String getName() {
     return "libopus" + OpusLibrary.getVersion();
@@ -222,6 +233,14 @@ public final class OpusDecoder
         skipSamples = 0;
         outputData.position(skipBytes);
       }
+    } else if (experimentalDiscardPaddingEnabled && inputBuffer.hasSupplementalData()) {
+      int discardPaddingSamples = getDiscardPaddingSamples(inputBuffer.supplementalData);
+      if (discardPaddingSamples > 0) {
+        int discardBytes = samplesToBytes(discardPaddingSamples, channelCount, outputFloat);
+        if (result >= discardBytes) {
+          outputData.limit(result - discardBytes);
+        }
+      }
     }
     return null;
   }
@@ -279,6 +298,25 @@ public final class OpusDecoder
     return DEFAULT_SEEK_PRE_ROLL_SAMPLES;
   }
 
+  /**
+   * Returns the number of discard padding samples specified by the supplemental data attached to an
+   * input buffer.
+   *
+   * @param supplementalData Supplemental data related to the an input buffer.
+   * @return The number of discard padding samples to remove from the decoder output.
+   */
+  @VisibleForTesting
+  /* package */ static int getDiscardPaddingSamples(@Nullable ByteBuffer supplementalData) {
+    if (supplementalData == null || supplementalData.remaining() != 8) {
+      return 0;
+    }
+    long discardPaddingNs = supplementalData.order(ByteOrder.LITTLE_ENDIAN).getLong();
+    if (discardPaddingNs < 0) {
+      return 0;
+    }
+    return (int) ((discardPaddingNs * SAMPLE_RATE) / C.NANOS_PER_SECOND);
+  }
+
   /** Returns number of bytes to represent {@code samples}. */
   private static int samplesToBytes(int samples, int channelCount, boolean outputFloat) {
     int bytesPerChannel = outputFloat ? 4 : 2;

extensions/opus/src/test/java/com/google/android/exoplayer2/ext/opus/OpusDecoderTest.java

@@ -52,6 +52,8 @@ public final class OpusDecoderTest {
 
   private static final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840;
 
+  private static final int DISCARD_PADDING_NANOS = 166667;
+
   private static final ImmutableList<byte[]> HEADER_ONLY_INITIALIZATION_DATA =
       ImmutableList.of(HEADER);
 
@@ -102,6 +104,20 @@ public final class OpusDecoderTest {
     assertThat(seekPreRollSamples).isEqualTo(DEFAULT_SEEK_PRE_ROLL_SAMPLES);
   }
 
+  @Test
+  public void getDiscardPaddingSamples_positiveSampleLength_returnSampleLength() {
+    int discardPaddingSamples =
+        OpusDecoder.getDiscardPaddingSamples(createSupplementalData(DISCARD_PADDING_NANOS));
+    assertThat(discardPaddingSamples).isEqualTo(nanosecondsToSampleCount(DISCARD_PADDING_NANOS));
+  }
+
+  @Test
+  public void getDiscardPaddingSamples_negativeSampleLength_returnZero() {
+    int discardPaddingSamples =
+        OpusDecoder.getDiscardPaddingSamples(createSupplementalData(-DISCARD_PADDING_NANOS));
+    assertThat(discardPaddingSamples).isEqualTo(0);
+  }
+
   @Test
   public void decode_removesPreSkipFromOutput() throws OpusDecoderException {
     OpusDecoder decoder =
@@ -120,6 +136,49 @@ public final class OpusDecoderTest {
         .isEqualTo(DECODED_DATA_SIZE - nanosecondsToBytes(PRE_SKIP_NANOS));
   }
 
+  @Test
+  public void decode_whenDiscardPaddingDisabled_returnsDiscardPadding()
+      throws OpusDecoderException {
+    OpusDecoder decoder =
+        new OpusDecoder(
+            /* numInputBuffers= */ 0,
+            /* numOutputBuffers= */ 0,
+            /* initialInputBufferSize= */ 0,
+            createInitializationData(/* preSkipNanos= */ 0),
+            /* cryptoConfig= */ null,
+            /* outputFloat= */ false);
+    DecoderInputBuffer input =
+        createInputBuffer(
+            decoder,
+            ENCODED_DATA,
+            /* supplementalData= */ buildNativeOrderByteArray(DISCARD_PADDING_NANOS));
+    SimpleDecoderOutputBuffer output = decoder.createOutputBuffer();
+    assertThat(decoder.decode(input, output, false)).isNull();
+    assertThat(output.data.remaining()).isEqualTo(DECODED_DATA_SIZE);
+  }
+
+  @Test
+  public void decode_whenDiscardPaddingEnabled_removesDiscardPadding() throws OpusDecoderException {
+    OpusDecoder decoder =
+        new OpusDecoder(
+            /* numInputBuffers= */ 0,
+            /* numOutputBuffers= */ 0,
+            /* initialInputBufferSize= */ 0,
+            createInitializationData(/* preSkipNanos= */ 0),
+            /* cryptoConfig= */ null,
+            /* outputFloat= */ false);
+    decoder.experimentalSetDiscardPaddingEnabled(true);
+    DecoderInputBuffer input =
+        createInputBuffer(
+            decoder,
+            ENCODED_DATA,
+            /* supplementalData= */ buildNativeOrderByteArray(DISCARD_PADDING_NANOS));
+    SimpleDecoderOutputBuffer output = decoder.createOutputBuffer();
+    assertThat(decoder.decode(input, output, false)).isNull();
+    assertThat(output.data.limit())
+        .isEqualTo(DECODED_DATA_SIZE - nanosecondsToBytes(DISCARD_PADDING_NANOS));
+  }
+
   private static long sampleCountToNanoseconds(long sampleCount) {
     return (sampleCount * C.NANOS_PER_SECOND) / OpusDecoder.SAMPLE_RATE;
   }
@@ -141,6 +200,10 @@ public final class OpusDecoderTest {
     return ImmutableList.of(HEADER, preSkip, CUSTOM_SEEK_PRE_ROLL_BYTES);
   }
 
+  private static ByteBuffer createSupplementalData(long value) {
+    return ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN).putLong(value).rewind();
+  }
+
   private static DecoderInputBuffer createInputBuffer(
       OpusDecoder decoder, byte[] data, @Nullable byte[] supplementalData) {
     DecoderInputBuffer input = decoder.createInputBuffer();

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mkv/MatroskaExtractor.java

@@ -191,6 +191,7 @@ public class MatroskaExtractor implements Extractor {
   private static final int ID_CODEC_PRIVATE = 0x63A2;
   private static final int ID_CODEC_DELAY = 0x56AA;
   private static final int ID_SEEK_PRE_ROLL = 0x56BB;
+  private static final int ID_DISCARD_PADDING = 0x75A2;
   private static final int ID_VIDEO = 0xE0;
   private static final int ID_PIXEL_WIDTH = 0xB0;
   private static final int ID_PIXEL_HEIGHT = 0xBA;
@@ -389,7 +390,7 @@ public class MatroskaExtractor implements Extractor {
   private final ParsableByteArray subtitleSample;
   private final ParsableByteArray encryptionInitializationVector;
   private final ParsableByteArray encryptionSubsampleData;
-  private final ParsableByteArray blockAdditionalData;
+  private final ParsableByteArray supplementalData;
   private @MonotonicNonNull ByteBuffer encryptionSubsampleDataBuffer;
 
   private long segmentContentSize;
@@ -432,6 +433,7 @@ public class MatroskaExtractor implements Extractor {
   private @C.BufferFlags int blockFlags;
   private int blockAdditionalId;
   private boolean blockHasReferenceBlock;
+  private long blockGroupDiscardPaddingNs;
 
   // Sample writing state.
   private int sampleBytesRead;
@@ -470,7 +472,7 @@ public class MatroskaExtractor implements Extractor {
     subtitleSample = new ParsableByteArray();
     encryptionInitializationVector = new ParsableByteArray(ENCRYPTION_IV_SIZE);
     encryptionSubsampleData = new ParsableByteArray();
-    blockAdditionalData = new ParsableByteArray();
+    supplementalData = new ParsableByteArray();
     blockSampleSizes = new int[1];
   }
 
@@ -577,6 +579,7 @@ public class MatroskaExtractor implements Extractor {
       case ID_BLOCK_ADD_ID_TYPE:
       case ID_CODEC_DELAY:
       case ID_SEEK_PRE_ROLL:
+      case ID_DISCARD_PADDING:
       case ID_CHANNELS:
       case ID_AUDIO_BIT_DEPTH:
       case ID_CONTENT_ENCODING_ORDER:
@@ -688,6 +691,7 @@ public class MatroskaExtractor implements Extractor {
         break;
       case ID_BLOCK_GROUP:
         blockHasReferenceBlock = false;
+        blockGroupDiscardPaddingNs = 0L;
         break;
       case ID_CONTENT_ENCODING:
         // TODO: check and fail if more than one content encoding is present.
@@ -748,13 +752,22 @@ public class MatroskaExtractor implements Extractor {
           // We've skipped this block (due to incompatible track number).
           return;
         }
+        Track track = tracks.get(blockTrackNumber);
+        track.assertOutputInitialized();
+        if (blockGroupDiscardPaddingNs > 0L && CODEC_ID_OPUS.equals(track.codecId)) {
+          // For Opus, attach DiscardPadding to the block group samples as supplemental data.
+          supplementalData.reset(
+              ByteBuffer.allocate(8)
+                  .order(ByteOrder.LITTLE_ENDIAN)
+                  .putLong(blockGroupDiscardPaddingNs)
+                  .array());
+        }
+
         // Commit sample metadata.
         int sampleOffset = 0;
         for (int i = 0; i < blockSampleCount; i++) {
           sampleOffset += blockSampleSizes[i];
         }
-        Track track = tracks.get(blockTrackNumber);
-        track.assertOutputInitialized();
         for (int i = 0; i < blockSampleCount; i++) {
           long sampleTimeUs = blockTimeUs + (i * track.defaultSampleDurationNs) / 1000;
           int sampleFlags = blockFlags;
@@ -886,6 +899,9 @@ public class MatroskaExtractor implements Extractor {
       case ID_SEEK_PRE_ROLL:
         getCurrentTrack(id).seekPreRollNs = value;
         break;
+      case ID_DISCARD_PADDING:
+        blockGroupDiscardPaddingNs = value;
+        break;
       case ID_CHANNELS:
         getCurrentTrack(id).channelCount = (int) value;
         break;
@@ -1279,7 +1295,9 @@ public class MatroskaExtractor implements Extractor {
           // For SimpleBlock, we can write sample data and immediately commit the corresponding
           // sample metadata.
           while (blockSampleIndex < blockSampleCount) {
-            int sampleSize = writeSampleData(input, track, blockSampleSizes[blockSampleIndex]);
+            int sampleSize =
+                writeSampleData(
+                    input, track, blockSampleSizes[blockSampleIndex], /* isBlockGroup= */ false);
             long sampleTimeUs =
                 blockTimeUs + (blockSampleIndex * track.defaultSampleDurationNs) / 1000;
             commitSampleToOutput(track, sampleTimeUs, blockFlags, sampleSize, /* offset= */ 0);
@@ -1294,7 +1312,8 @@ public class MatroskaExtractor implements Extractor {
           // the sample data, storing the final sample sizes for when we commit the metadata.
           while (blockSampleIndex < blockSampleCount) {
             blockSampleSizes[blockSampleIndex] =
-                writeSampleData(input, track, blockSampleSizes[blockSampleIndex]);
+                writeSampleData(
+                    input, track, blockSampleSizes[blockSampleIndex], /* isBlockGroup= */ true);
             blockSampleIndex++;
           }
         }
@@ -1330,8 +1349,8 @@ public class MatroskaExtractor implements Extractor {
       throws IOException {
     if (blockAdditionalId == BLOCK_ADDITIONAL_ID_VP9_ITU_T_35
         && CODEC_ID_VP9.equals(track.codecId)) {
-      blockAdditionalData.reset(contentSize);
+      supplementalData.reset(contentSize);
-      input.readFully(blockAdditionalData.getData(), 0, contentSize);
+      input.readFully(supplementalData.getData(), 0, contentSize);
     } else {
       // Unhandled block additional data.
       input.skipFully(contentSize);
@@ -1403,10 +1422,10 @@ public class MatroskaExtractor implements Extractor {
           flags &= ~C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA;
         } else {
           // Append supplemental data.
-          int blockAdditionalSize = blockAdditionalData.limit();
+          int supplementalDataSize = supplementalData.limit();
           track.output.sampleData(
-              blockAdditionalData, blockAdditionalSize, TrackOutput.SAMPLE_DATA_PART_SUPPLEMENTAL);
+              supplementalData, supplementalDataSize, TrackOutput.SAMPLE_DATA_PART_SUPPLEMENTAL);
-          size += blockAdditionalSize;
+          size += supplementalDataSize;
         }
       }
       track.output.sampleMetadata(timeUs, flags, size, offset, track.cryptoData);
@@ -1435,11 +1454,13 @@ public class MatroskaExtractor implements Extractor {
    * @param input The input from which to read sample data.
    * @param track The track to output the sample to.
    * @param size The size of the sample data on the input side.
+   * @param isBlockGroup Whether the samples are from a BlockGroup.
    * @return The final size of the written sample.
    * @throws IOException If an error occurs reading from the input.
    */
   @RequiresNonNull("#2.output")
-  private int writeSampleData(ExtractorInput input, Track track, int size) throws IOException {
+  private int writeSampleData(ExtractorInput input, Track track, int size, boolean isBlockGroup)
+      throws IOException {
     if (CODEC_ID_SUBRIP.equals(track.codecId)) {
       writeSubtitleSampleData(input, SUBRIP_PREFIX, size);
       return finishWriteSampleData();
@@ -1546,9 +1567,9 @@ public class MatroskaExtractor implements Extractor {
         sampleStrippedBytes.reset(track.sampleStrippedBytes, track.sampleStrippedBytes.length);
       }
 
-      if (track.maxBlockAdditionId > 0) {
+      if (track.samplesHaveSupplementalData(isBlockGroup)) {
         blockFlags |= C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA;
-        blockAdditionalData.reset(/* limit= */ 0);
+        supplementalData.reset(/* limit= */ 0);
         // If there is supplemental data, the structure of the sample data is:
         // encryption data (if any) || sample size (4 bytes) || sample data || supplemental data
         int sampleSize = size + sampleStrippedBytes.limit() - sampleBytesRead;
@@ -2335,6 +2356,21 @@ public class MatroskaExtractor implements Extractor {
       }
     }
 
+    /**
+     * Returns true if supplemental data will be attached to the samples.
+     *
+     * @param isBlockGroup Whether the samples are from a BlockGroup.
+     */
+    private boolean samplesHaveSupplementalData(boolean isBlockGroup) {
+      if (CODEC_ID_OPUS.equals(codecId)) {
+        // At the end of a BlockGroup, a positive DiscardPadding value will be written out as
+        // supplemental data for Opus codec. Otherwise (i.e. DiscardPadding <= 0) supplemental data
+        // size will be 0.
+        return isBlockGroup;
+      }
+      return maxBlockAdditionId > 0;
+    }
+
     /** Returns the HDR Static Info as defined in CTA-861.3. */
     @Nullable
     private byte[] getHdrStaticInfo() {