Merge pull request #53 from ittiam-systems:rtp_opus

PiperOrigin-RevId: 453490088

RELEASENOTES.md

@@ -127,6 +127,8 @@
         ([#10165](https://github.com/google/ExoPlayer/issues/10165)).
     *   Add RTP reader for VP9
         ([#47](https://github.com/androidx/media/pull/64)).
+    *   Add RTP reader for OPUS
+        ([#53](https://github.com/androidx/media/pull/53)).
 *   Session:
     *   Fix NPE in MediaControllerImplLegacy
         ([#59](https://github.com/androidx/media/pull/59)).

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java

@@ -46,6 +46,7 @@ public final class RtpPayloadFormat {
   private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES";
   private static final String RTP_MEDIA_H264 = "H264";
   private static final String RTP_MEDIA_H265 = "H265";
+  private static final String RTP_MEDIA_OPUS = "OPUS";
   private static final String RTP_MEDIA_PCM_L8 = "L8";
   private static final String RTP_MEDIA_PCM_L16 = "L16";
   private static final String RTP_MEDIA_PCMA = "PCMA";
@@ -63,6 +64,7 @@ public final class RtpPayloadFormat {
       case RTP_MEDIA_H265:
       case RTP_MEDIA_MPEG4_VIDEO:
       case RTP_MEDIA_MPEG4_GENERIC:
+      case RTP_MEDIA_OPUS:
       case RTP_MEDIA_PCM_L8:
       case RTP_MEDIA_PCM_L16:
       case RTP_MEDIA_PCMA:
@@ -92,6 +94,8 @@ public final class RtpPayloadFormat {
         return MimeTypes.AUDIO_AMR_WB;
       case RTP_MEDIA_MPEG4_GENERIC:
         return MimeTypes.AUDIO_AAC;
+      case RTP_MEDIA_OPUS:
+        return MimeTypes.AUDIO_OPUS;
       case RTP_MEDIA_PCM_L8:
       case RTP_MEDIA_PCM_L16:
         return MimeTypes.AUDIO_RAW;

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java

@@ -101,6 +101,9 @@ import com.google.common.collect.ImmutableMap;
    */
   private static final int DEFAULT_VP8_HEIGHT = 240;
 
+  /** RFC7587 Section 6.1 Sampling rate for OPUS is fixed at 48KHz. */
+  private static final int OPUS_CLOCK_RATE = 48_000;
+
   /**
    * Default width for VP9.
    *
@@ -201,6 +204,12 @@ import com.google.common.collect.ImmutableMap;
             !fmtpParameters.containsKey(PARAMETER_AMR_INTERLEAVING),
             "Interleaving mode is not currently supported.");
         break;
+      case MimeTypes.AUDIO_OPUS:
+        checkArgument(channelCount != C.INDEX_UNSET);
+        // RFC7587 Section 6.1: the RTP timestamp is incremented with a 48000 Hz clock rate
+        // for all modes of Opus and all sampling rates.
+        checkArgument(clockRate == OPUS_CLOCK_RATE, "Invalid OPUS clock rate.");
+        break;
       case MimeTypes.VIDEO_MP4V:
         checkArgument(!fmtpParameters.isEmpty());
         processMPEG4FmtpAttribute(formatBuilder, fmtpParameters);

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java

@@ -39,6 +39,8 @@ import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
       case MimeTypes.AUDIO_AMR_NB:
       case MimeTypes.AUDIO_AMR_WB:
         return new RtpAmrReader(payloadFormat);
+      case MimeTypes.AUDIO_OPUS:
+        return new RtpOpusReader(payloadFormat);
       case MimeTypes.AUDIO_RAW:
       case MimeTypes.AUDIO_ALAW:
       case MimeTypes.AUDIO_MLAW:

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReader.java

@@ -0,0 +1,157 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.rtsp.reader;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
+
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.util.Log;
+import androidx.media3.common.util.ParsableByteArray;
+import androidx.media3.common.util.Util;
+import androidx.media3.exoplayer.rtsp.RtpPacket;
+import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
+import androidx.media3.extractor.ExtractorOutput;
+import androidx.media3.extractor.OpusUtil;
+import androidx.media3.extractor.TrackOutput;
+import java.util.List;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
+/**
+ * Parses an OPUS byte stream carried on RTP packets and extracts individual samples. Refer to
+ * RFC7845 for more details.
+ */
+/* package */ final class RtpOpusReader implements RtpPayloadReader {
+  private static final String TAG = "RtpOpusReader";
+  /* Opus uses a fixed 48KHz media clock RFC7845 Section 4. */
+  private static final long MEDIA_CLOCK_FREQUENCY = 48_000;
+
+  private final RtpPayloadFormat payloadFormat;
+
+  private @MonotonicNonNull TrackOutput trackOutput;
+
+  /**
+   * First received RTP timestamp. All RTP timestamps are dimension-less, the time base is defined
+   * by {@link #MEDIA_CLOCK_FREQUENCY}.
+   */
+  private long firstReceivedTimestamp;
+
+  private long startTimeOffsetUs;
+  private int previousSequenceNumber;
+  private boolean foundOpusIDHeader;
+  private boolean foundOpusCommentHeader;
+
+  /** Creates an instance. */
+  public RtpOpusReader(RtpPayloadFormat payloadFormat) {
+    this.payloadFormat = payloadFormat;
+    this.firstReceivedTimestamp = C.INDEX_UNSET;
+    this.previousSequenceNumber = C.INDEX_UNSET;
+  }
+
+  // RtpPayloadReader implementation.
+
+  @Override
+  public void createTracks(ExtractorOutput extractorOutput, int trackId) {
+    trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_AUDIO);
+    trackOutput.format(payloadFormat.format);
+  }
+
+  @Override
+  public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {
+    this.firstReceivedTimestamp = timestamp;
+  }
+
+  @Override
+  public void consume(
+      ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) {
+    checkStateNotNull(trackOutput);
+
+    /* RFC7845 Section 3.
+     * +---------+ +----------------+ +--------------------+ +-----
+     * |ID Header| | Comment Header | |Audio Data Packet 1 | | ...
+     * +---------+ +----------------+ +--------------------+ +-----
+     */
+    if (!foundOpusIDHeader) {
+      validateOpusIdHeader(data);
+      List<byte[]> initializationData = OpusUtil.buildInitializationData(data.getData());
+      Format.Builder formatBuilder = payloadFormat.format.buildUpon();
+      formatBuilder.setInitializationData(initializationData);
+      trackOutput.format(formatBuilder.build());
+      foundOpusIDHeader = true;
+    } else if (!foundOpusCommentHeader) {
+      // Comment Header RFC7845 Section 5.2.
+      int sampleSize = data.limit();
+      checkArgument(sampleSize >= 8, "Comment Header has insufficient data");
+      String header = data.readString(8);
+      checkArgument(header.equals("OpusTags"), "Comment Header should follow ID Header");
+      foundOpusCommentHeader = true;
+    } else {
+      // Check that this packet is in the sequence of the previous packet.
+      int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber);
+      if (sequenceNumber != expectedSequenceNumber) {
+        Log.w(
+            TAG,
+            Util.formatInvariant(
+                "Received RTP packet with unexpected sequence number. Expected: %d; received: %d.",
+                expectedSequenceNumber, sequenceNumber));
+      }
+
+      // sending opus data.
+      int size = data.bytesLeft();
+      trackOutput.sampleData(data, size);
+      long timeUs = toSampleTimeUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
+      trackOutput.sampleMetadata(
+          timeUs, C.BUFFER_FLAG_KEY_FRAME, size, /* offset*/ 0, /* cryptoData*/ null);
+    }
+    previousSequenceNumber = sequenceNumber;
+  }
+
+  @Override
+  public void seek(long nextRtpTimestamp, long timeUs) {
+    firstReceivedTimestamp = nextRtpTimestamp;
+    startTimeOffsetUs = timeUs;
+  }
+
+  // Internal methods.
+
+  /**
+   * Validates the OPUS ID Header at {@code data}'s current position, throws {@link
+   * IllegalArgumentException} if the header is invalid.
+   *
+   * <p>{@code data}'s position does not change after returning.
+   */
+  private static void validateOpusIdHeader(ParsableByteArray data) {
+    int currPosition = data.getPosition();
+    int sampleSize = data.limit();
+    checkArgument(sampleSize > 18, "ID Header has insufficient data");
+    String header = data.readString(8);
+    // Identification header RFC7845 Section 5.1.
+    checkArgument(header.equals("OpusHead"), "ID Header missing");
+    checkArgument(data.readUnsignedByte() == 1, "version number must always be 1");
+    data.setPosition(currPosition);
+  }
+
+  /** Returns the correct sample time from RTP timestamp, accounting for the OPUS sampling rate. */
+  private static long toSampleTimeUs(
+      long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) {
+    return startTimeOffsetUs
+        + Util.scaleLargeTimestamp(
+            rtpTimestamp - firstReceivedRtpTimestamp,
+            /* multiplier= */ C.MICROS_PER_SECOND,
+            /* divisor= */ MEDIA_CLOCK_FREQUENCY);
+  }
+}

libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java

@@ -0,0 +1,200 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.rtsp.reader;
+
+import static androidx.media3.common.util.Util.getBytesFromHexString;
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.when;
+
+import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.ParsableByteArray;
+import androidx.media3.exoplayer.rtsp.RtpPacket;
+import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
+import androidx.media3.extractor.ExtractorOutput;
+import androidx.media3.test.utils.FakeTrackOutput;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableMap;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+
+/** Unit test for {@link RtpOpusReader}. */
+@RunWith(AndroidJUnit4.class)
+public final class RtpOpusReaderTest {
+
+  private static final RtpPayloadFormat OPUS_FORMAT =
+      new RtpPayloadFormat(
+          new Format.Builder()
+              .setChannelCount(6)
+              .setSampleMimeType(MimeTypes.AUDIO_OPUS)
+              .setSampleRate(48_000)
+              .build(),
+          /* rtpPayloadType= */ 97,
+          /* clockRate= */ 48_000,
+          /* fmtpParameters= */ ImmutableMap.of());
+
+  private static final RtpPacket OPUS_HEADER =
+      createRtpPacket(
+          /* timestamp= */ 2599168056L,
+          /* sequenceNumber= */ 40289,
+          /* payloadData= */ getBytesFromHexString("4F707573486561640102000000000000000000"));
+  private static final RtpPacket OPUS_TAGS =
+      createRtpPacket(
+          /* timestamp= */ 2599168056L,
+          /* sequenceNumber= */ 40290,
+          /* payloadData= */ getBytesFromHexString("4F707573546167730000000000000000000000"));
+  private static final RtpPacket OPUS_FRAME_1 =
+      createRtpPacket(
+          /* timestamp= */ 2599168056L,
+          /* sequenceNumber= */ 40292,
+          /* payloadData= */ getBytesFromHexString("010203"));
+  private static final RtpPacket OPUS_FRAME_2 =
+      createRtpPacket(
+          /* timestamp= */ 2599169592L,
+          /* sequenceNumber= */ 40293,
+          /* payloadData= */ getBytesFromHexString("04050607"));
+
+  @Rule public final MockitoRule mockito = MockitoJUnit.rule();
+
+  private ParsableByteArray packetData;
+  private RtpOpusReader opusReader;
+  private FakeTrackOutput trackOutput;
+  @Mock private ExtractorOutput extractorOutput;
+
+  @Before
+  public void setUp() {
+    packetData = new ParsableByteArray();
+    trackOutput = new FakeTrackOutput(/* deduplicateConsecutiveFormats= */ true);
+    when(extractorOutput.track(anyInt(), anyInt())).thenReturn(trackOutput);
+    opusReader = new RtpOpusReader(OPUS_FORMAT);
+    opusReader.createTracks(extractorOutput, /* trackId= */ 0);
+  }
+
+  @Test
+  public void consume_validPackets() {
+    opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
+    consume(OPUS_HEADER);
+    consume(OPUS_TAGS);
+    consume(OPUS_FRAME_1);
+    consume(OPUS_FRAME_2);
+
+    assertThat(trackOutput.getSampleCount()).isEqualTo(2);
+    assertThat(trackOutput.getSampleData(0)).isEqualTo(getBytesFromHexString("010203"));
+    assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0);
+    assertThat(trackOutput.getSampleData(1)).isEqualTo(getBytesFromHexString("04050607"));
+    assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(32000);
+  }
+
+  @Test
+  public void consume_opusHeaderWithInvalidHeader_throwsIllegalArgumentException() {
+    opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
+    assertThrows(
+        IllegalArgumentException.class,
+        () ->
+            consume(
+                createRtpPacket(
+                    /* timestamp= */ 2599168056L,
+                    /* sequenceNumber= */ 40289,
+                    // Modify "OpusHead" -> "OrusHead" (First 8 bytes).
+                    /* payloadData= */ getBytesFromHexString(
+                        "4F727573486561640102000000000000000000"))));
+  }
+
+  @Test
+  public void consume_opusHeaderWithInvalidSampleSize_throwsIllegalArgumentException() {
+    opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
+
+    assertThrows(
+        IllegalArgumentException.class,
+        () ->
+            consume(
+                createRtpPacket(
+                    /* timestamp= */ 2599168056L,
+                    /* sequenceNumber= */ 40289,
+                    // Truncate the opusHeader payload data.
+                    /* payloadData= */ getBytesFromHexString("4F707573486561640102"))));
+  }
+
+  @Test
+  public void consume_opusHeaderWithInvalidVersionNumber_throwsIllegalArgumentException() {
+    opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
+    assertThrows(
+        IllegalArgumentException.class,
+        () ->
+            consume(
+                createRtpPacket(
+                    /* timestamp= */ 2599168056L,
+                    /* sequenceNumber= */ 40289,
+                    // Modify version 1 -> 2 (9th byte)
+                    /* payloadData= */ getBytesFromHexString(
+                        "4f707573486561640202000000000000000000"))));
+  }
+
+  @Test
+  public void consume_invalidOpusTags_throwsIllegalArgumentException() {
+    opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
+    consume(OPUS_HEADER);
+    assertThrows(
+        IllegalArgumentException.class,
+        () ->
+            consume(
+                createRtpPacket(
+                    /* timestamp= */ 2599168056L,
+                    /* sequenceNumber= */ 40290,
+                    // Modify "OpusTags" -> "OpusTggs" (First 8 bytes)
+                    /* payloadData= */ getBytesFromHexString("4F70757354676773"))));
+  }
+
+  @Test
+  public void consume_skipOpusTags_throwsIllegalArgumentException() {
+    opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
+    consume(OPUS_HEADER);
+    assertThrows(IllegalArgumentException.class, () -> consume(OPUS_FRAME_1));
+  }
+
+  @Test
+  public void consume_skipOpusHeader_throwsIllegalArgumentException() {
+    opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
+    assertThrows(IllegalArgumentException.class, () -> consume(OPUS_TAGS));
+  }
+
+  @Test
+  public void consume_skipOpusHeaderAndOpusTags_throwsIllegalArgumentException() {
+    opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
+    assertThrows(IllegalArgumentException.class, () -> consume(OPUS_FRAME_1));
+  }
+
+  private static RtpPacket createRtpPacket(long timestamp, int sequenceNumber, byte[] payloadData) {
+    return new RtpPacket.Builder()
+        .setTimestamp(timestamp)
+        .setSequenceNumber(sequenceNumber)
+        .setMarker(false)
+        .setPayloadData(payloadData)
+        .build();
+  }
+
+  private void consume(RtpPacket rtpPacket) {
+    packetData.reset(rtpPacket.payloadData);
+    opusReader.consume(packetData, rtpPacket.timestamp, rtpPacket.sequenceNumber, rtpPacket.marker);
+  }
+}