Merge pull request #119 from ittiam-systems:rtp_h263_test_and_fix

PiperOrigin-RevId: 463146426

RELEASENOTES.md

@@ -30,6 +30,9 @@
         small icon ([#104](https://github.com/androidx/media/issues/104)).
     *   Ensure commands sent before `MediaController.release()` are not dropped
         ([#99](https://github.com/androidx/media/issues/99)).
+*   RTSP:
+    *   Add H263 fragmented packet handling
+        ([#119](https://github.com/androidx/media/pull/119)).
 
 ### 1.0.0-beta02 (2022-07-21)
 

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH263Reader.java

@@ -15,6 +15,8 @@
  */
 package androidx.media3.exoplayer.rtsp.reader;
 
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
 
 import androidx.media3.common.C;
@@ -61,6 +63,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private boolean isKeyFrame;
   private boolean isOutputFormatSet;
   private long startTimeOffsetUs;
+  private long fragmentedSampleTimeUs;
+  /**
+   * Whether the first packet of a H263 frame is received, it mark the start of a H263 partition. A
+   * H263 frame can be split into multiple RTP packets.
+   */
+  private boolean gotFirstPacketOfH263Frame;
 
   /** Creates an instance. */
   public RtpH263Reader(RtpPayloadFormat payloadFormat) {
@@ -76,7 +84,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   @Override
-  public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {}
+  public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {
+    checkState(firstReceivedTimestamp == C.TIME_UNSET);
+    firstReceivedTimestamp = timestamp;
+  }
 
   @Override
   public void consume(
@@ -103,6 +114,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
 
     if (pBitIsSet) {
+      if (gotFirstPacketOfH263Frame && fragmentedSampleSizeBytes > 0) {
+        // Received new H263 fragment, output data of previous fragment to decoder.
+        outputSampleMetadataForFragmentedPackets();
+      }
+      gotFirstPacketOfH263Frame = true;
+
       int payloadStartCode = data.peekUnsignedByte() & 0xFC;
       // Packets that begin with a Picture Start Code(100000). Refer RFC4629 Section 6.1.
       if (payloadStartCode < PICTURE_START_CODE) {
@@ -113,10 +130,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       data.getData()[currentPosition] = 0;
       data.getData()[currentPosition + 1] = 0;
       data.setPosition(currentPosition);
-    } else {
+    } else if (gotFirstPacketOfH263Frame) {
       // Check that this packet is in the sequence of the previous packet.
       int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber);
-      if (sequenceNumber != expectedSequenceNumber) {
+      if (sequenceNumber < expectedSequenceNumber) {
         Log.w(
             TAG,
             Util.formatInvariant(
@@ -125,6 +142,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                 expectedSequenceNumber, sequenceNumber));
         return;
       }
+    } else {
+      Log.w(
+          TAG,
+          "First payload octet of the H263 packet is not the beginning of a new H263 partition,"
+              + " Dropping current packet.");
+      return;
     }
 
     if (fragmentedSampleSizeBytes == 0) {
@@ -141,20 +164,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     // Write the video sample.
     trackOutput.sampleData(data, fragmentSize);
     fragmentedSampleSizeBytes += fragmentSize;
+    fragmentedSampleTimeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
 
     if (rtpMarker) {
-      if (firstReceivedTimestamp == C.TIME_UNSET) {
+      outputSampleMetadataForFragmentedPackets();
-        firstReceivedTimestamp = timestamp;
-      }
-      long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
-      trackOutput.sampleMetadata(
-          timeUs,
-          isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0,
-          fragmentedSampleSizeBytes,
-          /* offset= */ 0,
-          /* cryptoData= */ null);
-      fragmentedSampleSizeBytes = 0;
-      isKeyFrame = false;
     }
     previousSequenceNumber = sequenceNumber;
   }
@@ -167,8 +180,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   /**
-   * Parses and set VOP Coding type and resolution. The {@link ParsableByteArray#position} is
+   * Parses and set VOP Coding type and resolution. The {@linkplain ParsableByteArray#getPosition()
-   * preserved.
+   * position} is preserved.
    */
   private void parseVopHeader(ParsableByteArray data, boolean gotResolution) {
     // Picture Segment Packets (RFC4629 Section 6.1).
@@ -211,6 +224,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     isKeyFrame = false;
   }
 
+  /**
+   * Outputs sample metadata of the received fragmented packets.
+   *
+   * <p>Call this method only after receiving an end of a H263 partition.
+   */
+  private void outputSampleMetadataForFragmentedPackets() {
+    checkNotNull(trackOutput)
+        .sampleMetadata(
+            fragmentedSampleTimeUs,
+            isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0,
+            fragmentedSampleSizeBytes,
+            /* offset= */ 0,
+            /* cryptoData= */ null);
+    fragmentedSampleSizeBytes = 0;
+    fragmentedSampleTimeUs = C.TIME_UNSET;
+    isKeyFrame = false;
+    gotFirstPacketOfH263Frame = false;
+  }
+
   private static long toSampleUs(
       long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) {
     return startTimeOffsetUs

libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java

@@ -0,0 +1,222 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.rtsp.reader;
+
+import static androidx.media3.common.util.Util.getBytesFromHexString;
+import static com.google.common.truth.Truth.assertThat;
+
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.ParsableByteArray;
+import androidx.media3.common.util.Util;
+import androidx.media3.exoplayer.rtsp.RtpPacket;
+import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
+import androidx.media3.test.utils.FakeExtractorOutput;
+import androidx.media3.test.utils.FakeTrackOutput;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.primitives.Bytes;
+import java.util.Arrays;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit test for {@link RtpH263Reader}. */
+@RunWith(AndroidJUnit4.class)
+public final class RtpH263ReaderTest {
+
+  private static final long MEDIA_CLOCK_FREQUENCY = 90_000;
+
+  private static final byte[] FRAME_1_FRAGMENT_1_DATA =
+      getBytesFromHexString("80020c0419b7b7d9591f03023e0c37b");
+  private static final long PARTITION_1_RTP_TIMESTAMP = 2599168056L;
+  private static final RtpPacket PACKET_FRAME_1_FRAGMENT_1 =
+      new RtpPacket.Builder()
+          .setTimestamp(PARTITION_1_RTP_TIMESTAMP)
+          .setSequenceNumber(40289)
+          .setMarker(false)
+          .setPayloadData(
+              Bytes.concat(
+                  /*payload header */ getBytesFromHexString("0400"), FRAME_1_FRAGMENT_1_DATA))
+          .build();
+  private static final byte[] FRAME_1_FRAGMENT_2_DATA =
+      getBytesFromHexString("03140e0e77d5e83021a0c37");
+  private static final RtpPacket PACKET_FRAME_1_FRAGMENT_2 =
+      new RtpPacket.Builder()
+          .setTimestamp(PARTITION_1_RTP_TIMESTAMP)
+          .setSequenceNumber(40290)
+          .setMarker(true)
+          .setPayloadData(
+              Bytes.concat(
+                  /*payload header */ getBytesFromHexString("0000"), FRAME_1_FRAGMENT_2_DATA))
+          .build();
+  // Needs to add 0000 to byte stream, refer to RFC4629 Section 6.1.1.
+  private static final byte[] FRAME_1_DATA =
+      Bytes.concat(getBytesFromHexString("0000"), FRAME_1_FRAGMENT_1_DATA, FRAME_1_FRAGMENT_2_DATA);
+
+  private static final byte[] FRAME_2_FRAGMENT_1_DATA =
+      getBytesFromHexString("800a0e023ffffffffffffffffff");
+  private static final long PARTITION_2_RTP_TIMESTAMP = 2599168344L;
+  private static final RtpPacket PACKET_FRAME_2_FRAGMENT_1 =
+      new RtpPacket.Builder()
+          .setTimestamp(PARTITION_2_RTP_TIMESTAMP)
+          .setSequenceNumber(40291)
+          .setMarker(false)
+          .setPayloadData(
+              Bytes.concat(
+                  /*payload header */ getBytesFromHexString("0400"), FRAME_2_FRAGMENT_1_DATA))
+          .build();
+  private static final byte[] FRAME_2_FRAGMENT_2_DATA =
+      getBytesFromHexString("830df80c501839dfccdbdbecac");
+  private static final RtpPacket PACKET_FRAME_2_FRAGMENT_2 =
+      new RtpPacket.Builder()
+          .setTimestamp(PARTITION_2_RTP_TIMESTAMP)
+          .setSequenceNumber(40292)
+          .setMarker(true)
+          .setPayloadData(
+              Bytes.concat(
+                  /*payload header */ getBytesFromHexString("0000"), FRAME_2_FRAGMENT_2_DATA))
+          .build();
+  private static final byte[] FRAME_2_DATA =
+      Bytes.concat(getBytesFromHexString("0000"), FRAME_2_FRAGMENT_1_DATA, FRAME_2_FRAGMENT_2_DATA);
+
+  private static final long PARTITION_2_PRESENTATION_TIMESTAMP_US =
+      Util.scaleLargeTimestamp(
+          (PARTITION_2_RTP_TIMESTAMP - PARTITION_1_RTP_TIMESTAMP),
+          /* multiplier= */ C.MICROS_PER_SECOND,
+          /* divisor= */ MEDIA_CLOCK_FREQUENCY);
+
+  private static final RtpPayloadFormat H263_FORMAT =
+      new RtpPayloadFormat(
+          new Format.Builder()
+              .setSampleMimeType(MimeTypes.VIDEO_H263)
+              .setWidth(352)
+              .setHeight(288)
+              .build(),
+          /* rtpPayloadType= */ 96,
+          /* clockRate= */ (int) MEDIA_CLOCK_FREQUENCY,
+          /* fmtpParameters= */ ImmutableMap.of());
+
+  private FakeExtractorOutput extractorOutput;
+
+  @Before
+  public void setUp() {
+    extractorOutput = new FakeExtractorOutput();
+  }
+
+  @Test
+  public void consume_validPackets() {
+    RtpH263Reader h263Reader = new RtpH263Reader(H263_FORMAT);
+    h263Reader.createTracks(extractorOutput, /* trackId= */ 0);
+    h263Reader.onReceivingFirstPacket(
+        PACKET_FRAME_1_FRAGMENT_1.timestamp, PACKET_FRAME_1_FRAGMENT_1.sequenceNumber);
+    consume(h263Reader, PACKET_FRAME_1_FRAGMENT_1);
+    consume(h263Reader, PACKET_FRAME_1_FRAGMENT_2);
+    consume(h263Reader, PACKET_FRAME_2_FRAGMENT_1);
+    consume(h263Reader, PACKET_FRAME_2_FRAGMENT_2);
+
+    FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0);
+    assertThat(trackOutput.getSampleCount()).isEqualTo(2);
+    assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_DATA);
+    assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0);
+    assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA);
+    assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US);
+  }
+
+  @Test
+  public void consume_fragmentedFrameMissingFirstFragment() {
+    RtpH263Reader h263Reader = new RtpH263Reader(H263_FORMAT);
+    h263Reader.createTracks(extractorOutput, /* trackId= */ 0);
+    h263Reader.onReceivingFirstPacket(
+        PACKET_FRAME_1_FRAGMENT_1.timestamp, PACKET_FRAME_1_FRAGMENT_1.sequenceNumber);
+    consume(h263Reader, PACKET_FRAME_1_FRAGMENT_2);
+    consume(h263Reader, PACKET_FRAME_2_FRAGMENT_1);
+    consume(h263Reader, PACKET_FRAME_2_FRAGMENT_2);
+
+    FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0);
+    assertThat(trackOutput.getSampleCount()).isEqualTo(1);
+    assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_2_DATA);
+    assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US);
+  }
+
+  @Test
+  public void consume_fragmentedFrameMissingBoundaryFragment() {
+    RtpH263Reader h263Reader = new RtpH263Reader(H263_FORMAT);
+    h263Reader.createTracks(extractorOutput, /* trackId= */ 0);
+    h263Reader.onReceivingFirstPacket(
+        PACKET_FRAME_1_FRAGMENT_1.timestamp, PACKET_FRAME_1_FRAGMENT_1.sequenceNumber);
+    consume(h263Reader, PACKET_FRAME_1_FRAGMENT_1);
+    consume(h263Reader, PACKET_FRAME_2_FRAGMENT_1);
+    consume(h263Reader, PACKET_FRAME_2_FRAGMENT_2);
+
+    FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0);
+    assertThat(trackOutput.getSampleCount()).isEqualTo(2);
+    assertThat(trackOutput.getSampleData(0))
+        .isEqualTo(Bytes.concat(getBytesFromHexString("0000"), FRAME_1_FRAGMENT_1_DATA));
+    assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0);
+    assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA);
+    assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US);
+  }
+
+  @Test
+  public void consume_outOfOrderPackets() {
+    RtpH263Reader h263Reader = new RtpH263Reader(H263_FORMAT);
+    h263Reader.createTracks(extractorOutput, /* trackId= */ 0);
+    h263Reader.onReceivingFirstPacket(
+        PACKET_FRAME_1_FRAGMENT_1.timestamp, PACKET_FRAME_1_FRAGMENT_1.sequenceNumber);
+    consume(h263Reader, PACKET_FRAME_1_FRAGMENT_1);
+    consume(h263Reader, PACKET_FRAME_2_FRAGMENT_1);
+    consume(h263Reader, PACKET_FRAME_1_FRAGMENT_2);
+    consume(h263Reader, PACKET_FRAME_2_FRAGMENT_2);
+
+    FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0);
+    assertThat(trackOutput.getSampleCount()).isEqualTo(2);
+    assertThat(trackOutput.getSampleData(0))
+        .isEqualTo(Bytes.concat(getBytesFromHexString("0000"), FRAME_1_FRAGMENT_1_DATA));
+    assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0);
+    assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA);
+    assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US);
+  }
+
+  private static void consume(RtpH263Reader h263Reader, RtpPacket rtpPacket) {
+    rtpPacket = copyPacket(rtpPacket);
+    h263Reader.consume(
+        new ParsableByteArray(rtpPacket.payloadData),
+        rtpPacket.timestamp,
+        rtpPacket.sequenceNumber,
+        rtpPacket.marker);
+  }
+
+  private static RtpPacket copyPacket(RtpPacket packet) {
+    RtpPacket.Builder builder =
+        new RtpPacket.Builder()
+            .setPadding(packet.padding)
+            .setMarker(packet.marker)
+            .setPayloadType(packet.payloadType)
+            .setSequenceNumber(packet.sequenceNumber)
+            .setTimestamp(packet.timestamp)
+            .setSsrc(packet.ssrc);
+
+    if (packet.csrc.length > 0) {
+      builder.setCsrc(Arrays.copyOf(packet.csrc, packet.csrc.length));
+    }
+    if (packet.payloadData.length > 0) {
+      builder.setPayloadData(Arrays.copyOf(packet.payloadData, packet.payloadData.length));
+    }
+    return builder.build();
+  }
+}