diff --git a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtpPayloadFormat.java b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtpPayloadFormat.java index 602e823d13..d6fbf43f0e 100644 --- a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtpPayloadFormat.java +++ b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtpPayloadFormat.java @@ -15,7 +15,10 @@ */ package com.google.android.exoplayer2.source.rtsp; +import static com.google.android.exoplayer2.util.Assertions.checkArgument; + import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.util.MimeTypes; import com.google.common.base.Ascii; @@ -40,6 +43,10 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; private static final String RTP_MEDIA_H264 = "H264"; private static final String RTP_MEDIA_H265 = "H265"; + private static final String RTP_MEDIA_PCM_L8 = "L8"; + private static final String RTP_MEDIA_PCM_L16 = "L16"; + private static final String RTP_MEDIA_PCMA = "PCMA"; + private static final String RTP_MEDIA_PCMU = "PCMU"; private static final String RTP_MEDIA_VP8 = "VP8"; /** Returns whether the format of a {@link MediaDescription} is supported. */ @@ -51,6 +58,10 @@ public final class RtpPayloadFormat { case RTP_MEDIA_H264: case RTP_MEDIA_H265: case RTP_MEDIA_MPEG4_GENERIC: + case RTP_MEDIA_PCM_L8: + case RTP_MEDIA_PCM_L16: + case RTP_MEDIA_PCMA: + case RTP_MEDIA_PCMU: case RTP_MEDIA_VP8: return true; default: @@ -79,6 +90,13 @@ public final class RtpPayloadFormat { return MimeTypes.VIDEO_H265; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; + case RTP_MEDIA_PCM_L8: + case RTP_MEDIA_PCM_L16: + return MimeTypes.AUDIO_RAW; + case RTP_MEDIA_PCMA: + return MimeTypes.AUDIO_ALAW; + case RTP_MEDIA_PCMU: + return MimeTypes.AUDIO_MLAW; case RTP_MEDIA_VP8: return MimeTypes.VIDEO_VP8; default: @@ -86,6 +104,15 @@ public final class RtpPayloadFormat { } } + /** Returns the PCM encoding type for {@code mediaEncoding}. */ + public static @C.PcmEncoding int getRawPcmEncodingType(String mediaEncoding) { + checkArgument( + mediaEncoding.equals(RTP_MEDIA_PCM_L8) || mediaEncoding.equals(RTP_MEDIA_PCM_L16)); + return mediaEncoding.equals(RtpPayloadFormat.RTP_MEDIA_PCM_L8) + ? C.ENCODING_PCM_8BIT + : C.ENCODING_PCM_16BIT_BIG_ENDIAN; + } + /** The payload type associated with this format. */ public final int rtpPayloadType; /** The clock rate in Hertz, associated with the format. */ diff --git a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java index 105b9006e8..46072189e3 100644 --- a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java +++ b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java @@ -121,8 +121,9 @@ import com.google.common.collect.ImmutableMap; } int rtpPayloadType = mediaDescription.rtpMapAttribute.payloadType; + String mediaEncoding = mediaDescription.rtpMapAttribute.mediaEncoding; - String mimeType = getMimeTypeFromRtpMediaType(mediaDescription.rtpMapAttribute.mediaEncoding); + String mimeType = getMimeTypeFromRtpMediaType(mediaEncoding); formatBuilder.setSampleMimeType(mimeType); int clockRate = mediaDescription.rtpMapAttribute.clockRate; @@ -166,8 +167,13 @@ import com.google.common.collect.ImmutableMap; // width and height. formatBuilder.setWidth(DEFAULT_VP8_WIDTH).setHeight(DEFAULT_VP8_HEIGHT); break; + case MimeTypes.AUDIO_RAW: + formatBuilder.setPcmEncoding(RtpPayloadFormat.getRawPcmEncodingType(mediaEncoding)); + break; case MimeTypes.AUDIO_AC3: - // AC3 does not require a fmtp attribute. Fall through. + case MimeTypes.AUDIO_ALAW: + case MimeTypes.AUDIO_MLAW: + // Does not require a fmtp attribute. Fall through. default: // Do nothing. } diff --git a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/DefaultRtpPayloadReaderFactory.java index e22530b9e4..5f5ea99bf2 100644 --- a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -37,6 +37,10 @@ import com.google.android.exoplayer2.util.MimeTypes; case MimeTypes.AUDIO_AMR_NB: case MimeTypes.AUDIO_AMR_WB: return new RtpAmrReader(payloadFormat); + case MimeTypes.AUDIO_RAW: + case MimeTypes.AUDIO_ALAW: + case MimeTypes.AUDIO_MLAW: + return new RtpPcmReader(payloadFormat); case MimeTypes.VIDEO_H264: return new RtpH264Reader(payloadFormat); case MimeTypes.VIDEO_H265: diff --git a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/RtpPcmReader.java b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/RtpPcmReader.java new file mode 100644 index 0000000000..beaa575764 --- /dev/null +++ b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/RtpPcmReader.java @@ -0,0 +1,104 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source.rtsp.reader; + +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; + +import android.util.Log; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.extractor.ExtractorOutput; +import com.google.android.exoplayer2.extractor.TrackOutput; +import com.google.android.exoplayer2.source.rtsp.RtpPacket; +import com.google.android.exoplayer2.source.rtsp.RtpPayloadFormat; +import com.google.android.exoplayer2.util.ParsableByteArray; +import com.google.android.exoplayer2.util.Util; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses byte stream carried on RTP packets, and extracts PCM frames. Refer to RFC3551 for more + * details. + */ +/* package */ public final class RtpPcmReader implements RtpPayloadReader { + + private static final String TAG = "RtpPcmReader"; + private final RtpPayloadFormat payloadFormat; + + private @MonotonicNonNull TrackOutput trackOutput; + private long firstReceivedTimestamp; + private long startTimeOffsetUs; + private int previousSequenceNumber; + + public RtpPcmReader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + firstReceivedTimestamp = C.TIME_UNSET; + // Start time offset must be 0 before the first seek. + startTimeOffsetUs = 0; + previousSequenceNumber = C.INDEX_UNSET; + } + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_AUDIO); + trackOutput.format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + firstReceivedTimestamp = timestamp; + } + + @Override + public void consume( + ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) { + checkNotNull(trackOutput); + if (previousSequenceNumber != C.INDEX_UNSET) { + int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); + if (sequenceNumber != expectedSequenceNumber) { + Log.w( + TAG, + Util.formatInvariant( + "Received RTP packet with unexpected sequence number. Expected: %d; received: %d.", + expectedSequenceNumber, sequenceNumber)); + } + } + + long sampleTimeUs = + toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp, payloadFormat.clockRate); + int size = data.bytesLeft(); + trackOutput.sampleData(data, size); + trackOutput.sampleMetadata( + sampleTimeUs, C.BUFFER_FLAG_KEY_FRAME, size, /* offset= */ 0, /* cryptoData= */ null); + + previousSequenceNumber = sequenceNumber; + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + // TODO(b/198620566) Rename firstReceivedTimestamp to timestampBase for all RtpPayloadReaders. + firstReceivedTimestamp = nextRtpTimestamp; + startTimeOffsetUs = timeUs; + } + + /** Returns the correct sample time from RTP timestamp, accounting for the given clock rate. */ + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp, int clockRate) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + rtpTimestamp - firstReceivedRtpTimestamp, + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ clockRate); + } +} diff --git a/library/rtsp/src/test/java/com/google/android/exoplayer2/source/rtsp/reader/RtpPcmReaderTest.java b/library/rtsp/src/test/java/com/google/android/exoplayer2/source/rtsp/reader/RtpPcmReaderTest.java new file mode 100644 index 0000000000..393ff5c9c2 --- /dev/null +++ b/library/rtsp/src/test/java/com/google/android/exoplayer2/source/rtsp/reader/RtpPcmReaderTest.java @@ -0,0 +1,183 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source.rtsp.reader; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.source.rtsp.RtpPacket; +import com.google.android.exoplayer2.source.rtsp.RtpPayloadFormat; +import com.google.android.exoplayer2.testutil.FakeExtractorOutput; +import com.google.android.exoplayer2.testutil.FakeTrackOutput; +import com.google.android.exoplayer2.testutil.TestUtil; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.ParsableByteArray; +import com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link RtpPcmReader}. */ +@RunWith(AndroidJUnit4.class) +public final class RtpPcmReaderTest { + + // A typical RTP payload type for audio. + private static final int RTP_PAYLOAD_TYPE = 97; + private static final byte[] FRAME_1_PAYLOAD = TestUtil.buildTestData(/* length= */ 4); + private static final byte[] FRAME_2_PAYLOAD = TestUtil.buildTestData(/* length= */ 4); + + private static final RtpPacket PACKET_1 = + createRtpPacket(/* timestamp= */ 2599168056L, /* sequenceNumber= */ 40289, FRAME_1_PAYLOAD); + private static final RtpPacket PACKET_2 = + createRtpPacket(/* timestamp= */ 2599169592L, /* sequenceNumber= */ 40290, FRAME_2_PAYLOAD); + + private ParsableByteArray packetData; + private FakeExtractorOutput extractorOutput; + private RtpPcmReader pcmReader; + + @Before + public void setUp() { + packetData = new ParsableByteArray(); + extractorOutput = new FakeExtractorOutput(); + } + + @Test + public void consume_twoDualChannelWav8bitPackets() { + pcmReader = + new RtpPcmReader( + new RtpPayloadFormat( + new Format.Builder() + .setChannelCount(2) + .setSampleMimeType(MimeTypes.AUDIO_WAV) + .setPcmEncoding(C.ENCODING_PCM_8BIT) + .setSampleRate(48_000) + .build(), + /* rtpPayloadType= */ RTP_PAYLOAD_TYPE, + /* clockRate= */ 48_000, + /* fmtpParameters= */ ImmutableMap.of())); + + pcmReader.createTracks(extractorOutput, /* trackId= */ 0); + pcmReader.onReceivingFirstPacket(PACKET_1.timestamp, PACKET_1.sequenceNumber); + consume(PACKET_1); + consume(PACKET_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_PAYLOAD); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_PAYLOAD); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(32000); + } + + @Test + public void consume_twoSingleChannelWav16bitPackets() { + pcmReader = + new RtpPcmReader( + new RtpPayloadFormat( + new Format.Builder() + .setChannelCount(1) + .setSampleMimeType(MimeTypes.AUDIO_WAV) + .setPcmEncoding(C.ENCODING_PCM_16BIT_BIG_ENDIAN) + .setSampleRate(60_000) + .build(), + /* rtpPayloadType= */ RTP_PAYLOAD_TYPE, + /* clockRate= */ 60_000, + /* fmtpParameters= */ ImmutableMap.of())); + + pcmReader.createTracks(extractorOutput, /* trackId= */ 0); + pcmReader.onReceivingFirstPacket(PACKET_1.timestamp, PACKET_1.sequenceNumber); + consume(PACKET_1); + consume(PACKET_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_PAYLOAD); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_PAYLOAD); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(25600); + } + + @Test + public void consume_twoDualChannelAlawPackets() { + pcmReader = + new RtpPcmReader( + new RtpPayloadFormat( + new Format.Builder() + .setChannelCount(2) + .setSampleMimeType(MimeTypes.AUDIO_ALAW) + .setSampleRate(16_000) + .build(), + /* rtpPayloadType= */ RTP_PAYLOAD_TYPE, + /* clockRate= */ 16_000, + /* fmtpParameters= */ ImmutableMap.of())); + + pcmReader.createTracks(extractorOutput, /* trackId= */ 0); + pcmReader.onReceivingFirstPacket(PACKET_1.timestamp, PACKET_1.sequenceNumber); + consume(PACKET_1); + consume(PACKET_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_PAYLOAD); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_PAYLOAD); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(96000); + } + + @Test + public void consume_twoDualChannelMlawPackets() { + pcmReader = + new RtpPcmReader( + new RtpPayloadFormat( + new Format.Builder() + .setChannelCount(2) + .setSampleMimeType(MimeTypes.AUDIO_MLAW) + .setSampleRate(24_000) + .build(), + /* rtpPayloadType= */ RTP_PAYLOAD_TYPE, + /* clockRate= */ 24_000, + /* fmtpParameters= */ ImmutableMap.of())); + + pcmReader.createTracks(extractorOutput, /* trackId= */ 0); + pcmReader.onReceivingFirstPacket(PACKET_1.timestamp, PACKET_1.sequenceNumber); + consume(PACKET_1); + consume(PACKET_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_PAYLOAD); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_PAYLOAD); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(64000); + } + + private static RtpPacket createRtpPacket(long timestamp, int sequenceNumber, byte[] payloadData) { + return new RtpPacket.Builder() + .setTimestamp(timestamp) + .setSequenceNumber(sequenceNumber) + // RFC3551 Section 4.1. + .setMarker(false) + .setPayloadData(payloadData) + .build(); + } + + private void consume(RtpPacket frame) { + packetData.reset(frame.payloadData); + pcmReader.consume(packetData, frame.timestamp, frame.sequenceNumber, frame.marker); + } +}