loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #53 from ittiam-systems:rtp_opus

Browse Source 
PiperOrigin-RevId: 453490088
(cherry picked from commit a2a450432904c3d8ecc25e00c7148e73f1923d6e)
This commit is contained in:
Marc Baechinger 2022-06-09 17:37:09 +00:00
parent 2ed7efb1c3
commit fd1eb4b466
6 changed files with 374 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -127,6 +127,8 @@
([#10165](https://github.com/google/ExoPlayer/issues/10165)).
* Add RTP reader for VP9
([#47](https://github.com/androidx/media/pull/64)).
* Add RTP reader for OPUS
([#53](https://github.com/androidx/media/pull/53)).
* Session:
* Fix NPE in MediaControllerImplLegacy
([#59](https://github.com/androidx/media/pull/59)).

4
libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java
View File

@ -46,6 +46,7 @@ public final class RtpPayloadFormat {
private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES";
private static final String RTP_MEDIA_H264 = "H264";
private static final String RTP_MEDIA_H265 = "H265";
private static final String RTP_MEDIA_OPUS = "OPUS";
private static final String RTP_MEDIA_PCM_L8 = "L8";
private static final String RTP_MEDIA_PCM_L16 = "L16";
private static final String RTP_MEDIA_PCMA = "PCMA";
@ -63,6 +64,7 @@ public final class RtpPayloadFormat {
case RTP_MEDIA_H265:
case RTP_MEDIA_MPEG4_VIDEO:
case RTP_MEDIA_MPEG4_GENERIC:
case RTP_MEDIA_OPUS:
case RTP_MEDIA_PCM_L8:
case RTP_MEDIA_PCM_L16:
case RTP_MEDIA_PCMA:
@ -92,6 +94,8 @@ public final class RtpPayloadFormat {
return MimeTypes.AUDIO_AMR_WB;
case RTP_MEDIA_MPEG4_GENERIC:
return MimeTypes.AUDIO_AAC;
case RTP_MEDIA_OPUS:
return MimeTypes.AUDIO_OPUS;
case RTP_MEDIA_PCM_L8:
case RTP_MEDIA_PCM_L16:
return MimeTypes.AUDIO_RAW;

9
libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java
View File

@ -101,6 +101,9 @@ import com.google.common.collect.ImmutableMap;
*/
private static final int DEFAULT_VP8_HEIGHT = 240;
/** RFC7587 Section 6.1 Sampling rate for OPUS is fixed at 48KHz. */
private static final int OPUS_CLOCK_RATE = 48_000;
/**
* Default width for VP9.
*
@ -201,6 +204,12 @@ import com.google.common.collect.ImmutableMap;
!fmtpParameters.containsKey(PARAMETER_AMR_INTERLEAVING),
"Interleaving mode is not currently supported.");
break;
case MimeTypes.AUDIO_OPUS:
checkArgument(channelCount != C.INDEX_UNSET);
// RFC7587 Section 6.1: the RTP timestamp is incremented with a 48000 Hz clock rate
// for all modes of Opus and all sampling rates.
checkArgument(clockRate == OPUS_CLOCK_RATE, "Invalid OPUS clock rate.");
break;
case MimeTypes.VIDEO_MP4V:
checkArgument(!fmtpParameters.isEmpty());
processMPEG4FmtpAttribute(formatBuilder, fmtpParameters);

2
libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java
View File

@ -39,6 +39,8 @@ import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
case MimeTypes.AUDIO_AMR_NB:
case MimeTypes.AUDIO_AMR_WB:
return new RtpAmrReader(payloadFormat);
case MimeTypes.AUDIO_OPUS:
return new RtpOpusReader(payloadFormat);
case MimeTypes.AUDIO_RAW:
case MimeTypes.AUDIO_ALAW:
case MimeTypes.AUDIO_MLAW:

157
libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReader.java Normal file
View File

@ -0,0 +1,157 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.rtsp.reader;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.rtsp.RtpPacket;
import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.OpusUtil;
import androidx.media3.extractor.TrackOutput;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Parses an OPUS byte stream carried on RTP packets and extracts individual samples. Refer to
* RFC7845 for more details.
*/
/* package */ final class RtpOpusReader implements RtpPayloadReader {
private static final String TAG = "RtpOpusReader";
/* Opus uses a fixed 48KHz media clock RFC7845 Section 4. */
private static final long MEDIA_CLOCK_FREQUENCY = 48_000;
private final RtpPayloadFormat payloadFormat;
private @MonotonicNonNull TrackOutput trackOutput;
/**
* First received RTP timestamp. All RTP timestamps are dimension-less, the time base is defined
* by {@link #MEDIA_CLOCK_FREQUENCY}.
*/
private long firstReceivedTimestamp;
private long startTimeOffsetUs;
private int previousSequenceNumber;
private boolean foundOpusIDHeader;
private boolean foundOpusCommentHeader;
/** Creates an instance. */
public RtpOpusReader(RtpPayloadFormat payloadFormat) {
this.payloadFormat = payloadFormat;
this.firstReceivedTimestamp = C.INDEX_UNSET;
this.previousSequenceNumber = C.INDEX_UNSET;
}
// RtpPayloadReader implementation.
@Override
public void createTracks(ExtractorOutput extractorOutput, int trackId) {
trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_AUDIO);
trackOutput.format(payloadFormat.format);
}
@Override
public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {
this.firstReceivedTimestamp = timestamp;
}
@Override
public void consume(
ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) {
checkStateNotNull(trackOutput);
/* RFC7845 Section 3.
* +---------+ +----------------+ +--------------------+ +-----
* |ID Header| | Comment Header | |Audio Data Packet 1 | | ...
* +---------+ +----------------+ +--------------------+ +-----
*/
if (!foundOpusIDHeader) {
validateOpusIdHeader(data);
List<byte[]> initializationData = OpusUtil.buildInitializationData(data.getData());
Format.Builder formatBuilder = payloadFormat.format.buildUpon();
formatBuilder.setInitializationData(initializationData);
trackOutput.format(formatBuilder.build());
foundOpusIDHeader = true;
} else if (!foundOpusCommentHeader) {
// Comment Header RFC7845 Section 5.2.
int sampleSize = data.limit();
checkArgument(sampleSize >= 8, "Comment Header has insufficient data");
String header = data.readString(8);
checkArgument(header.equals("OpusTags"), "Comment Header should follow ID Header");
foundOpusCommentHeader = true;
} else {
// Check that this packet is in the sequence of the previous packet.
int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber);
if (sequenceNumber != expectedSequenceNumber) {
Log.w(
TAG,
Util.formatInvariant(
"Received RTP packet with unexpected sequence number. Expected: %d; received: %d.",
expectedSequenceNumber, sequenceNumber));
}
// sending opus data.
int size = data.bytesLeft();
trackOutput.sampleData(data, size);
long timeUs = toSampleTimeUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
trackOutput.sampleMetadata(
timeUs, C.BUFFER_FLAG_KEY_FRAME, size, /* offset*/ 0, /* cryptoData*/ null);
}
previousSequenceNumber = sequenceNumber;
}
@Override
public void seek(long nextRtpTimestamp, long timeUs) {
firstReceivedTimestamp = nextRtpTimestamp;
startTimeOffsetUs = timeUs;
}
// Internal methods.
/**
* Validates the OPUS ID Header at {@code data}'s current position, throws {@link
* IllegalArgumentException} if the header is invalid.
*
* <p>{@code data}'s position does not change after returning.
*/
private static void validateOpusIdHeader(ParsableByteArray data) {
int currPosition = data.getPosition();
int sampleSize = data.limit();
checkArgument(sampleSize > 18, "ID Header has insufficient data");
String header = data.readString(8);
// Identification header RFC7845 Section 5.1.
checkArgument(header.equals("OpusHead"), "ID Header missing");
checkArgument(data.readUnsignedByte() == 1, "version number must always be 1");
data.setPosition(currPosition);
}
/** Returns the correct sample time from RTP timestamp, accounting for the OPUS sampling rate. */
private static long toSampleTimeUs(
long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) {
return startTimeOffsetUs
+ Util.scaleLargeTimestamp(
rtpTimestamp - firstReceivedRtpTimestamp,
/* multiplier= */ C.MICROS_PER_SECOND,
/* divisor= */ MEDIA_CLOCK_FREQUENCY);
}
}

200
libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java Normal file
View File

@ -0,0 +1,200 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.rtsp.reader;
import static androidx.media3.common.util.Util.getBytesFromHexString;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.Mockito.when;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.exoplayer.rtsp.RtpPacket;
import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.test.utils.FakeTrackOutput;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableMap;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;
/** Unit test for {@link RtpOpusReader}. */
@RunWith(AndroidJUnit4.class)
public final class RtpOpusReaderTest {
private static final RtpPayloadFormat OPUS_FORMAT =
new RtpPayloadFormat(
new Format.Builder()
.setChannelCount(6)
.setSampleMimeType(MimeTypes.AUDIO_OPUS)
.setSampleRate(48_000)
.build(),
/* rtpPayloadType= */ 97,
/* clockRate= */ 48_000,
/* fmtpParameters= */ ImmutableMap.of());
private static final RtpPacket OPUS_HEADER =
createRtpPacket(
/* timestamp= */ 2599168056L,
/* sequenceNumber= */ 40289,
/* payloadData= */ getBytesFromHexString("4F707573486561640102000000000000000000"));
private static final RtpPacket OPUS_TAGS =
createRtpPacket(
/* timestamp= */ 2599168056L,
/* sequenceNumber= */ 40290,
/* payloadData= */ getBytesFromHexString("4F707573546167730000000000000000000000"));
private static final RtpPacket OPUS_FRAME_1 =
createRtpPacket(
/* timestamp= */ 2599168056L,
/* sequenceNumber= */ 40292,
/* payloadData= */ getBytesFromHexString("010203"));
private static final RtpPacket OPUS_FRAME_2 =
createRtpPacket(
/* timestamp= */ 2599169592L,
/* sequenceNumber= */ 40293,
/* payloadData= */ getBytesFromHexString("04050607"));
@Rule public final MockitoRule mockito = MockitoJUnit.rule();
private ParsableByteArray packetData;
private RtpOpusReader opusReader;
private FakeTrackOutput trackOutput;
@Mock private ExtractorOutput extractorOutput;
@Before
public void setUp() {
packetData = new ParsableByteArray();
trackOutput = new FakeTrackOutput(/* deduplicateConsecutiveFormats= */ true);
when(extractorOutput.track(anyInt(), anyInt())).thenReturn(trackOutput);
opusReader = new RtpOpusReader(OPUS_FORMAT);
opusReader.createTracks(extractorOutput, /* trackId= */ 0);
}
@Test
public void consume_validPackets() {
opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
consume(OPUS_HEADER);
consume(OPUS_TAGS);
consume(OPUS_FRAME_1);
consume(OPUS_FRAME_2);
assertThat(trackOutput.getSampleCount()).isEqualTo(2);
assertThat(trackOutput.getSampleData(0)).isEqualTo(getBytesFromHexString("010203"));
assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0);
assertThat(trackOutput.getSampleData(1)).isEqualTo(getBytesFromHexString("04050607"));
assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(32000);
}
@Test
public void consume_opusHeaderWithInvalidHeader_throwsIllegalArgumentException() {
opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
assertThrows(
IllegalArgumentException.class,
() ->
consume(
createRtpPacket(
/* timestamp= */ 2599168056L,
/* sequenceNumber= */ 40289,
// Modify "OpusHead" -> "OrusHead" (First 8 bytes).
/* payloadData= */ getBytesFromHexString(
"4F727573486561640102000000000000000000"))));
}
@Test
public void consume_opusHeaderWithInvalidSampleSize_throwsIllegalArgumentException() {
opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
assertThrows(
IllegalArgumentException.class,
() ->
consume(
createRtpPacket(
/* timestamp= */ 2599168056L,
/* sequenceNumber= */ 40289,
// Truncate the opusHeader payload data.
/* payloadData= */ getBytesFromHexString("4F707573486561640102"))));
}
@Test
public void consume_opusHeaderWithInvalidVersionNumber_throwsIllegalArgumentException() {
opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
assertThrows(
IllegalArgumentException.class,
() ->
consume(
createRtpPacket(
/* timestamp= */ 2599168056L,
/* sequenceNumber= */ 40289,
// Modify version 1 -> 2 (9th byte)
/* payloadData= */ getBytesFromHexString(
"4f707573486561640202000000000000000000"))));
}
@Test
public void consume_invalidOpusTags_throwsIllegalArgumentException() {
opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
consume(OPUS_HEADER);
assertThrows(
IllegalArgumentException.class,
() ->
consume(
createRtpPacket(
/* timestamp= */ 2599168056L,
/* sequenceNumber= */ 40290,
// Modify "OpusTags" -> "OpusTggs" (First 8 bytes)
/* payloadData= */ getBytesFromHexString("4F70757354676773"))));
}
@Test
public void consume_skipOpusTags_throwsIllegalArgumentException() {
opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
consume(OPUS_HEADER);
assertThrows(IllegalArgumentException.class, () -> consume(OPUS_FRAME_1));
}
@Test
public void consume_skipOpusHeader_throwsIllegalArgumentException() {
opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
assertThrows(IllegalArgumentException.class, () -> consume(OPUS_TAGS));
}
@Test
public void consume_skipOpusHeaderAndOpusTags_throwsIllegalArgumentException() {
opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber);
assertThrows(IllegalArgumentException.class, () -> consume(OPUS_FRAME_1));
}
private static RtpPacket createRtpPacket(long timestamp, int sequenceNumber, byte[] payloadData) {
return new RtpPacket.Builder()
.setTimestamp(timestamp)
.setSequenceNumber(sequenceNumber)
.setMarker(false)
.setPayloadData(payloadData)
.build();
}
private void consume(RtpPacket rtpPacket) {
packetData.reset(rtpPacket.payloadData);
opusReader.consume(packetData, rtpPacket.timestamp, rtpPacket.sequenceNumber, rtpPacket.marker);
}
}