From ace363e1839c010755cefbf5e133367dd6da4370 Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Tue, 8 Feb 2022 16:41:45 +0530 Subject: [PATCH] Fix review comments in RtpH265Reader --- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 56 +++++------ .../exoplayer/rtsp/reader/RtpH265Reader.java | 97 ++++++++----------- 2 files changed, 72 insertions(+), 81 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index d7bb6d432a..afa3327efb 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -169,6 +169,26 @@ import com.google.common.collect.ImmutableMap; AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount))); } + /** Returns H264/H265 initialization data from RTP parameter set. */ + private static byte[] getInitializationDataFromParameterSet(String parameterSet) { + byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT); + byte[] decodedParameterNalUnit = + new byte[decodedParameterNalData.length + NAL_START_CODE.length]; + System.arraycopy( + NAL_START_CODE, + /* srcPos= */ 0, + decodedParameterNalUnit, + /* destPos= */ 0, + NAL_START_CODE.length); + System.arraycopy( + decodedParameterNalData, + /* srcPos= */ 0, + decodedParameterNalUnit, + /* destPos= */ NAL_START_CODE.length, + decodedParameterNalData.length); + return decodedParameterNalUnit; + } + private static void processH264FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS)); @@ -200,25 +220,6 @@ import com.google.common.collect.ImmutableMap; } } - private static byte[] getInitializationDataFromParameterSet(String parameterSet) { - byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT); - byte[] decodedParameterNalUnit = - new byte[decodedParameterNalData.length + NAL_START_CODE.length]; - System.arraycopy( - NAL_START_CODE, - /* srcPos= */ 0, - decodedParameterNalUnit, - /* destPos= */ 0, - NAL_START_CODE.length); - System.arraycopy( - decodedParameterNalData, - /* srcPos= */ 0, - decodedParameterNalUnit, - /* destPos= */ NAL_START_CODE.length, - decodedParameterNalData.length); - return decodedParameterNalUnit; - } - private static void processH265FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { if (fmtpAttributes.containsKey(PARAMETER_SPROP_H265_MAX_DON_DIFF)) { @@ -228,23 +229,24 @@ import com.google.common.collect.ImmutableMap; "non-zero sprop-max-don-diff is not supported"); } + checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_VPS)); + String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_VPS)); checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_SPS)); String spropSPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_SPS)); checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_PPS)); String spropPPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_PPS)); - checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_VPS)); - String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_VPS)); - - byte[] vpsNalData = getInitializationDataFromParameterSet(spropVPS); - byte[] spsNalData = getInitializationDataFromParameterSet(spropSPS); - byte[] ppsNalData = getInitializationDataFromParameterSet(spropPPS); - ImmutableList initializationData = ImmutableList.of(vpsNalData, spsNalData, ppsNalData); + ImmutableList initializationData = + ImmutableList.of( + getInitializationDataFromParameterSet(spropVPS), + getInitializationDataFromParameterSet(spropSPS), + getInitializationDataFromParameterSet(spropPPS)); formatBuilder.setInitializationData(initializationData); // Process SPS (Sequence Parameter Set). + byte[] spsNalDataWithStartCode = initializationData.get(1); NalUnitUtil.H265SpsData spsData = NalUnitUtil.parseH265SpsNalUnit( - spsNalData, NAL_START_CODE.length, spsNalData.length); + spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length); formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio); formatBuilder.setHeight(spsData.height); formatBuilder.setWidth(spsData.width); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java index 56ccd23382..2bc1ee2eae 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java @@ -42,17 +42,28 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private static final long MEDIA_CLOCK_FREQUENCY = 90_000; - /** Offset of payload data within a FU type A payload. */ + /** Offset of payload data within a FU payload. */ private static final int FU_PAYLOAD_OFFSET = 3; - /** Single Time Aggregation Packet type A. */ - private static final int RTP_PACKET_TYPE_STAP_A = 48; // RFC7798 Section 4.4.2 - /** Fragmentation Unit type A. */ - private static final int RTP_PACKET_TYPE_FU_A = 49; + /** + * Aggregation Packet. + * + * @see + * RFC7798 Section 4.4.2 + */ + private static final int RTP_PACKET_TYPE_AP = 48; + /** + * Fragmentation Unit. + * + * @see + * RFC7798 Section 4.4.3 + */ + private static final int RTP_PACKET_TYPE_FU = 49; /** IDR NAL unit type. */ - private static final int NAL_IDR_W_LP = 19; - + private static final int NAL_IDR_W_RADL = 19; private static final int NAL_IDR_N_LP = 20; /** Scratch for Fragmentation Unit RTP packets. */ @@ -65,12 +76,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private @MonotonicNonNull TrackOutput trackOutput; @C.BufferFlags private int bufferFlags; - private long firstReceivedTimestamp; private int previousSequenceNumber; /** The combined size of a sample that is fragmented into multiple RTP packets. */ private int fragmentedSampleSizeBytes; - private long startTimeOffsetUs; /** Creates an instance. */ @@ -84,7 +93,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override public void createTracks(ExtractorOutput extractorOutput, int trackId) { trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); - castNonNull(trackOutput).format(payloadFormat.format); } @@ -94,7 +102,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) throws ParserException { - int payloadType; try { // RFC7798 Section 1.1.4. NAL Unit Header @@ -104,11 +111,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } checkStateNotNull(trackOutput); - if (payloadType >= 0 && payloadType < RTP_PACKET_TYPE_STAP_A) { + if (payloadType >= 0 && payloadType < RTP_PACKET_TYPE_AP) { processSingleNalUnitPacket(data); - } else if (payloadType == RTP_PACKET_TYPE_STAP_A) { - processSingleTimeAggregationPacket(data); - } else if (payloadType == RTP_PACKET_TYPE_FU_A) { + } else if (payloadType == RTP_PACKET_TYPE_AP) { + processAggregationPacket(data); + } else if (payloadType == RTP_PACKET_TYPE_FU) { processFragmentationUnitPacket(data, sequenceNumber); } else { throw ParserException.createForMalformedManifest( @@ -173,46 +180,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; bufferFlags = getBufferFlagsFromNalType(nalHeaderType); } - /** - * Processes STAP Type A packet (RFC7798 Section 4.4.2). + * Processes an AP packet (RFC7798 Section 4.4.2). * *

Outputs the received aggregation units (with start code prepended) to {@link #trackOutput}. * Sets {@link #bufferFlags} and {@link #fragmentedSampleSizeBytes} accordingly. */ @RequiresNonNull("trackOutput") - private void processSingleTimeAggregationPacket(ParsableByteArray data) throws ParserException { - // An Example of an AP Packet Containing Two Aggregation - // Units without the DONL and DOND Fields. - // 0 1 2 3 - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | RTP Header | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | PayloadHdr (Type=48) | NALU 1 Size | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | NALU 1 HDR | | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- NALU 1 Data | - // | | - // | | - // + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | | NALU 2 Size | NALU 2 HDR | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | NALU 2 HDR | | - // +-+-+-+-+-+-+-+- NALU 2 Data | - // | | - // | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | :...OPTIONAL RTP padding | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - + private void processAggregationPacket(ParsableByteArray data) throws ParserException { throw ParserException.createForMalformedManifest( - "need to implement processSingleTimeAggregationPacket", + "need to implement processAggregationPacket", /* cause= */ null); - } /** - * Processes Fragmentation Unit Type A packet (RFC7798 Section 4.4.3). + * Processes Fragmentation Unit packet (RFC7798 Section 4.4.3). * *

This method will be invoked multiple times to receive a single frame that is broken down * into a series of fragmentation units in multiple RTP packets. @@ -241,15 +223,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; // +-+-+-+-+-+-+-+-+ // |S|E| FuType | // +---------------+ + // Structure of HEVC NAL unit header + // +---------------+---------------+ + // |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // |F| Type | LayerId | TID | + // +-------------+-----------------+ - int tid = (data.getData()[1] & 0x7); // last 3 bits in byte 1 of payload header section 1.1.4 + int tid = + (data.getData()[1] & 0x7); // last 3 bits in byte 1 of payload header, RFC7798 Section 1.1.4 int fuHeader = data.getData()[2]; int nalUnitType = fuHeader & 0x3F; byte nalHeader[] = new byte[2]; - nalHeader[0] = (byte) (nalUnitType << 1); // Section: 1.1.4 - // According to section 1.1.4 in rfc7798, layerId is required to be zero so keeping its value - // zero and copying only tid. + nalHeader[0] = (byte) (nalUnitType << 1); // RFC7798 Section 1.1.4 + // layerId must be zero according to RFC7798 Section 1.1.4, so copying the tid only nalHeader[1] = (byte) tid; boolean isFirstFuPacket = (fuHeader & 0x80) > 0; boolean isLastFuPacket = (fuHeader & 0x40) > 0; @@ -258,12 +246,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; // Prepends starter code. fragmentedSampleSizeBytes += writeStartCode(); - // The bytes needed is 2 (NALU header) + payload size. The original data array has size 3 - // (2 payload + 1 FU header) + payload size. Thus setting the correct header and set position - // to 1. - // Overwrite byte 1 of payload header with byte 0 of HEVC nal header + // Overwrite a few bytes in Rtp buffer to get HEVC NAL unit + // Rtp Byte 0 -> Ignore + // Rtp Byte 1 -> Byte 0 of HEVC NAL header + // Rtp Byte 2 -> Byte 1 of HEVC NAL header + // Rtp Payload -> HEVC NAL bytes, so leave them unchanged + // Set data position from byte 1 as byte 0 was ignored data.getData()[1] = (byte) nalHeader[0]; - // Overwrite byte FU Header with byte 1 of HEVC nal header data.getData()[2] = (byte) nalHeader[1]; fuScratchBuffer.reset(data.getData()); fuScratchBuffer.setPosition(1); @@ -312,6 +301,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @C.BufferFlags private static int getBufferFlagsFromNalType(int nalType) { - return (nalType == NAL_IDR_W_LP || nalType == NAL_IDR_N_LP) ? C.BUFFER_FLAG_KEY_FRAME : 0; + return (nalType == NAL_IDR_W_RADL || nalType == NAL_IDR_N_LP) ? C.BUFFER_FLAG_KEY_FRAME : 0; } }