diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 297353167b..e96006ea71 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -40,6 +40,7 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; private static final String RTP_MEDIA_H264 = "H264"; private static final String RTP_MEDIA_H265 = "H265"; + private static final String RTP_MEDIA_VP9 = "VP9"; /** Returns whether the format of a {@link MediaDescription} is supported. */ public static boolean isFormatSupported(MediaDescription mediaDescription) { @@ -48,6 +49,7 @@ public final class RtpPayloadFormat { case RTP_MEDIA_H264: case RTP_MEDIA_H265: case RTP_MEDIA_MPEG4_GENERIC: + case RTP_MEDIA_VP9: return true; default: return false; @@ -71,6 +73,8 @@ public final class RtpPayloadFormat { return MimeTypes.VIDEO_H265; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; + case RTP_MEDIA_VP9: + return MimeTypes.VIDEO_VP9; default: throw new IllegalArgumentException(mediaType); } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 7547f1ea18..f8aa5c2d8d 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -56,6 +56,10 @@ import com.google.common.collect.ImmutableMap; private static final String GENERIC_CONTROL_ATTR = "*"; + /** Default width and height for VP9. */ + private static final int DEFAULT_VP9_WIDTH = 320; + private static final int DEFAULT_VP9_HEIGHT = 240; + /** The track's associated {@link RtpPayloadFormat}. */ public final RtpPayloadFormat payloadFormat; /** The track's URI. */ @@ -129,6 +133,10 @@ import com.google.common.collect.ImmutableMap; checkArgument(!fmtpParameters.isEmpty()); processH265FmtpAttribute(formatBuilder, fmtpParameters); break; + case MimeTypes.VIDEO_VP9: + // VP9 does not require a FMTP attribute. So Setting default width and height. + formatBuilder.setWidth(DEFAULT_VP9_WIDTH).setHeight(DEFAULT_VP9_HEIGHT); + break; case MimeTypes.AUDIO_AC3: // AC3 does not require a FMTP attribute. Fall through. default: diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 888939b7e8..277e717f02 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -40,6 +40,8 @@ import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; return new RtpH264Reader(payloadFormat); case MimeTypes.VIDEO_H265: return new RtpH265Reader(payloadFormat); + case MimeTypes.VIDEO_VP9: + return new RtpVP9Reader(payloadFormat); default: // No supported reader, returning null. } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP9Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP9Reader.java new file mode 100644 index 0000000000..3280c0b413 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP9Reader.java @@ -0,0 +1,230 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.castNonNull; + +import androidx.media3.common.C; +import androidx.media3.common.ParserException; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.TrackOutput; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses an VP9 byte stream carried on RTP packets, and extracts VP9 Access Units. Refer to + * @link https://datatracker.ietf.org/doc/html/draft-ietf-payload-vp9 for more details. + */ +/* package */ final class RtpVP9Reader implements RtpPayloadReader { + + private static final String TAG = "RtpVP9Reader"; + + private static final long MEDIA_CLOCK_FREQUENCY = 90_000; + + private final RtpPayloadFormat payloadFormat; + + private @MonotonicNonNull TrackOutput trackOutput; + @C.BufferFlags private int bufferFlags; + + private long firstReceivedTimestamp; + private long startTimeOffsetUs; + private static int previousSequenceNumber; + /** The combined size of a sample that is fragmented into multiple RTP packets. */ + private int fragmentedSampleSizeBytes; + private static int width; + private static int height; + private static boolean gotFirstPacketOfVP9Frame; + private boolean isKeyFrame; + private boolean isOutputFormatSet; + + /** Creates an instance. */ + public RtpVP9Reader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + firstReceivedTimestamp = C.TIME_UNSET; + startTimeOffsetUs = 0; + previousSequenceNumber = C.INDEX_UNSET; + fragmentedSampleSizeBytes = 0; + width = C.INDEX_UNSET; + height = C.INDEX_UNSET; + gotFirstPacketOfVP9Frame = false; + isKeyFrame = false; + isOutputFormatSet = false; + } + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); + castNonNull(trackOutput).format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} + + @Override + public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) + throws ParserException { + checkStateNotNull(trackOutput); + + if (parseVP9Descriptor(data, sequenceNumber)) { + if (fragmentedSampleSizeBytes == 0 && gotFirstPacketOfVP9Frame) { + isKeyFrame = (data.peekUnsignedByte() & 0x04) == 0; + } + + if (!isOutputFormatSet && width > 0 && height > 0) { + if (width != payloadFormat.format.width || height != payloadFormat.format.height) { + trackOutput.format( + payloadFormat.format.buildUpon().setWidth(width).setHeight(height).build()); + } + isOutputFormatSet = true; + } + + int fragmentSize = data.bytesLeft(); + // Write the video sample. + trackOutput.sampleData(data, fragmentSize); + fragmentedSampleSizeBytes += fragmentSize; + + if (rtpMarker) { + if (firstReceivedTimestamp == C.TIME_UNSET) { + firstReceivedTimestamp = timestamp; + } + bufferFlags = isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0; + long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); + trackOutput.sampleMetadata( + timeUs, + bufferFlags, + fragmentedSampleSizeBytes, + /* offset= */ 0, + /* encryptionData= */ null); + fragmentedSampleSizeBytes = 0; + gotFirstPacketOfVP9Frame = false; + } + previousSequenceNumber = sequenceNumber; + } + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + fragmentedSampleSizeBytes = 0; + startTimeOffsetUs = timeUs; + } + + // Internal methods. + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + (rtpTimestamp - firstReceivedRtpTimestamp), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } + + private static boolean parseVP9Descriptor(ParsableByteArray payload, int packetSequenceNumber) + throws ParserException { + // VP9 Payload Descriptor, Section 4.2 + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // |I|P|L|F|B|E|V|Z| (REQUIRED) + // +-+-+-+-+-+-+-+-+ + // I: |M| PICTURE ID | (RECOMMENDED) + // +-+-+-+-+-+-+-+-+ + // M: | EXTENDED PID | (RECOMMENDED) + // +-+-+-+-+-+-+-+-+ + // L: | TID |U| SID |D| (Conditionally RECOMMENDED) + // +-+-+-+-+-+-+-+-+ + // | TL0PICIDX | (Conditionally REQUIRED) + // +-+-+-+-+-+-+-+-+ + // V: | SS | + // | .. | + // +-+-+-+-+-+-+-+-+ + + int header = payload.readUnsignedByte(); + if (!gotFirstPacketOfVP9Frame) { + // For start of VP9 partition B=1 as per VP9 RFC Section 4.2. + if ((header & 0x08) == 0) { + Log.w( + TAG, + Util.formatInvariant( + "First payload octet of the RTP packet is not the beginning of a new VP9 partition," + + " Dropping current packet." + header)); + return false; + } + gotFirstPacketOfVP9Frame = true; + } else { + // Check that this packet is in the sequence of the previous packet. + int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); + if (packetSequenceNumber != expectedSequenceNumber) { + Log.w( + TAG, + Util.formatInvariant( + "Received RTP packet with unexpected sequence number. Expected: %d; received: %d." + + " Dropping packet.", + expectedSequenceNumber, packetSequenceNumber)); + return false; + } + } + + // Check I optional header present. + if ((header & 0x80) != 0) { + int optionalHeader = payload.readUnsignedByte(); + // Check M for 15 bits PictureID. + if ((optionalHeader & 0x80) != 0) { + if (payload.bytesLeft() < 1) { + return false; + } + } + } + + // Flexible-mode not implemented. + checkArgument((header & 0x10) == 0, "VP9 flexible mode unsupported"); + + // Check L optional header present. + if ((header & 0x20) != 0) { + payload.skipBytes(1); + if (payload.bytesLeft() < 1) { + return false; + } + // Check TL0PICIDX header present (non-flexible mode). + if ((header & 0x10) == 0) { + payload.skipBytes(1); + } + } + + // Check V optional header present, Refer Section 4.2.1. + if ((header & 0x02) != 0) { + int scalabilityStr = payload.readUnsignedByte(); + int numSpatialLayers = (scalabilityStr & 0xe0) >> 5; + int scalabilityStrLength = ((scalabilityStr & 0x10) != 0) ? numSpatialLayers + 1 : 0; + + if ((scalabilityStr & 0x10) != 0) { + if (payload.bytesLeft() < scalabilityStrLength * 4) { + return false; + } + for (int index = 0; index < scalabilityStrLength; index++) { + width = payload.readUnsignedShort(); + height = payload.readUnsignedShort(); + } + } + } + return true; + } +}