loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #35 from ittiam-systems:rtp-mpeg4

Browse Source 
PiperOrigin-RevId: 438000682
This commit is contained in:
Ian Baker 2022-04-06 11:19:08 +01:00
commit f48babb4ed
6 changed files with 298 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -36,6 +36,8 @@
dependency from the UI module to the ExoPlayer module. This is a
breaking change.
* RTSP:
* Add RTP reader for MPEG4
([#35](https://github.com/androidx/media/pull/35))
* Add RTP reader for HEVC
([#36](https://github.com/androidx/media/pull/36)).
* Add RTP reader for AMR. Currently only mono-channel, non-interleaved

89
libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java
View File

@ -15,6 +15,8 @@
*/
package androidx.media3.common.util;
import static androidx.media3.common.util.Assertions.checkArgument;
import android.util.Pair;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
@ -31,6 +33,12 @@ public final class CodecSpecificDataUtil {
private static final String[] HEVC_GENERAL_PROFILE_SPACE_STRINGS =
new String[] {"", "A", "B", "C"};
// MP4V-ES
private static final int VISUAL_OBJECT_LAYER = 1;
private static final int VISUAL_OBJECT_LAYER_START = 0x20;
private static final int EXTENDED_PAR = 0x0F;
private static final int RECTANGULAR = 0x00;
/**
* Parses an ALAC AudioSpecificConfig (i.e. an <a
* href="https://github.com/macosforge/alac/blob/master/ALACMagicCookieDescription.txt">ALACSpecificConfig</a>).
@ -72,6 +80,87 @@ public final class CodecSpecificDataUtil {
&& initializationData.get(0)[0] == 1;
}
/**
* Parses an MPEG-4 Visual configuration information, as defined in ISO/IEC14496-2.
*
* @param videoSpecificConfig A byte array containing the MPEG-4 Visual configuration information
* to parse.
* @return A pair of the video's width and height.
*/
public static Pair<Integer, Integer> getVideoResolutionFromMpeg4VideoConfig(
byte[] videoSpecificConfig) {
int offset = 0;
boolean foundVOL = false;
ParsableByteArray scratchBytes = new ParsableByteArray(videoSpecificConfig);
while (offset + 3 < videoSpecificConfig.length) {
if (scratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER
|| (videoSpecificConfig[offset + 3] & 0xF0) != VISUAL_OBJECT_LAYER_START) {
scratchBytes.setPosition(scratchBytes.getPosition() - 2);
offset++;
continue;
}
foundVOL = true;
break;
}
checkArgument(foundVOL, "Invalid input: VOL not found.");
ParsableBitArray scratchBits = new ParsableBitArray(videoSpecificConfig);
// Skip the start codecs from the bitstream
scratchBits.skipBits((offset + 4) * 8);
scratchBits.skipBits(1); // random_accessible_vol
scratchBits.skipBits(8); // video_object_type_indication
if (scratchBits.readBit()) { // object_layer_identifier
scratchBits.skipBits(4); // video_object_layer_verid
scratchBits.skipBits(3); // video_object_layer_priority
}
int aspectRatioInfo = scratchBits.readBits(4);
if (aspectRatioInfo == EXTENDED_PAR) {
scratchBits.skipBits(8); // par_width
scratchBits.skipBits(8); // par_height
}
if (scratchBits.readBit()) { // vol_control_parameters
scratchBits.skipBits(2); // chroma_format
scratchBits.skipBits(1); // low_delay
if (scratchBits.readBit()) { // vbv_parameters
scratchBits.skipBits(79);
}
}
int videoObjectLayerShape = scratchBits.readBits(2);
checkArgument(
videoObjectLayerShape == RECTANGULAR,
"Only supports rectangular video object layer shape.");
checkArgument(scratchBits.readBit()); // marker_bit
int vopTimeIncrementResolution = scratchBits.readBits(16);
checkArgument(scratchBits.readBit()); // marker_bit
if (scratchBits.readBit()) { // fixed_vop_rate
checkArgument(vopTimeIncrementResolution > 0);
vopTimeIncrementResolution--;
int numBitsToSkip = 0;
while (vopTimeIncrementResolution > 0) {
numBitsToSkip++;
vopTimeIncrementResolution >>= 1;
}
scratchBits.skipBits(numBitsToSkip); // fixed_vop_time_increment
}
checkArgument(scratchBits.readBit()); // marker_bit
int videoObjectLayerWidth = scratchBits.readBits(13);
checkArgument(scratchBits.readBit()); // marker_bit
int videoObjectLayerHeight = scratchBits.readBits(13);
checkArgument(scratchBits.readBit()); // marker_bit
scratchBits.skipBits(1); // interlaced
return Pair.create(videoObjectLayerWidth, videoObjectLayerHeight);
}
/**
* Builds an RFC 6381 AVC codec string using the provided parameters.
*

12
libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java
View File

@ -43,6 +43,7 @@ public final class RtpPayloadFormat {
private static final String RTP_MEDIA_AMR = "AMR";
private static final String RTP_MEDIA_AMR_WB = "AMR-WB";
private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC";
private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES";
private static final String RTP_MEDIA_H264 = "H264";
private static final String RTP_MEDIA_H265 = "H265";
private static final String RTP_MEDIA_PCM_L8 = "L8";
@ -59,6 +60,7 @@ public final class RtpPayloadFormat {
case RTP_MEDIA_AMR_WB:
case RTP_MEDIA_H264:
case RTP_MEDIA_H265:
case RTP_MEDIA_MPEG4_VIDEO:
case RTP_MEDIA_MPEG4_GENERIC:
case RTP_MEDIA_PCM_L8:
case RTP_MEDIA_PCM_L16:
@ -86,10 +88,6 @@ public final class RtpPayloadFormat {
return MimeTypes.AUDIO_AMR_NB;
case RTP_MEDIA_AMR_WB:
return MimeTypes.AUDIO_AMR_WB;
case RTP_MEDIA_H264:
return MimeTypes.VIDEO_H264;
case RTP_MEDIA_H265:
return MimeTypes.VIDEO_H265;
case RTP_MEDIA_MPEG4_GENERIC:
return MimeTypes.AUDIO_AAC;
case RTP_MEDIA_PCM_L8:
@ -99,6 +97,12 @@ public final class RtpPayloadFormat {
return MimeTypes.AUDIO_ALAW;
case RTP_MEDIA_PCMU:
return MimeTypes.AUDIO_MLAW;
case RTP_MEDIA_H264:
return MimeTypes.VIDEO_H264;
case RTP_MEDIA_H265:
return MimeTypes.VIDEO_H265;
case RTP_MEDIA_MPEG4_VIDEO:
return MimeTypes.VIDEO_MP4V;
case RTP_MEDIA_VP8:
return MimeTypes.VIDEO_VP8;
default:

49
libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java
View File

@ -25,6 +25,7 @@ import static androidx.media3.extractor.NalUnitUtil.NAL_START_CODE;
import android.net.Uri;
import android.util.Base64;
import android.util.Pair;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
@ -44,19 +45,42 @@ import com.google.common.collect.ImmutableMap;
// Format specific parameter names.
private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id";
private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets";
private static final String PARAMETER_AMR_OCTET_ALIGN = "octet-align";
private static final String PARAMETER_AMR_INTERLEAVING = "interleaving";
private static final String PARAMETER_H265_SPROP_SPS = "sprop-sps";
private static final String PARAMETER_H265_SPROP_PPS = "sprop-pps";
private static final String PARAMETER_H265_SPROP_VPS = "sprop-vps";
private static final String PARAMETER_H265_SPROP_MAX_DON_DIFF = "sprop-max-don-diff";
private static final String PARAMETER_AMR_OCTET_ALIGN = "octet-align";
private static final String PARAMETER_AMR_INTERLEAVING = "interleaving";
private static final String PARAMETER_MP4V_CONFIG = "config";
/** Prefix for the RFC6381 codecs string for AAC formats. */
private static final String AAC_CODECS_PREFIX = "mp4a.40.";
/** Prefix for the RFC6381 codecs string for AVC formats. */
private static final String H264_CODECS_PREFIX = "avc1.";
/** Prefix for the RFC6416 codecs string for MPEG4V-ES formats. */
private static final String MPEG4_CODECS_PREFIX = "mp4v.";
private static final String GENERIC_CONTROL_ATTR = "*";
/**
* Default height for MP4V.
*
* <p>RFC6416 does not mandate codec specific data (like width and height) in the fmtp attribute.
* These values are taken from <a
* href=https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/codec2/components/mpeg4_h263/C2SoftMpeg4Dec.cpp;l=130
* >Android's software MP4V decoder</a>.
*/
private static final int DEFAULT_MP4V_WIDTH = 352;
/**
* Default height for MP4V.
*
* <p>RFC6416 does not mandate codec specific data (like width and height) in the fmtp attribute.
* These values are taken from <a
* href=https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/codec2/components/mpeg4_h263/C2SoftMpeg4Dec.cpp;l=130
* >Android's software MP4V decoder</a>.
*/
private static final int DEFAULT_MP4V_HEIGHT = 288;
/**
* Default width for VP8.
@ -156,6 +180,10 @@ import com.google.common.collect.ImmutableMap;
!fmtpParameters.containsKey(PARAMETER_AMR_INTERLEAVING),
"Interleaving mode is not currently supported.");
break;
case MimeTypes.VIDEO_MP4V:
checkArgument(!fmtpParameters.isEmpty());
processMPEG4FmtpAttribute(formatBuilder, fmtpParameters);
break;
case MimeTypes.VIDEO_H264:
checkArgument(!fmtpParameters.isEmpty());
processH264FmtpAttribute(formatBuilder, fmtpParameters);
@ -214,6 +242,23 @@ import com.google.common.collect.ImmutableMap;
AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount)));
}
private static void processMPEG4FmtpAttribute(
Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
@Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4V_CONFIG);
if (configInput != null) {
byte[] configBuffer = Util.getBytesFromHexString(configInput);
formatBuilder.setInitializationData(ImmutableList.of(configBuffer));
Pair<Integer, Integer> resolution =
CodecSpecificDataUtil.getVideoResolutionFromMpeg4VideoConfig(configBuffer);
formatBuilder.setWidth(resolution.first).setHeight(resolution.second);
} else {
// set the default width and height
formatBuilder.setWidth(DEFAULT_MP4V_WIDTH).setHeight(DEFAULT_MP4V_HEIGHT);
}
@Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID);
formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + (profileLevel == null ? "1" : profileLevel));
}
/** Returns H264/H265 initialization data from the RTP parameter set. */
private static byte[] getInitializationDataFromParameterSet(String parameterSet) {
byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT);

2
libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java
View File

@ -47,6 +47,8 @@ import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
return new RtpH264Reader(payloadFormat);
case MimeTypes.VIDEO_H265:
return new RtpH265Reader(payloadFormat);
case MimeTypes.VIDEO_MP4V:
return new RtpMpeg4Reader(payloadFormat);
case MimeTypes.VIDEO_VP8:
return new RtpVp8Reader(payloadFormat);
default:

150
libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMpeg4Reader.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.rtsp.reader;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.common.util.Util.castNonNull;
import androidx.media3.common.C;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.rtsp.RtpPacket;
import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.TrackOutput;
import com.google.common.primitives.Bytes;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Parses an MPEG4 byte stream carried on RTP packets, and extracts MPEG4 Access Units. Refer to
* RFC6416 for more details.
*/
@UnstableApi
/* package */ final class RtpMpeg4Reader implements RtpPayloadReader {
private static final String TAG = "RtpMpeg4Reader";
private static final long MEDIA_CLOCK_FREQUENCY = 90_000;
/** VOP (Video Object Plane) unit type. */
private static final int I_VOP = 0;
private final RtpPayloadFormat payloadFormat;
private @MonotonicNonNull TrackOutput trackOutput;
private @C.BufferFlags int bufferFlags;
/**
* First received RTP timestamp. All RTP timestamps are dimension-less, the time base is defined
* by {@link #MEDIA_CLOCK_FREQUENCY}.
*/
private long firstReceivedTimestamp;
private int previousSequenceNumber;
private long startTimeOffsetUs;
private int sampleLength;
/** Creates an instance. */
public RtpMpeg4Reader(RtpPayloadFormat payloadFormat) {
this.payloadFormat = payloadFormat;
firstReceivedTimestamp = C.TIME_UNSET;
previousSequenceNumber = C.INDEX_UNSET;
sampleLength = 0;
}
@Override
public void createTracks(ExtractorOutput extractorOutput, int trackId) {
trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO);
castNonNull(trackOutput).format(payloadFormat.format);
}
@Override
public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {}
@Override
public void consume(
ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) {
checkStateNotNull(trackOutput);
// Check that this packet is in the sequence of the previous packet.
if (previousSequenceNumber != C.INDEX_UNSET) {
int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber);
if (sequenceNumber != expectedSequenceNumber) {
Log.w(
TAG,
Util.formatInvariant(
"Received RTP packet with unexpected sequence number. Expected: %d; received: %d."
+ " Dropping packet.",
expectedSequenceNumber, sequenceNumber));
}
}
// Parse VOP Type and get the buffer flags
int limit = data.bytesLeft();
trackOutput.sampleData(data, limit);
if (sampleLength == 0) {
bufferFlags = getBufferFlagsFromVop(data);
}
sampleLength += limit;
// RTP marker indicates the last packet carrying a VOP.
if (rtpMarker) {
if (firstReceivedTimestamp == C.TIME_UNSET) {
firstReceivedTimestamp = timestamp;
}
long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
trackOutput.sampleMetadata(timeUs, bufferFlags, sampleLength, 0, null);
sampleLength = 0;
}
previousSequenceNumber = sequenceNumber;
}
@Override
public void seek(long nextRtpTimestamp, long timeUs) {
firstReceivedTimestamp = nextRtpTimestamp;
startTimeOffsetUs = timeUs;
sampleLength = 0;
}
// Internal methods.
/**
* Returns VOP (Video Object Plane) Coding type.
*
* <p>Sets {@link #bufferFlags} according to the VOP Coding type.
*/
private static @C.BufferFlags int getBufferFlagsFromVop(ParsableByteArray data) {
// search for VOP_START_CODE (00 00 01 B6)
byte[] inputData = data.getData();
byte[] startCode = new byte[] {0x0, 0x0, 0x1, (byte) 0xB6};
int vopStartCodePos = Bytes.indexOf(inputData, startCode);
if (vopStartCodePos != -1) {
data.setPosition(vopStartCodePos + 4);
int vopType = data.peekUnsignedByte() >> 6;
return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0;
}
return 0;
}
private static long toSampleUs(
long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) {
return startTimeOffsetUs
+ Util.scaleLargeTimestamp(
(rtpTimestamp - firstReceivedRtpTimestamp),
/* multiplier= */ C.MICROS_PER_SECOND,
/* divisor= */ MEDIA_CLOCK_FREQUENCY);
}
}