From 6a3b43fa14626013f129c2fe00f7803d84318f7f Mon Sep 17 00:00:00 2001 From: Dan Minor Date: Fri, 8 May 2020 18:10:59 +0000 Subject: [PATCH] Bug 1164187 - Add jsep support for rtx; r=bwc Differential Revision: https://phabricator.services.mozilla.com/D72225 --- .../signaling/gtest/jsep_track_unittest.cpp | 6 +- .../signaling/src/jsep/JsepCodecDescription.h | 71 +++++++++++++++ .../signaling/src/jsep/JsepSessionImpl.cpp | 6 +- media/webrtc/signaling/src/jsep/JsepTrack.cpp | 86 +++++++++++++++---- media/webrtc/signaling/src/jsep/JsepTrack.h | 43 +++++++++- 5 files changed, 191 insertions(+), 21 deletions(-) diff --git a/media/webrtc/signaling/gtest/jsep_track_unittest.cpp b/media/webrtc/signaling/gtest/jsep_track_unittest.cpp index 0daf635490f8..14a5c86d9b5f 100644 --- a/media/webrtc/signaling/gtest/jsep_track_unittest.cpp +++ b/media/webrtc/signaling/gtest/jsep_track_unittest.cpp @@ -1275,7 +1275,8 @@ TEST_F(JsepTrackTest, SimulcastOfferer) { CreateOffer(); CreateAnswer(); // Add simulcast/rid to answer - mRecvAns.AddToMsection(constraints, sdp::kRecv, mSsrcGenerator, &GetAnswer()); + mRecvAns.AddToMsection(constraints, sdp::kRecv, mSsrcGenerator, false, + &GetAnswer()); Negotiate(); ASSERT_TRUE(mSendOff.GetNegotiatedDetails()); ASSERT_EQ(2U, mSendOff.GetNegotiatedDetails()->GetEncodingCount()); @@ -1303,7 +1304,8 @@ TEST_F(JsepTrackTest, SimulcastAnswerer) { mSendAns.SetJsConstraints(constraints); CreateOffer(); // Add simulcast/rid to offer - mRecvOff.AddToMsection(constraints, sdp::kRecv, mSsrcGenerator, &GetOffer()); + mRecvOff.AddToMsection(constraints, sdp::kRecv, mSsrcGenerator, false, + &GetOffer()); CreateAnswer(); Negotiate(); ASSERT_TRUE(mSendAns.GetNegotiatedDetails()); diff --git a/media/webrtc/signaling/src/jsep/JsepCodecDescription.h b/media/webrtc/signaling/src/jsep/JsepCodecDescription.h index 43fbdc7862c8..a47870d9cbcd 100644 --- a/media/webrtc/signaling/src/jsep/JsepCodecDescription.h +++ b/media/webrtc/signaling/src/jsep/JsepCodecDescription.h @@ -203,6 +203,7 @@ class JsepVideoCodecDescription : public JsepCodecDescription { mRembEnabled(false), mFECEnabled(false), mTransportCCEnabled(false), + mRtxEnabled(false), mREDPayloadType(0), mULPFECPayloadType(0), mProfileLevelId(0), @@ -260,6 +261,11 @@ class JsepVideoCodecDescription : public JsepCodecDescription { } } + void EnableRtx(const std::string& rtxPayloadType) { + mRtxEnabled = true; + mRtxPayloadType = rtxPayloadType; + } + void AddParametersToMSection(SdpMediaSection& msection) const override { AddFmtpsToMSection(msection); AddRtcpFbsToMSection(msection); @@ -311,6 +317,20 @@ class JsepVideoCodecDescription : public JsepCodecDescription { msection.SetFmtp(SdpFmtpAttributeList::Fmtp(mDefaultPt, vp8Params)); } } + + if (mRtxEnabled && mDirection == sdp::kRecv) { + SdpFmtpAttributeList::RtxParameters params( + GetRtxParameters(mDefaultPt, msection)); + uint16_t apt; + if (SdpHelper::GetPtAsInt(mDefaultPt, &apt)) { + if (apt <= 127) { + msection.AddCodec(mRtxPayloadType, "rtx", mClock, mChannels); + + params.apt = apt; + msection.SetFmtp(SdpFmtpAttributeList::Fmtp(mRtxPayloadType, params)); + } + } + } } void AddRtcpFbsToMSection(SdpMediaSection& msection) const { @@ -364,6 +384,45 @@ class JsepVideoCodecDescription : public JsepCodecDescription { return result; } + SdpFmtpAttributeList::RtxParameters GetRtxParameters( + const std::string& pt, const SdpMediaSection& msection) const { + SdpFmtpAttributeList::RtxParameters result; + const auto* params = msection.FindFmtp(pt); + + if (params && params->codec_type == SdpRtpmapAttributeList::kRtx) { + result = static_cast(*params); + } + + return result; + } + + Maybe GetRtxPtByApt(const std::string& apt, + const SdpMediaSection& msection) const { + Maybe result; + uint16_t aptAsInt; + if (!SdpHelper::GetPtAsInt(apt, &aptAsInt)) { + return result; + } + + const SdpAttributeList& attrs = msection.GetAttributeList(); + if (attrs.HasAttribute(SdpAttribute::kFmtpAttribute)) { + for (const auto& fmtpAttr : attrs.GetFmtp().mFmtps) { + if (fmtpAttr.parameters) { + auto* params = fmtpAttr.parameters.get(); + if (params && params->codec_type == SdpRtpmapAttributeList::kRtx) { + const SdpFmtpAttributeList::RtxParameters* rtxParams = + static_cast(params); + if (rtxParams->apt == aptAsInt) { + result = Some(fmtpAttr.format); + break; + } + } + } + } + } + return result; + } + SdpFmtpAttributeList::VP8Parameters GetVP8Parameters( const std::string& pt, const SdpMediaSection& msection) const { SdpRtpmapAttributeList::CodecType expectedType( @@ -458,6 +517,16 @@ class JsepVideoCodecDescription : public JsepCodecDescription { } } + if (mRtxEnabled && (mDirection == sdp::kSend || isOffer)) { + Maybe rtxPt = GetRtxPtByApt(mDefaultPt, remoteMsection); + if (rtxPt.isSome()) { + EnableRtx(*rtxPt); + } else { + mRtxEnabled = false; + mRtxPayloadType = ""; + } + } + NegotiateRtcpFb(remoteMsection); return true; } @@ -693,8 +762,10 @@ class JsepVideoCodecDescription : public JsepCodecDescription { bool mRembEnabled; bool mFECEnabled; bool mTransportCCEnabled; + bool mRtxEnabled; uint8_t mREDPayloadType; uint8_t mULPFECPayloadType; + std::string mRtxPayloadType; std::vector mRedundantEncodings; // H264-specific stuff diff --git a/media/webrtc/signaling/src/jsep/JsepSessionImpl.cpp b/media/webrtc/signaling/src/jsep/JsepSessionImpl.cpp index e04cba6130e7..9a24a3665b13 100644 --- a/media/webrtc/signaling/src/jsep/JsepSessionImpl.cpp +++ b/media/webrtc/signaling/src/jsep/JsepSessionImpl.cpp @@ -94,7 +94,7 @@ nsresult JsepSessionImpl::AddTransceiver(RefPtr transceiver) { if (transceiver->GetMediaType() != SdpMediaSection::kApplication) { // Make sure we have an ssrc. Might already be set. - transceiver->mSendTrack.EnsureSsrcs(mSsrcGenerator); + transceiver->mSendTrack.EnsureSsrcs(mSsrcGenerator, 1U); transceiver->mSendTrack.SetCNAME(mCNAME); // Make sure we have identifiers for send track, just in case. @@ -1950,6 +1950,7 @@ void JsepSessionImpl::SetupDefaultCodecs() { // Defaults for mandatory params vp8->mConstraints.maxFs = 12288; // Enough for 2048x1536 vp8->mConstraints.maxFps = 60; + vp8->EnableRtx("124"); mSupportedCodecs.push_back(std::move(vp8)); UniquePtr vp9( @@ -1957,6 +1958,7 @@ void JsepSessionImpl::SetupDefaultCodecs() { // Defaults for mandatory params vp9->mConstraints.maxFs = 12288; // Enough for 2048x1536 vp9->mConstraints.maxFps = 60; + vp9->EnableRtx("125"); mSupportedCodecs.push_back(std::move(vp9)); UniquePtr h264_1( @@ -1964,6 +1966,7 @@ void JsepSessionImpl::SetupDefaultCodecs() { h264_1->mPacketizationMode = 1; // Defaults for mandatory params h264_1->mProfileLevelId = 0x42E00D; + h264_1->EnableRtx("127"); mSupportedCodecs.push_back(std::move(h264_1)); UniquePtr h264_0( @@ -1971,6 +1974,7 @@ void JsepSessionImpl::SetupDefaultCodecs() { h264_0->mPacketizationMode = 0; // Defaults for mandatory params h264_0->mProfileLevelId = 0x42E00D; + h264_0->EnableRtx("98"); mSupportedCodecs.push_back(std::move(h264_0)); UniquePtr ulpfec(new JsepVideoCodecDescription( diff --git a/media/webrtc/signaling/src/jsep/JsepTrack.cpp b/media/webrtc/signaling/src/jsep/JsepTrack.cpp index 19e139342486..891def99d517 100644 --- a/media/webrtc/signaling/src/jsep/JsepTrack.cpp +++ b/media/webrtc/signaling/src/jsep/JsepTrack.cpp @@ -83,13 +83,16 @@ void JsepTrack::EnsureNoDuplicatePayloadTypes( } } -void JsepTrack::EnsureSsrcs(SsrcGenerator& ssrcGenerator) { - if (mSsrcs.empty()) { - uint32_t ssrc; - if (!ssrcGenerator.GenerateSsrc(&ssrc)) { +void JsepTrack::EnsureSsrcs(SsrcGenerator& ssrcGenerator, size_t aNumber) { + while (mSsrcs.size() < aNumber) { + uint32_t ssrc, rtxSsrc; + if (!ssrcGenerator.GenerateSsrc(&ssrc) || + !ssrcGenerator.GenerateSsrc(&rtxSsrc)) { return; } mSsrcs.push_back(ssrc); + mSsrcToRtxSsrc[ssrc] = rtxSsrc; + MOZ_ASSERT(mSsrcs.size() == mSsrcToRtxSsrc.size()); } } @@ -115,7 +118,9 @@ void JsepTrack::AddToOffer(SsrcGenerator& ssrcGenerator, if (offer->IsSending()) { constraints = mJsEncodeConstraints; } - AddToMsection(constraints, sdp::kSend, ssrcGenerator, offer); + + AddToMsection(constraints, sdp::kSend, ssrcGenerator, + IsRtxEnabled(mPrototypeCodecs), offer); } } @@ -140,7 +145,9 @@ void JsepTrack::AddToAnswer(const SdpMediaSection& offer, GetRids(offer, sdp::kRecv, &rids); NegotiateRids(rids, &constraints); } - AddToMsection(constraints, sdp::kSend, ssrcGenerator, answer); + + AddToMsection(constraints, sdp::kSend, ssrcGenerator, IsRtxEnabled(codecs), + answer); } } @@ -214,22 +221,44 @@ void JsepTrack::UpdateSsrcs(SsrcGenerator& ssrcGenerator, size_t encodings) { // Right now, the spec does not permit changing the number of encodings after // the initial creation of the sender, so we don't need to worry about things // like a new encoding inserted in between two pre-existing encodings. - while (mSsrcs.size() < numSsrcs) { - uint32_t ssrc; - if (!ssrcGenerator.GenerateSsrc(&ssrc)) { - return; + EnsureSsrcs(ssrcGenerator, numSsrcs); + PruneSsrcs(numSsrcs); + + MOZ_ASSERT(!mSsrcs.empty()); +} + +void JsepTrack::PruneSsrcs(size_t aNumSsrcs) { + mSsrcs.resize(aNumSsrcs); + + // We might have duplicate entries in mSsrcs, so we need to resize first and + // then remove ummatched rtx ssrcs. + auto itor = mSsrcToRtxSsrc.begin(); + while (itor != mSsrcToRtxSsrc.end()) { + if (std::find(mSsrcs.begin(), mSsrcs.end(), itor->first) == mSsrcs.end()) { + itor = mSsrcToRtxSsrc.erase(itor); + } else { + ++itor; + } + } +} + +bool JsepTrack::IsRtxEnabled( + const std::vector>& codecs) const { + for (const auto& codec : codecs) { + if (codec->mType == SdpMediaSection::kVideo && + static_cast(codec.get()) + ->mRtxEnabled) { + return true; } - mSsrcs.push_back(ssrc); } - mSsrcs.resize(numSsrcs); - MOZ_ASSERT(!mSsrcs.empty()); + return false; } void JsepTrack::AddToMsection(const std::vector& constraintsList, sdp::Direction direction, SsrcGenerator& ssrcGenerator, - SdpMediaSection* msection) { + bool requireRtxSsrcs, SdpMediaSection* msection) { UniquePtr simulcast(new SdpSimulcastAttribute); UniquePtr rids(new SdpRidAttributeList); for (const JsConstraints& constraints : constraintsList) { @@ -257,7 +286,21 @@ void JsepTrack::AddToMsection(const std::vector& constraintsList, if (mType != SdpMediaSection::kApplication && mDirection == sdp::kSend) { UpdateSsrcs(ssrcGenerator, constraintsList.size()); - msection->SetSsrcs(mSsrcs, mCNAME); + + if (requireRtxSsrcs) { + MOZ_ASSERT(mSsrcs.size() == mSsrcToRtxSsrc.size()); + std::vector allSsrcs; + UniquePtr group(new SdpSsrcGroupAttributeList); + for (const auto& [ssrc, rtxSsrc] : mSsrcToRtxSsrc) { + allSsrcs.push_back(ssrc); + allSsrcs.push_back(rtxSsrc); + group->PushEntry(SdpSsrcGroupAttributeList::kFid, {ssrc, rtxSsrc}); + } + msection->SetSsrcs(allSsrcs, mCNAME); + msection->GetAttributeList().SetAttribute(group.release()); + } else { + msection->SetSsrcs(mSsrcs, mCNAME); + } } } @@ -330,7 +373,7 @@ void JsepTrack::CreateEncodings( // Drop SSRCs if less RIDs were offered than we have encoding constraints // Just in case. if (mSsrcs.size() > max_streams) { - mSsrcs.resize(max_streams); + PruneSsrcs(max_streams); } // For each stream make sure we have an encoding, and configure @@ -391,6 +434,17 @@ std::vector> JsepTrack::NegotiateCodecs( if (clone->Negotiate(fmt, remote, isOffer)) { // If negotiation changed the payload type, remember that for later codec->mDefaultPt = clone->mDefaultPt; + + // Remember whether we negotiated rtx and the associated pt for later. + if (codec->mType == SdpMediaSection::kVideo) { + JsepVideoCodecDescription* videoCodec = + static_cast(codec.get()); + JsepVideoCodecDescription* cloneVideoCodec = + static_cast(clone.get()); + videoCodec->mRtxEnabled = cloneVideoCodec->mRtxEnabled; + videoCodec->mRtxPayloadType = cloneVideoCodec->mRtxPayloadType; + } + // Moves the codec out of mPrototypeCodecs, leaving an empty // UniquePtr, so we don't use it again. Also causes successfully // negotiated codecs to be placed up front in the future. diff --git a/media/webrtc/signaling/src/jsep/JsepTrack.h b/media/webrtc/signaling/src/jsep/JsepTrack.h index 3fd9ad0bc243..dde1e3a67d0b 100644 --- a/media/webrtc/signaling/src/jsep/JsepTrack.h +++ b/media/webrtc/signaling/src/jsep/JsepTrack.h @@ -13,6 +13,7 @@ #include #include +#include "mozilla/Preferences.h" #include "nsError.h" #include "signaling/src/jsep/JsepTrackEncoding.h" @@ -117,6 +118,31 @@ class JsepTrack { mSsrcs.push_back(ssrcAttr.ssrc); } } + + // Use FID ssrc-group to associate rtx ssrcs with "regular" ssrcs. Despite + // not being part of RFC 4588, this is how rtx is negotiated by libwebrtc + // and jitsi. + mSsrcToRtxSsrc.clear(); + if (msection.GetAttributeList().HasAttribute( + SdpAttribute::kSsrcGroupAttribute)) { + for (const auto& group : + msection.GetAttributeList().GetSsrcGroup().mSsrcGroups) { + if (group.semantics == SdpSsrcGroupAttributeList::kFid && + group.ssrcs.size() == 2) { + // Ensure we have a "regular" ssrc for each rtx ssrc. + if (std::find(mSsrcs.begin(), mSsrcs.end(), group.ssrcs[0]) != + mSsrcs.end()) { + mSsrcToRtxSsrc[group.ssrcs[0]] = group.ssrcs[1]; + + // Remove rtx ssrcs from mSsrcs + auto res = std::remove_if( + mSsrcs.begin(), mSsrcs.end(), + [group](uint32_t ssrc) { return ssrc == group.ssrcs[1]; }); + mSsrcs.erase(res, mSsrcs.end()); + } + } + } + } } JsepTrack(const JsepTrack& orig) { *this = orig; } @@ -133,6 +159,7 @@ class JsepTrack { mDirection = rhs.mDirection; mJsEncodeConstraints = rhs.mJsEncodeConstraints; mSsrcs = rhs.mSsrcs; + mSsrcToRtxSsrc = rhs.mSsrcToRtxSsrc; mActive = rhs.mActive; mRemoteSetSendBit = rhs.mRemoteSetSendBit; @@ -164,7 +191,15 @@ class JsepTrack { virtual const std::vector& GetSsrcs() const { return mSsrcs; } - virtual void EnsureSsrcs(SsrcGenerator& ssrcGenerator); + virtual std::vector GetRtxSsrcs() const { + std::vector result; + std::for_each( + mSsrcToRtxSsrc.begin(), mSsrcToRtxSsrc.end(), + [&result](const auto& pair) { result.push_back(pair.second); }); + return result; + } + + virtual void EnsureSsrcs(SsrcGenerator& ssrcGenerator, size_t aNumber); bool GetActive() const { return mActive; } @@ -234,7 +269,7 @@ class JsepTrack { void AddToMsection(const std::vector& constraintsList, sdp::Direction direction, SsrcGenerator& ssrcGenerator, - SdpMediaSection* msection); + bool requireRtxSsrcs, SdpMediaSection* msection); private: std::vector> GetCodecClones() const; @@ -263,6 +298,9 @@ class JsepTrack { const std::vector>& rids, std::vector* constraints) const; void UpdateSsrcs(SsrcGenerator& ssrcGenerator, size_t encodings); + void PruneSsrcs(size_t aNumSsrcs); + bool IsRtxEnabled( + const std::vector>& codecs) const; mozilla::SdpMediaSection::MediaType mType; // These are the ids that everyone outside of JsepSession care about @@ -277,6 +315,7 @@ class JsepTrack { std::vector mJsEncodeConstraints; UniquePtr mNegotiatedDetails; std::vector mSsrcs; + std::map mSsrcToRtxSsrc; bool mActive; bool mRemoteSetSendBit; };