Bug 1616875: Use timestamps for SSRC/CSRC statistics that are consistent with JS timestamps. r=ng

This means we won't have to re-implement all of our JSImpl workarounds in c++.
Also, rename a variable in a way that made it easier to read this code.

Differential Revision: https://phabricator.services.mozilla.com/D50395

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Byron Campen [:bwc] 2020-03-19 18:41:38 +00:00
Родитель f995b16e1b
Коммит 2c26ca9900
6 изменённых файлов: 71 добавлений и 53 удалений

Просмотреть файл

@ -1,4 +1,5 @@
#include <RtpSourceObserver.h>
#include "RTCStatsReport.h"
#include "webrtc/modules/include/module_common_types.h"
#define GTEST_HAS_RTTI 0
#include "gtest/gtest.h"
@ -19,7 +20,7 @@ class RtpSourcesTest : public ::testing::Test {
EXPECT_EQ(history.mDetailedHistory.size(), static_cast<size_t>(0));
const auto& e = history.mLatestEviction;
EXPECT_FALSE(history.mHasEvictedEntry);
EXPECT_EQ(e.jitterAdjustedTimestamp, 0);
EXPECT_EQ(e.predictedPlayoutTime, 0);
EXPECT_FALSE(e.hasAudioLevel);
EXPECT_EQ(e.audioLevel, 0);
}
@ -40,7 +41,7 @@ class RtpSourcesTest : public ::testing::Test {
auto entry = history.FindClosestNotAfter(i + jitter);
ASSERT_NE(entry, nullptr);
if (entry) {
EXPECT_EQ(entry->jitterAdjustedTimestamp, i + jitter);
EXPECT_EQ(entry->predictedPlayoutTime, i + jitter);
EXPECT_EQ(entry->hasAudioLevel, hasAudioLevel);
EXPECT_EQ(entry->audioLevel, audioLevel);
}
@ -74,25 +75,25 @@ class RtpSourcesTest : public ::testing::Test {
history.Insert(timeNow, time1, 1, true, 0);
// Check that the jitter window buffer hasn't been used
EXPECT_TRUE(history.Empty());
ASSERT_EQ(history.mLatestEviction.jitterAdjustedTimestamp, time1);
ASSERT_EQ(history.mLatestEviction.predictedPlayoutTime, time1);
EXPECT_TRUE(history.mHasEvictedEntry);
// time2
history.Insert(timeNow, time2, 2, true, 0);
EXPECT_TRUE(history.Empty());
ASSERT_EQ(history.mLatestEviction.jitterAdjustedTimestamp, time2);
ASSERT_EQ(history.mLatestEviction.predictedPlayoutTime, time2);
EXPECT_TRUE(history.mHasEvictedEntry);
// time3
history.Insert(timeNow, time3, 3, true, 0);
EXPECT_TRUE(history.Empty());
ASSERT_EQ(history.mLatestEviction.jitterAdjustedTimestamp, time2);
ASSERT_EQ(history.mLatestEviction.predictedPlayoutTime, time2);
EXPECT_TRUE(history.mHasEvictedEntry);
// pruneTime0
history.Prune(pruneTime0);
EXPECT_TRUE(history.Empty());
ASSERT_EQ(history.mLatestEviction.jitterAdjustedTimestamp, time2);
ASSERT_EQ(history.mLatestEviction.predictedPlayoutTime, time2);
EXPECT_TRUE(history.mHasEvictedEntry);
// pruneTime1
@ -134,7 +135,7 @@ class RtpSourcesTest : public ::testing::Test {
ASSERT_EQ(history.mMaxJitterWindow, jitterWindow);
EXPECT_EQ(history.mDetailedHistory.size(), static_cast<size_t>(1));
EXPECT_TRUE(history.mHasEvictedEntry);
ASSERT_EQ(history.mLatestEviction.jitterAdjustedTimestamp, time0);
ASSERT_EQ(history.mLatestEviction.predictedPlayoutTime, time0);
ASSERT_EQ(history.mLatestEviction.hasAudioLevel, false);
ASSERT_EQ(history.mLatestEviction.audioLevel, 1);
@ -143,7 +144,7 @@ class RtpSourcesTest : public ::testing::Test {
EXPECT_EQ(history.mMaxJitterWindow, jitterWindow);
EXPECT_EQ(history.mDetailedHistory.size(), static_cast<size_t>(0));
EXPECT_TRUE(history.mHasEvictedEntry);
EXPECT_EQ(history.mLatestEviction.jitterAdjustedTimestamp, time1);
EXPECT_EQ(history.mLatestEviction.predictedPlayoutTime, time1);
EXPECT_EQ(history.mLatestEviction.hasAudioLevel, true);
EXPECT_EQ(history.mLatestEviction.audioLevel, 2);
}
@ -217,7 +218,7 @@ class RtpSourcesTest : public ::testing::Test {
history.Prune(timeNow + (jitter * 3) + 1);
EXPECT_EQ(history.mDetailedHistory.size(), static_cast<size_t>(0));
EXPECT_TRUE(history.mHasEvictedEntry);
EXPECT_EQ(jitterAdjusted, history.mLatestEviction.jitterAdjustedTimestamp);
EXPECT_EQ(jitterAdjusted, history.mLatestEviction.predictedPlayoutTime);
}
// Observer tests that keys are properly handled
@ -242,7 +243,8 @@ class RtpSourcesTest : public ::testing::Test {
// Observer a header with a single Csrc
void TestObserveOneCsrc() {
RtpSourceObserver observer;
RtpSourceObserver observer =
RtpSourceObserver(dom::RTCStatsTimestampMaker());
webrtc::RTPHeader header;
constexpr unsigned int ssrc = 857265;
constexpr unsigned int csrc = 3268365;
@ -257,7 +259,7 @@ class RtpSourcesTest : public ::testing::Test {
// One for the SSRC, one for the CSRC
EXPECT_EQ(observer.mRtpSources.size(), static_cast<size_t>(2));
nsTArray<dom::RTCRtpSourceEntry> outLevels;
observer.GetRtpSources(timestamp, outLevels);
observer.GetRtpSources(outLevels);
EXPECT_EQ(outLevels.Length(), static_cast<size_t>(2));
bool ssrcFound = false;
bool csrcFound = true;
@ -278,7 +280,8 @@ class RtpSourcesTest : public ::testing::Test {
// Observer a header with two CSRCs
void TestObserveTwoCsrcs() {
RtpSourceObserver observer;
RtpSourceObserver observer =
RtpSourceObserver(dom::RTCStatsTimestampMaker());
webrtc::RTPHeader header;
constexpr unsigned int ssrc = 239485;
constexpr unsigned int csrc0 = 3425;
@ -295,7 +298,7 @@ class RtpSourcesTest : public ::testing::Test {
// One for the SSRC, two for the CSRCs
EXPECT_EQ(observer.mRtpSources.size(), static_cast<size_t>(3));
nsTArray<dom::RTCRtpSourceEntry> outLevels;
observer.GetRtpSources(timestamp, outLevels);
observer.GetRtpSources(outLevels);
EXPECT_EQ(outLevels.Length(), static_cast<size_t>(3));
bool ssrcFound = false;
bool csrc0Found = true;
@ -322,7 +325,8 @@ class RtpSourcesTest : public ::testing::Test {
// Observer a header with a CSRC with audio level extension
void TestObserveCsrcWithAudioLevel() {
RtpSourceObserver observer;
RtpSourceObserver observer =
RtpSourceObserver(dom::RTCStatsTimestampMaker());
webrtc::RTPHeader header;
}
};

Просмотреть файл

@ -311,9 +311,9 @@ void WebrtcAudioConduit::SetRtcpEventObserver(
}
void WebrtcAudioConduit::GetRtpSources(
const int64_t aTimeNow, nsTArray<dom::RTCRtpSourceEntry>& outSources) {
nsTArray<dom::RTCRtpSourceEntry>& outSources) {
MOZ_ASSERT(NS_IsMainThread());
return mRtpSourceObserver.GetRtpSources(aTimeNow, outSources);
return mRtpSourceObserver.GetRtpSources(outSources);
}
// test-only: inserts a CSRC entry in a RtpSourceObserver's history for

Просмотреть файл

@ -197,6 +197,7 @@ class WebrtcAudioConduit : public AudioSessionConduit,
mSendChannel(-1),
mDtmfEnabled(false),
mMutex("WebrtcAudioConduit::mMutex"),
mRtpSourceObserver(mCall->GetTimestampMaker()),
mStsThread(aStsThread) {}
virtual ~WebrtcAudioConduit();
@ -241,8 +242,7 @@ class WebrtcAudioConduit : public AudioSessionConduit,
bool InsertDTMFTone(int channel, int eventCode, bool outOfBand, int lengthMs,
int attenuationDb) override;
void GetRtpSources(const int64_t aTimeNow,
nsTArray<dom::RTCRtpSourceEntry>& outSources) override;
void GetRtpSources(nsTArray<dom::RTCRtpSourceEntry>& outSources) override;
void OnRtpPacket(const webrtc::RTPHeader& aRtpHeader,
const int64_t aTimestamp, const uint32_t aJitter) override;

Просмотреть файл

@ -312,6 +312,10 @@ class WebRtcCallWrapper : public RefCounted<WebRtcCallWrapper> {
DOMHighResTimeStamp GetNow() const { return mTimestampMaker.GetNow(); }
const dom::RTCStatsTimestampMaker& GetTimestampMaker() const {
return mTimestampMaker;
}
MOZ_DECLARE_REFCOUNTED_TYPENAME(WebRtcCallWrapper)
rtc::scoped_refptr<webrtc::AudioDecoderFactory> mDecoderFactory;
@ -597,8 +601,7 @@ class AudioSessionConduit : public MediaSessionConduit {
virtual bool InsertDTMFTone(int channel, int eventCode, bool outOfBand,
int lengthMs, int attenuationDb) = 0;
virtual void GetRtpSources(const int64_t aTimeNow,
nsTArray<dom::RTCRtpSourceEntry>& outSources) = 0;
virtual void GetRtpSources(nsTArray<dom::RTCRtpSourceEntry>& outSources) = 0;
};
} // namespace mozilla
#endif

Просмотреть файл

@ -22,21 +22,35 @@ double RtpSourceObserver::RtpSourceEntry::ToLinearAudioLevel() const {
return std::pow(10, -static_cast<double>(audioLevel) / 20);
}
RtpSourceObserver::RtpSourceObserver()
: mMaxJitterWindow(0), mLevelGuard("RtpSourceObserver::mLevelGuard") {}
RtpSourceObserver::RtpSourceObserver(
const dom::RTCStatsTimestampMaker& aTimestampMaker)
: mMaxJitterWindow(0),
mLevelGuard("RtpSourceObserver::mLevelGuard"),
mTimestampMaker(aTimestampMaker) {}
void RtpSourceObserver::OnRtpPacket(const webrtc::RTPHeader& aHeader,
const int64_t aTimestamp,
const uint32_t aJitter) {
MutexAutoLock lock(mLevelGuard);
{
// We assume that aTimestamp is not significantly in the past, and just get
// a JS timestamp for the current time instead of converting aTimestamp.
DOMHighResTimeStamp jsNow = mTimestampMaker.GetNow();
mMaxJitterWindow =
std::max(mMaxJitterWindow, static_cast<int64_t>(aJitter) * 2);
const auto jitterAdjusted = aTimestamp + aJitter;
// We are supposed to report the time at which this packet was played out,
// but we have only just received the packet. We try to guess when it will
// be played out.
// TODO: We need to move where we update these stats to MediaPipeline, where
// we send frames to the media track graph. In order to do that, we will
// need to have the ssrc (and csrc) for decoded frames, but we don't have
// that right now. Once we move this to the correct place, we will no longer
// need to keep anything but the most recent data.
const auto predictedPlayoutTime = jsNow + aJitter;
auto& hist = mRtpSources[GetKey(aHeader.ssrc, EntryType::Synchronization)];
hist.Prune(aTimestamp);
hist.Prune(jsNow);
// ssrc-audio-level handling
hist.Insert(aTimestamp, jitterAdjusted, aHeader.timestamp,
hist.Insert(jsNow, predictedPlayoutTime, aHeader.timestamp,
aHeader.extension.hasAudioLevel, aHeader.extension.audioLevel);
// csrc-audio-level handling
@ -44,27 +58,27 @@ void RtpSourceObserver::OnRtpPacket(const webrtc::RTPHeader& aHeader,
for (uint8_t i = 0; i < aHeader.numCSRCs; i++) {
const uint32_t& csrc = aHeader.arrOfCSRCs[i];
auto& hist = mRtpSources[GetKey(csrc, EntryType::Contributing)];
hist.Prune(aTimestamp);
hist.Prune(jsNow);
bool hasLevel = i < list.numAudioLevels;
uint8_t level = hasLevel ? list.arrOfAudioLevels[i] : 0;
hist.Insert(aTimestamp, jitterAdjusted, aHeader.timestamp, hasLevel,
hist.Insert(jsNow, predictedPlayoutTime, aHeader.timestamp, hasLevel,
level);
}
}
}
void RtpSourceObserver::GetRtpSources(
const int64_t aTimeNow,
nsTArray<dom::RTCRtpSourceEntry>& outSources) const {
MutexAutoLock lock(mLevelGuard);
outSources.Clear();
for (const auto& it : mRtpSources) {
const RtpSourceEntry* entry = it.second.FindClosestNotAfter(aTimeNow);
const RtpSourceEntry* entry =
it.second.FindClosestNotAfter(mTimestampMaker.GetNow());
if (entry) {
dom::RTCRtpSourceEntry domEntry;
domEntry.mSource = GetSourceFromKey(it.first);
domEntry.mSourceType = GetTypeFromKey(it.first);
domEntry.mTimestamp = entry->jitterAdjustedTimestamp;
domEntry.mTimestamp = entry->predictedPlayoutTime;
domEntry.mRtpTimestamp = entry->rtpTimestamp;
if (entry->hasAudioLevel) {
domEntry.mAudioLevel.Construct(entry->ToLinearAudioLevel());
@ -74,10 +88,6 @@ void RtpSourceObserver::GetRtpSources(
}
}
int64_t RtpSourceObserver::NowInReportClockTime() {
return webrtc::Clock::GetRealTimeClock()->TimeInMilliseconds();
}
const RtpSourceObserver::RtpSourceEntry*
RtpSourceObserver::RtpSourceHistory::FindClosestNotAfter(int64_t aTime) const {
// This method scans the history for the entry whose timestamp is closest to a
@ -88,7 +98,7 @@ RtpSourceObserver::RtpSourceHistory::FindClosestNotAfter(int64_t aTime) const {
auto lastFound = mDetailedHistory.cbegin();
bool found = false;
for (const auto& it : mDetailedHistory) {
if (it.second.jitterAdjustedTimestamp > aTime) {
if (it.second.predictedPlayoutTime > aTime) {
break;
}
// lastFound can't start before begin, so the first inc must be skipped
@ -100,7 +110,7 @@ RtpSourceObserver::RtpSourceHistory::FindClosestNotAfter(int64_t aTime) const {
if (found) {
return &lastFound->second;
}
if (HasEvicted() && aTime >= mLatestEviction.jitterAdjustedTimestamp) {
if (HasEvicted() && aTime >= mLatestEviction.predictedPlayoutTime) {
return &mLatestEviction;
}
return nullptr;
@ -113,7 +123,7 @@ void RtpSourceObserver::RtpSourceHistory::Prune(const int64_t aTimeNow) {
// New lower bound of the map
auto lower = mDetailedHistory.begin();
for (auto& it : mDetailedHistory) {
if (it.second.jitterAdjustedTimestamp > aTimeT) {
if (it.second.predictedPlayoutTime > aTimeT) {
found = true;
break;
}
@ -123,7 +133,7 @@ void RtpSourceObserver::RtpSourceHistory::Prune(const int64_t aTimeNow) {
found = true;
}
if (found) {
if (lower->second.jitterAdjustedTimestamp > aTimePrehistory) {
if (lower->second.predictedPlayoutTime > aTimePrehistory) {
mLatestEviction = lower->second;
mHasEvictedEntry = true;
}
@ -131,7 +141,7 @@ void RtpSourceObserver::RtpSourceHistory::Prune(const int64_t aTimeNow) {
mDetailedHistory.erase(mDetailedHistory.begin(), lower);
}
if (HasEvicted() &&
(mLatestEviction.jitterAdjustedTimestamp + kHistoryWindow) < aTimeNow) {
(mLatestEviction.predictedPlayoutTime + kHistoryWindow) < aTimeNow) {
mHasEvictedEntry = false;
}
}
@ -156,7 +166,7 @@ RtpSourceObserver::RtpSourceEntry& RtpSourceObserver::RtpSourceHistory::Insert(
// x or x T J
// |------Z-----|ABC| -> |------Z-----|ABC|
if ((aTimestamp + kHistoryWindow) < aTimeNow ||
aTimestamp < mLatestEviction.jitterAdjustedTimestamp) {
aTimestamp < mLatestEviction.predictedPlayoutTime) {
return mPrehistory; // A.K.A. /dev/null
}
mMaxJitterWindow = std::max(mMaxJitterWindow, (aTimestamp - aTimeNow) * 2);

Просмотреть файл

@ -14,6 +14,7 @@
#include "nsISupportsImpl.h"
#include "mozilla/dom/RTCRtpSourcesBinding.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_packet_observer.h"
#include "RTCStatsReport.h"
// Unit Test class
namespace test {
@ -29,29 +30,21 @@ namespace mozilla {
*/
class RtpSourceObserver : public webrtc::RtpPacketObserver {
public:
RtpSourceObserver();
explicit RtpSourceObserver(
const dom::RTCStatsTimestampMaker& aTimestampMaker);
virtual ~RtpSourceObserver(){};
void OnRtpPacket(const webrtc::RTPHeader& aRtpHeader,
const int64_t aTimestamp, const uint32_t aJitter) override;
/* Get the local time in MS from the same clock source that is used
* to generate the capture timestamps. Use for computing the age of
* an entry relative to another clock, e.g. the JS
* @return time of now in MS
*/
static int64_t NowInReportClockTime();
/*
* Get the most recent 10 second window of CSRC and SSRC sources.
* @param aTimeNow the current report clock time, @see NowInReportClockTime.
* @param outLevels will be popluted with source entries
* Note: this takes jitter into account when calculating the window so
* the window is actually [time - jitter - 10 sec .. time - jitter]
*/
void GetRtpSources(const int64_t aTimeNow,
nsTArray<dom::RTCRtpSourceEntry>& outSources) const;
void GetRtpSources(nsTArray<dom::RTCRtpSourceEntry>& outSources) const;
private:
// Note: these are pool allocated
@ -59,7 +52,7 @@ class RtpSourceObserver : public webrtc::RtpPacketObserver {
RtpSourceEntry() = default;
void Update(const int64_t aTimestamp, const uint32_t aRtpTimestamp,
const bool aHasAudioLevel, const uint8_t aAudioLevel) {
jitterAdjustedTimestamp = aTimestamp;
predictedPlayoutTime = aTimestamp;
rtpTimestamp = aRtpTimestamp;
// Audio level range is 0 - 127 inclusive
hasAudioLevel = aHasAudioLevel && !(aAudioLevel & 0x80);
@ -69,12 +62,19 @@ class RtpSourceObserver : public webrtc::RtpPacketObserver {
// outlined in the webrtc-pc spec.
double ToLinearAudioLevel() const;
// Time this information was received + jitter
int64_t jitterAdjustedTimestamp = 0;
int64_t predictedPlayoutTime = 0;
// The original RTP timestamp in the received packet
uint32_t rtpTimestamp = 0;
bool hasAudioLevel = false;
uint8_t audioLevel = 0;
};
/* Why this is needed:
* We are supposed to only report source stats for packets that have already
* been rendered. Unfortunately, we only know when these packets are
* _received_ right now. So, we need to make a guess at when each packet will
* be rendered, and hide its statistics until the clock reaches that estimate.
*/
/* Maintains a history of packets for reporting with getContributingSources
* and getSynchronizationSources. It is expected that entries will not always
* be observed in chronological order, and that the correct entry for a query
@ -166,6 +166,7 @@ class RtpSourceObserver : public webrtc::RtpPacketObserver {
int64_t mMaxJitterWindow;
// Guards statistics
mutable Mutex mLevelGuard;
dom::RTCStatsTimestampMaker mTimestampMaker;
// Unit test
friend test::RtpSourcesTest;