mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1311935 - P3. Implement safebrowsing v4 caching logic. r=francois 

LookupCacheV4::Has implements safebrowsing v4 caching logic.
1. Check if fullhash match any prefix in local database:
  - If not, the URL is safe.
2. Check if prefix is in the cache(prefix is always the first 4-byte of
   the fullhash, Bug 1323953):
  - If not, send fullhash request
3. Check if fullhash is in the positive cache:
  - If fullhash is found and it is not expired, the URL is not safe.
  - If fullhash is found and it is expired, send fullhash request.
4. If fullhash is not found, check negative cache expired time:
  - If negative cache time is not expired, the URL is safe.
  - If negative cache time is expired, send fullhash request.

MozReview-Commit-ID: GRX7CP8ig49

--HG--
extra : rebase_source : 6d3ae8929b11731584810c44d46a1526f7d83e12
This commit is contained in:
dimi 2017-04-10 14:21:08 +08:00
Родитель 127351f6ef
Коммит 7842472dad
7 изменённых файлов: 268 добавлений и 112 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

35
toolkit/components/url-classifier/Classifier.cpp
Просмотреть файл

@ -496,25 +496,18 @@ Classifier::Check(const nsACString& aSpec,
for (uint32_t i = 0; i < cacheArray.Length(); i++) {
LookupCache *cache = cacheArray[i];
bool has, fromCache;
bool has, fromCache, confirmed;
uint32_t matchLength;
rv = cache->Has(lookupHash, &has, &matchLength, &fromCache);
rv = cache->Has(lookupHash, mTableFreshness, aFreshnessGuarantee,
&has, &matchLength, &confirmed, &fromCache);
NS_ENSURE_SUCCESS(rv, rv);
if (has) {
LookupResult *result = aResults.AppendElement();
if (!result)
return NS_ERROR_OUT_OF_MEMORY;
// For V2, there is no TTL for caching, so we use table freshness to
// decide if matching a completion should trigger a gethash request or not.
// For V4, this is done by Positive Caching & Negative Caching mechanism.
bool confirmed = false;
if (fromCache) {
cache->IsHashEntryConfirmed(lookupHash, mTableFreshness,
aFreshnessGuarantee, &confirmed);
}
LOG(("Found a result in %s: %s",
cache->TableName().get(),
confirmed ? "confirmed." : "Not confirmed."));
@ -1333,6 +1326,11 @@ Classifier::UpdateTableV4(nsTArray<TableUpdate*>* aUpdates,
return NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND;
}
// Remove cache entries whose negative cache time is expired when update.
// We don't check if positive cache time is expired here because we want to
// keep the eviction rule simple when doing an update.
lookupCache->InvalidateExpiredCacheEntry();
nsresult rv = NS_OK;
// If there are multiple updates for the same table, prefixes1 & prefixes2
@ -1423,8 +1421,19 @@ Classifier::UpdateCache(TableUpdate* aUpdate)
return NS_ERROR_FAILURE;
}
auto updateV2 = TableUpdate::Cast<TableUpdateV2>(aUpdate);
lookupCache->AddCompletionsToCache(updateV2->AddCompletes());
auto lookupV2 = LookupCache::Cast<LookupCacheV2>(lookupCache);
if (lookupV2) {
auto updateV2 = TableUpdate::Cast<TableUpdateV2>(aUpdate);
lookupV2->AddCompletionsToCache(updateV2->AddCompletes());
} else {
auto lookupV4 = LookupCache::Cast<LookupCacheV4>(lookupCache);
if (!lookupV4) {
return NS_ERROR_FAILURE;
}
auto updateV4 = TableUpdate::Cast<TableUpdateV4>(aUpdate);
lookupV4->AddFullHashResponseToCache(updateV4->FullHashResponse());
}
#if defined(DEBUG)
lookupCache->DumpCache();

102
toolkit/components/url-classifier/LookupCache.cpp
Просмотреть файл

@ -96,34 +96,6 @@ LookupCache::UpdateRootDirHandle(nsIFile* aNewRootStoreDirectory)
return rv;
}
nsresult
LookupCache::AddCompletionsToCache(AddCompleteArray& aAddCompletes)
{
for (uint32_t i = 0; i < aAddCompletes.Length(); i++) {
if (mGetHashCache.BinaryIndexOf(aAddCompletes[i].CompleteHash()) == mGetHashCache.NoIndex) {
mGetHashCache.AppendElement(aAddCompletes[i].CompleteHash());
}
}
mGetHashCache.Sort();
return NS_OK;
}
#if defined(DEBUG)
void
LookupCache::DumpCache()
{
if (!LOG_ENABLED())
return;
for (uint32_t i = 0; i < mGetHashCache.Length(); i++) {
nsAutoCString str;
mGetHashCache[i].ToHexString(str);
LOG(("Caches: %s", str.get()));
}
}
#endif
nsresult
LookupCache::WriteFile()
{
@ -152,12 +124,6 @@ LookupCache::ClearAll()
mPrimed = false;
}
void
LookupCache::ClearCache()
{
mGetHashCache.Clear();
}
/* static */ bool
LookupCache::IsCanonicalizedIP(const nsACString& aHost)
{
@ -401,10 +367,12 @@ LookupCacheV2::ClearAll()
nsresult
LookupCacheV2::Has(const Completion& aCompletion,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aHas, uint32_t* aMatchLength,
bool* aFromCache)
bool* aConfirmed, bool* aFromCache)
{
*aHas = *aFromCache = false;
*aHas = *aConfirmed = *aFromCache = false;
*aMatchLength = 0;
uint32_t prefix = aCompletion.ToUint32();
@ -426,30 +394,20 @@ LookupCacheV2::Has(const Completion& aCompletion,
*aFromCache = true;
*aHas = true;
*aMatchLength = COMPLETE_SIZE;
int64_t ageSec; // in seconds
if (aTableFreshness.Get(mTableName, &ageSec)) {
int64_t nowSec = (PR_Now() / PR_USEC_PER_SEC);
MOZ_ASSERT(ageSec <= nowSec);
// Considered completion as unsafe if its table is up-to-date.
*aConfirmed = (nowSec - ageSec) < aFreshnessGuarantee;
}
}
return NS_OK;
}
void
LookupCacheV2::IsHashEntryConfirmed(const Completion& aEntry,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aConfirmed)
{
int64_t age; // in seconds
bool found = aTableFreshness.Get(mTableName, &age);
if (!found) {
*aConfirmed = false;
} else {
int64_t now = (PR_Now() / PR_USEC_PER_SEC);
MOZ_ASSERT(age <= now);
// Considered completion as unsafe if its table is up-to-date.
*aConfirmed = (now - age) < aFreshnessGuarantee;
}
}
bool
LookupCacheV2::IsEmpty()
{
@ -494,6 +452,19 @@ LookupCacheV2::GetPrefixes(FallibleTArray<uint32_t>& aAddPrefixes)
return mPrefixSet->GetPrefixesNative(aAddPrefixes);
}
nsresult
LookupCacheV2::AddCompletionsToCache(AddCompleteArray& aAddCompletes)
{
for (uint32_t i = 0; i < aAddCompletes.Length(); i++) {
if (mGetHashCache.BinaryIndexOf(aAddCompletes[i].CompleteHash()) == mGetHashCache.NoIndex) {
mGetHashCache.AppendElement(aAddCompletes[i].CompleteHash());
}
}
mGetHashCache.Sort();
return NS_OK;
}
nsresult
LookupCacheV2::ReadCompletions()
{
@ -512,6 +483,12 @@ LookupCacheV2::ReadCompletions()
return NS_OK;
}
void
LookupCacheV2::ClearCache()
{
mGetHashCache.Clear();
}
nsresult
LookupCacheV2::ClearPrefixes()
{
@ -592,6 +569,21 @@ LookupCacheV2::ConstructPrefixSet(AddPrefixArray& aAddPrefixes)
}
#if defined(DEBUG)
void
LookupCacheV2::DumpCache()
{
if (!LOG_ENABLED()) {
return;
}
for (uint32_t i = 0; i < mGetHashCache.Length(); i++) {
nsAutoCString str;
mGetHashCache[i].ToHexString(str);
LOG(("Caches: %s", str.get()));
}
}
void
LookupCacheV2::DumpCompletions()
{

45
toolkit/components/url-classifier/LookupCache.h
Просмотреть файл

@ -195,37 +195,31 @@ public:
// be moved away when a backup is made.
nsresult UpdateRootDirHandle(nsIFile* aRootStoreDirectory);
// This will Clear() the passed arrays when done.
nsresult AddCompletionsToCache(AddCompleteArray& aAddCompletes);
// Write data stored in lookup cache to disk.
nsresult WriteFile();
// Clear completions retrieved from gethash request.
void ClearCache();
bool IsPrimed() const { return mPrimed; };
#if DEBUG
void DumpCache();
#endif
virtual nsresult Open();
virtual nsresult Init() = 0;
virtual nsresult ClearPrefixes() = 0;
virtual nsresult Has(const Completion& aCompletion,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aHas, uint32_t* aMatchLength,
bool* aFromCache) = 0;
bool* aConfirmed, bool* aFromCache) = 0;
virtual void IsHashEntryConfirmed(const Completion& aEntry,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aConfirmed) = 0;
// Clear completions retrieved from gethash request.
virtual void ClearCache() = 0;
virtual bool IsEmpty() = 0;
virtual void ClearAll();
#if DEBUG
virtual void DumpCache() = 0;
#endif
template<typename T>
static T* Cast(LookupCache* aThat) {
return ((aThat && T::VER == aThat->Ver()) ? reinterpret_cast<T*>(aThat) : nullptr);
@ -247,9 +241,6 @@ protected:
nsCOMPtr<nsIFile> mRootStoreDirectory;
nsCOMPtr<nsIFile> mStoreDirectory;
// Full length hashes obtained in gethash request
CompletionArray mGetHashCache;
// For gtest to inspect private members.
friend class PerProviderDirectoryTestUtils;
};
@ -265,15 +256,13 @@ public:
virtual nsresult Init() override;
virtual nsresult Open() override;
virtual void ClearCache() override;
virtual void ClearAll() override;
virtual nsresult Has(const Completion& aCompletion,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aHas, uint32_t* aMatchLength,
bool* aFromCache) override;
virtual void IsHashEntryConfirmed(const Completion& aEntry,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aConfirmed) override;
bool* aConfirmed, bool* aFromCache) override;
virtual bool IsEmpty() override;
@ -282,7 +271,12 @@ public:
nsresult GetPrefixes(FallibleTArray<uint32_t>& aAddPrefixes);
// This will Clear() the passed arrays when done.
nsresult AddCompletionsToCache(AddCompleteArray& aAddCompletes);
#if DEBUG
virtual void DumpCache() override;
void DumpCompletions();
#endif
@ -308,6 +302,9 @@ private:
// Set of prefixes known to be in the database
RefPtr<nsUrlClassifierPrefixSet> mPrefixSet;
// Full length hashes obtained in gethash request
CompletionArray mGetHashCache;
};
} // namespace safebrowsing

169
toolkit/components/url-classifier/LookupCacheV4.cpp
Просмотреть файл

@ -80,10 +80,12 @@ LookupCacheV4::Init()
nsresult
LookupCacheV4::Has(const Completion& aCompletion,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aHas, uint32_t* aMatchLength,
bool* aFromCache)
bool* aConfirmed, bool* aFromCache)
{
*aHas = *aFromCache = false;
*aHas = *aConfirmed = *aFromCache = false;
*aMatchLength = 0;
uint32_t length = 0;
@ -93,6 +95,8 @@ LookupCacheV4::Has(const Completion& aCompletion,
nsresult rv = mVLPrefixSet->Matches(fullhash, &length);
NS_ENSURE_SUCCESS(rv, rv);
MOZ_ASSERT(length == 0 || (length >= PREFIX_SIZE && length <= COMPLETE_SIZE));
*aHas = length >= PREFIX_SIZE;
*aMatchLength = length;
@ -102,21 +106,71 @@ LookupCacheV4::Has(const Completion& aCompletion,
prefix, *aHas, length == COMPLETE_SIZE));
}
// TODO : Bug 1311935 - Implement v4 caching
// Check if fullhash match any prefix in the local database
if (!(*aHas)) {
return NS_OK;
}
// We always send 4-bytes for completion(Bug 1323953) so the prefix used to
// lookup for cache should be 4-bytes too.
nsDependentCSubstring prefix(reinterpret_cast<const char*>(aCompletion.buf),
PREFIX_SIZE);
// Check if prefix can be found in cache.
CachedFullHashResponse* fullHashResponse = mCache.Get(prefix);
if (!fullHashResponse) {
return NS_OK;
}
*aFromCache = true;
int64_t nowSec = PR_Now() / PR_USEC_PER_SEC;
int64_t expiryTime;
FullHashExpiryCache& fullHashes = fullHashResponse->fullHashes;
nsDependentCSubstring completion(
reinterpret_cast<const char*>(aCompletion.buf), COMPLETE_SIZE);
// Check if we can find the fullhash in positive cache
if (fullHashes.Get(completion, &expiryTime)) {
if (nowSec <= expiryTime) {
// Url is NOT safe.
*aConfirmed = true;
LOG(("Found a valid fullhash in the positive cache"));
} else {
// Trigger a gethash request in this case(aConfirmed is false).
LOG(("Found an expired fullhash in the positive cache"));
// Remove fullhash entry from the cache when the negative cache
// is also expired because whether or not the fullhash is cached
// locally, we will need to consult the server next time we
// lookup this hash. We may as well remove it from our cache.
if (fullHashResponse->negativeCacheExpirySec < expiryTime) {
fullHashes.Remove(completion);
if (fullHashes.Count() == 0 &&
fullHashResponse->negativeCacheExpirySec < nowSec) {
mCache.Remove(prefix);
}
}
}
return NS_OK;
}
// Check negative cache.
if (fullHashResponse->negativeCacheExpirySec >= nowSec) {
// Url is safe.
LOG(("Found a valid prefix in the negative cache"));
*aHas = false;
} else {
LOG(("Found an expired prefix in the negative cache"));
if (fullHashes.Count() == 0) {
mCache.Remove(prefix);
}
}
return NS_OK;
}
void
LookupCacheV4::IsHashEntryConfirmed(const Completion& aEntry,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aConfirmed)
{
// TODO : Bug 1311935 - Implement v4 caching
*aConfirmed = true;
}
bool
LookupCacheV4::IsEmpty()
{
@ -329,6 +383,17 @@ LookupCacheV4::ApplyUpdate(TableUpdateV4* aTableUpdate,
return NS_OK;
}
nsresult
LookupCacheV4::AddFullHashResponseToCache(const FullHashResponseMap& aResponseMap)
{
for (auto iter = aResponseMap.ConstIter(); !iter.Done(); iter.Next()) {
CachedFullHashResponse* response = mCache.LookupOrAdd(iter.Key());
*response = *(iter.Data());
}
return NS_OK;
}
nsresult
LookupCacheV4::InitCrypto(nsCOMPtr<nsICryptoHash>& aCrypto)
{
@ -539,6 +604,84 @@ LookupCacheV4::LoadMetadata(nsACString& aState, nsACString& aChecksum)
return rv;
}
void
LookupCacheV4::ClearCache()
{
mCache.Clear();
}
// This function remove cache entries whose negative cache time is expired.
// It is possible that a cache entry whose positive cache time is not yet
// expired but still being removed after calling this API. Right now we call
// this on every update.
void
LookupCacheV4::InvalidateExpiredCacheEntry()
{
int64_t nowSec = PR_Now() / PR_USEC_PER_SEC;
for (auto iter = mCache.Iter(); !iter.Done(); iter.Next()) {
CachedFullHashResponse* response = iter.Data();
if (response->negativeCacheExpirySec < nowSec) {
iter.Remove();
}
}
}
#if defined(DEBUG)
static
void CStringToHexString(const nsACString& aIn, nsACString& aOut)
{
static const char* const lut = "0123456789ABCDEF";
// 32 bytes is the longest hash
size_t len = COMPLETE_SIZE;
aOut.SetCapacity(2 * len);
for (size_t i = 0; i < aIn.Length(); ++i) {
const char c = static_cast<const char>(aIn[i]);
aOut.Append(lut[(c >> 4) & 0x0F]);
aOut.Append(lut[c & 15]);
}
}
static
nsCString GetFormattedTimeString(int64_t aCurTimeSec)
{
PRExplodedTime pret;
PR_ExplodeTime(aCurTimeSec * PR_USEC_PER_SEC, PR_GMTParameters, &pret);
return nsPrintfCString(
"%04d-%02d-%02d %02d:%02d:%02d UTC",
pret.tm_year, pret.tm_month + 1, pret.tm_mday,
pret.tm_hour, pret.tm_min, pret.tm_sec);
}
void
LookupCacheV4::DumpCache()
{
if (!LOG_ENABLED()) {
return;
}
for (auto iter = mCache.ConstIter(); !iter.Done(); iter.Next()) {
nsAutoCString strPrefix;
CStringToHexString(iter.Key(), strPrefix);
CachedFullHashResponse* response = iter.Data();
LOG(("Caches prefix: %s, Expire time: %s",
strPrefix.get(),
GetFormattedTimeString(response->negativeCacheExpirySec).get()));
FullHashExpiryCache& fullHashes = response->fullHashes;
for (auto iter2 = fullHashes.ConstIter(); !iter2.Done(); iter2.Next()) {
nsAutoCString strFullhash;
CStringToHexString(iter2.Key(), strFullhash);
LOG((" - %s, Expire time: %s", strFullhash.get(),
GetFormattedTimeString(iter2.Data()).get()));
}
}
}
#endif
VLPrefixSet::VLPrefixSet(const PrefixStringMap& aMap)
: mCount(0)
{

19
toolkit/components/url-classifier/LookupCacheV4.h
Просмотреть файл

@ -25,13 +25,16 @@ public:
virtual nsresult Init() override;
virtual nsresult Has(const Completion& aCompletion,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aHas, uint32_t* aMatchLength,
bool* aFromCache) override;
bool* aConfirmed, bool* aFromCache) override;
virtual void IsHashEntryConfirmed(const Completion& aEntry,
const TableFreshnessMap& aTableFreshness,
uint32_t aFreshnessGuarantee,
bool* aConfirmed) override;
virtual void ClearCache() override;
#if DEBUG
virtual void DumpCache() override;
#endif
virtual bool IsEmpty() override;
@ -45,9 +48,13 @@ public:
PrefixStringMap& aInputMap,
PrefixStringMap& aOutputMap);
nsresult AddFullHashResponseToCache(const FullHashResponseMap& aResponseMap);
nsresult WriteMetadata(TableUpdateV4* aTableUpdate);
nsresult LoadMetadata(nsACString& aState, nsACString& aChecksum);
void InvalidateExpiredCacheEntry();
static const int VER;
protected:
@ -63,6 +70,8 @@ private:
nsresult VerifyChecksum(const nsACString& aChecksum);
RefPtr<VariableLengthPrefixSet> mVLPrefixSet;
FullHashResponseMap mCache;
};
} // namespace safebrowsing

1
toolkit/components/url-classifier/VariableLengthPrefixSet.cpp
Просмотреть файл

@ -188,6 +188,7 @@ VariableLengthPrefixSet::Matches(const nsACString& aFullHash, uint32_t* aLength)
for (auto iter = mVLPrefixSet.ConstIter(); !iter.Done(); iter.Next()) {
if (BinarySearch(aFullHash, *iter.Data(), iter.Key())) {
*aLength = iter.Key();
MOZ_ASSERT(*aLength > 4);
return NS_OK;
}
}

9
toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp
Просмотреть файл

@ -47,13 +47,18 @@ TestHasPrefix(const _Fragment& aFragment, bool aExpectedHas, bool aExpectedCompl
nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
lookupHash.FromPlaintext(aFragment, cryptoHash);
bool has, fromCache;
bool has, confirmed, fromCache;
uint32_t matchLength;
nsresult rv = cache->Has(lookupHash, &has, &matchLength, &fromCache);
// Freshness is not used in V4 so we just put dummy values here.
TableFreshnessMap dummy;
nsresult rv = cache->Has(lookupHash, dummy, 0,
&has, &matchLength, &confirmed, &fromCache);
EXPECT_EQ(rv, NS_OK);
EXPECT_EQ(has, aExpectedHas);
EXPECT_EQ(matchLength == COMPLETE_SIZE, aExpectedComplete);
EXPECT_EQ(confirmed, false);
EXPECT_EQ(fromCache, false);
cache->ClearAll();
});