mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1353956 - P3. Separate file processing and prefix data processing for SafeBrowsing prefix set. r=gcp 

SafeBrowsing prefix files LOAD/SAVE operations are handled in xxxPrefixSet.cpp.
It would be more clear if xxxPrefixSet.cpp only processes prefix data,
while LookupCacheV2/LookupCacheV4 which use prefix set process file.

This patch doesn't change any behavior, testcases need to update because
the LookupCache & xxxPrefixSet APIs are changed.

Differential Revision: https://phabricator.services.mozilla.com/D21462

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Dimi Lee 2019-03-04 21:22:46 +00:00
Родитель d253a723f7
Коммит b8f59ed8b6
12 изменённых файлов: 389 добавлений и 416 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

66
toolkit/components/url-classifier/LookupCache.cpp
Просмотреть файл

@ -651,11 +651,73 @@ nsresult LookupCacheV2::ClearPrefixes() {
}
nsresult LookupCacheV2::StoreToFile(nsCOMPtr<nsIFile>& aFile) {
return mPrefixSet->StoreToFile(aFile);
nsCOMPtr<nsIOutputStream> localOutFile;
nsresult rv =
NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
NS_ENSURE_SUCCESS(rv, rv);
uint32_t fileSize;
// Preallocate the file storage
{
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FALLOCATE_TIME> timer;
fileSize = mPrefixSet->CalculatePreallocateSize();
// Ignore failure, the preallocation is a hint and we write out the entire
// file later on
Unused << fos->Preallocate(fileSize);
}
// Convert to buffered stream
nsCOMPtr<nsIOutputStream> out;
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
std::min(fileSize, MAX_BUFFER_SIZE));
NS_ENSURE_SUCCESS(rv, rv);
rv = mPrefixSet->WritePrefixes(out);
NS_ENSURE_SUCCESS(rv, rv);
LOG(("[%s] Storing PrefixSet successful", mTableName.get()));
return NS_OK;
}
nsresult LookupCacheV2::LoadFromFile(nsCOMPtr<nsIFile>& aFile) {
return mPrefixSet->LoadFromFile(aFile);
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FILELOAD_TIME> timer;
nsCOMPtr<nsIInputStream> localInFile;
nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile,
PR_RDONLY | nsIFile::OS_READAHEAD);
NS_ENSURE_SUCCESS(rv, rv);
// Calculate how big the file is, make sure our read buffer isn't bigger
// than the file itself which is just wasting memory.
int64_t fileSize;
rv = aFile->GetFileSize(&fileSize);
NS_ENSURE_SUCCESS(rv, rv);
if (fileSize < 0 || fileSize > UINT32_MAX) {
return NS_ERROR_FAILURE;
}
uint32_t bufferSize =
std::min<uint32_t>(static_cast<uint32_t>(fileSize), MAX_BUFFER_SIZE);
// Convert to buffered stream
nsCOMPtr<nsIInputStream> in;
rv = NS_NewBufferedInputStream(getter_AddRefs(in), localInFile.forget(),
bufferSize);
NS_ENSURE_SUCCESS(rv, rv);
rv = mPrefixSet->LoadPrefixes(in);
NS_ENSURE_SUCCESS(rv, rv);
mPrimed = true;
LOG(("[%s] Loading PrefixSet successful", mTableName.get()));
return NS_OK;
}
size_t LookupCacheV2::SizeOfPrefixSet() const {

12
toolkit/components/url-classifier/LookupCache.h
Просмотреть файл

@ -231,6 +231,9 @@ class LookupCache {
virtual nsresult Has(const Completion& aCompletion, bool* aHas,
uint32_t* aMatchLength, bool* aConfirmed) = 0;
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) = 0;
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) = 0;
virtual bool IsEmpty() const = 0;
virtual void ClearAll();
@ -250,8 +253,6 @@ class LookupCache {
private:
nsresult LoadPrefixSet();
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) = 0;
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) = 0;
virtual size_t SizeOfPrefixSet() const = 0;
virtual int Ver() const = 0;
@ -259,6 +260,8 @@ class LookupCache {
protected:
virtual ~LookupCache() {}
static const uint32_t MAX_BUFFER_SIZE = 64 * 1024;
// Check completions in positive cache and prefix in negative cache.
// 'aHas' and 'aConfirmed' are output parameters.
nsresult CheckCache(const Completion& aCompletion, bool* aHas,
@ -292,6 +295,9 @@ class LookupCacheV2 final : public LookupCache {
virtual nsresult Has(const Completion& aCompletion, bool* aHas,
uint32_t* aMatchLength, bool* aConfirmed) override;
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) override;
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) override;
virtual bool IsEmpty() const override;
nsresult Build(AddPrefixArray& aAddPrefixes, AddCompleteArray& aAddCompletes);
@ -314,8 +320,6 @@ class LookupCacheV2 final : public LookupCache {
nsresult ReadCompletions();
virtual nsresult ClearPrefixes() override;
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) override;
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) override;
virtual size_t SizeOfPrefixSet() const override;
private:

67
toolkit/components/url-classifier/LookupCacheV4.cpp
Просмотреть файл

@ -164,11 +164,74 @@ nsresult LookupCacheV4::ClearPrefixes() {
}
nsresult LookupCacheV4::StoreToFile(nsCOMPtr<nsIFile>& aFile) {
return mVLPrefixSet->StoreToFile(aFile);
NS_ENSURE_ARG_POINTER(aFile);
nsCOMPtr<nsIOutputStream> localOutFile;
nsresult rv =
NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
NS_ENSURE_SUCCESS(rv, rv);
uint32_t fileSize = 0;
// Preallocate the file storage
{
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FALLOCATE_TIME> timer;
fileSize = mVLPrefixSet->CalculatePreallocateSize();
Unused << fos->Preallocate(fileSize);
}
// Convert to buffered stream
nsCOMPtr<nsIOutputStream> out;
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
std::min(fileSize, MAX_BUFFER_SIZE));
NS_ENSURE_SUCCESS(rv, rv);
rv = mVLPrefixSet->WritePrefixes(out);
NS_ENSURE_SUCCESS(rv, rv);
LOG(("[%s] Storing PrefixSet successful", mTableName.get()));
return NS_OK;
}
nsresult LookupCacheV4::LoadFromFile(nsCOMPtr<nsIFile>& aFile) {
return mVLPrefixSet->LoadFromFile(aFile);
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FILELOAD_TIME> timer;
nsCOMPtr<nsIInputStream> localInFile;
nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile,
PR_RDONLY | nsIFile::OS_READAHEAD);
NS_ENSURE_SUCCESS(rv, rv);
// Calculate how big the file is, make sure our read buffer isn't bigger
// than the file itself which is just wasting memory.
int64_t fileSize;
rv = aFile->GetFileSize(&fileSize);
NS_ENSURE_SUCCESS(rv, rv);
if (fileSize < 0 || fileSize > UINT32_MAX) {
return NS_ERROR_FAILURE;
}
uint32_t bufferSize =
std::min<uint32_t>(static_cast<uint32_t>(fileSize), MAX_BUFFER_SIZE);
// Convert to buffered stream
nsCOMPtr<nsIInputStream> in;
rv = NS_NewBufferedInputStream(getter_AddRefs(in), localInFile.forget(),
bufferSize);
NS_ENSURE_SUCCESS(rv, rv);
rv = mVLPrefixSet->LoadPrefixes(in);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
mPrimed = true;
LOG(("[%s] Loading PrefixSet successful", mTableName.get()));
return NS_OK;
}
size_t LookupCacheV4::SizeOfPrefixSet() const {

5
toolkit/components/url-classifier/LookupCacheV4.h
Просмотреть файл

@ -25,6 +25,9 @@ class LookupCacheV4 final : public LookupCache {
virtual nsresult Has(const Completion& aCompletion, bool* aHas,
uint32_t* aMatchLength, bool* aConfirmed) override;
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) override;
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) override;
virtual bool IsEmpty() const override;
nsresult Build(PrefixStringMap& aPrefixMap);
@ -47,8 +50,6 @@ class LookupCacheV4 final : public LookupCache {
protected:
virtual nsresult ClearPrefixes() override;
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) override;
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) override;
virtual size_t SizeOfPrefixSet() const override;
private:

102
toolkit/components/url-classifier/VariableLengthPrefixSet.cpp
Просмотреть файл

@ -204,91 +204,18 @@ nsresult VariableLengthPrefixSet::IsEmpty(bool* aEmpty) const {
return NS_OK;
}
nsresult VariableLengthPrefixSet::LoadFromFile(nsCOMPtr<nsIFile>& aFile) {
MutexAutoLock lock(mLock);
NS_ENSURE_ARG_POINTER(aFile);
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FILELOAD_TIME> timer;
nsCOMPtr<nsIInputStream> localInFile;
nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile,
PR_RDONLY | nsIFile::OS_READAHEAD);
NS_ENSURE_SUCCESS(rv, rv);
// Calculate how big the file is, make sure our read buffer isn't bigger
// than the file itself which is just wasting memory.
int64_t fileSize;
rv = aFile->GetFileSize(&fileSize);
NS_ENSURE_SUCCESS(rv, rv);
if (fileSize < 0 || fileSize > UINT32_MAX) {
return NS_ERROR_FAILURE;
}
uint32_t bufferSize =
std::min<uint32_t>(static_cast<uint32_t>(fileSize), MAX_BUFFER_SIZE);
// Convert to buffered stream
nsCOMPtr<nsIInputStream> in;
rv = NS_NewBufferedInputStream(getter_AddRefs(in), localInFile.forget(),
bufferSize);
NS_ENSURE_SUCCESS(rv, rv);
rv = mFixedPrefixSet->LoadPrefixes(in);
NS_ENSURE_SUCCESS(rv, rv);
rv = LoadPrefixes(in);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
;
}
nsresult VariableLengthPrefixSet::StoreToFile(nsCOMPtr<nsIFile>& aFile) const {
NS_ENSURE_ARG_POINTER(aFile);
MutexAutoLock lock(mLock);
nsCOMPtr<nsIOutputStream> localOutFile;
nsresult rv =
NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
NS_ENSURE_SUCCESS(rv, rv);
uint32_t fileSize = 0;
// Preallocate the file storage
{
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FALLOCATE_TIME> timer;
fileSize += mFixedPrefixSet->CalculatePreallocateSize();
fileSize += CalculatePreallocateSize();
Unused << fos->Preallocate(fileSize);
}
// Convert to buffered stream
nsCOMPtr<nsIOutputStream> out;
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
std::min(fileSize, MAX_BUFFER_SIZE));
NS_ENSURE_SUCCESS(rv, rv);
rv = mFixedPrefixSet->WritePrefixes(out);
NS_ENSURE_SUCCESS(rv, rv);
rv = WritePrefixes(out);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
nsresult VariableLengthPrefixSet::LoadPrefixes(nsCOMPtr<nsIInputStream>& in) {
MutexAutoLock lock(mLock);
// First read prefixes from fixed-length prefix set
nsresult rv = mFixedPrefixSet->LoadPrefixes(in);
NS_ENSURE_SUCCESS(rv, rv);
// Then read prefixes from variable-length prefix set
uint32_t magic;
uint32_t read;
nsresult rv =
in->Read(reinterpret_cast<char*>(&magic), sizeof(uint32_t), &read);
rv = in->Read(reinterpret_cast<char*>(&magic), sizeof(uint32_t), &read);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE);
@ -349,6 +276,10 @@ nsresult VariableLengthPrefixSet::LoadPrefixes(nsCOMPtr<nsIInputStream>& in) {
uint32_t VariableLengthPrefixSet::CalculatePreallocateSize() const {
uint32_t fileSize = 0;
// Size of fixed length prefix set.
fileSize += mFixedPrefixSet->CalculatePreallocateSize();
// Size of variable length prefix set.
// Store how many prefix string.
fileSize += sizeof(uint32_t);
@ -363,10 +294,17 @@ uint32_t VariableLengthPrefixSet::CalculatePreallocateSize() const {
nsresult VariableLengthPrefixSet::WritePrefixes(
nsCOMPtr<nsIOutputStream>& out) const {
MutexAutoLock lock(mLock);
// First, write fixed length prefix set
nsresult rv = mFixedPrefixSet->WritePrefixes(out);
NS_ENSURE_SUCCESS(rv, rv);
// Then, write variable length prefix set
uint32_t written;
uint32_t writelen = sizeof(uint32_t);
uint32_t magic = PREFIXSET_VERSION_MAGIC;
nsresult rv = out->Write(reinterpret_cast<char*>(&magic), writelen, &written);
rv = out->Write(reinterpret_cast<char*>(&magic), writelen, &written);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE);

10
toolkit/components/url-classifier/VariableLengthPrefixSet.h
Просмотреть файл

@ -30,8 +30,10 @@ class VariableLengthPrefixSet final : public nsIMemoryReporter {
nsresult GetFixedLengthPrefixes(FallibleTArray<uint32_t>& aPrefixes);
nsresult Matches(const nsACString& aFullHash, uint32_t* aLength) const;
nsresult IsEmpty(bool* aEmpty) const;
nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile);
nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) const;
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
uint32_t CalculatePreallocateSize() const;
size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
@ -47,10 +49,6 @@ class VariableLengthPrefixSet final : public nsIMemoryReporter {
bool BinarySearch(const nsACString& aFullHash, const nsACString& aPrefixes,
uint32_t aPrefixSize) const;
uint32_t CalculatePreallocateSize() const;
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
// Lock to prevent races between the url-classifier thread (which does most
// of the operations) and the main thread (which does memory reporting).
// It should be held for all operations between Init() and destruction that

2
toolkit/components/url-classifier/nsIUrlClassifierPrefixSet.idl
Просмотреть файл

@ -24,6 +24,4 @@ interface nsIUrlClassifierPrefixSet : nsISupports
// Do a lookup in the PrefixSet, return whether the value is present.
boolean contains(in unsigned long aPrefix);
boolean isEmpty();
void loadFromFile(in nsIFile aFile);
void storeToFile(in nsIFile aFile);
};

82
toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp
Просмотреть файл

@ -12,16 +12,13 @@
#include "nsPrintfCString.h"
#include "nsTArray.h"
#include "nsString.h"
#include "nsIFile.h"
#include "nsTArray.h"
#include "nsThreadUtils.h"
#include "nsNetUtil.h"
#include "nsISeekableStream.h"
#include "nsIBufferedStreams.h"
#include "nsIFileStreams.h"
#include "mozilla/MemoryReporting.h"
#include "mozilla/Telemetry.h"
#include "mozilla/FileUtils.h"
#include "mozilla/Logging.h"
#include "mozilla/Unused.h"
#include <algorithm>
@ -345,83 +342,10 @@ nsUrlClassifierPrefixSet::IsEmpty(bool* aEmpty) {
return NS_OK;
}
NS_IMETHODIMP
nsUrlClassifierPrefixSet::LoadFromFile(nsIFile* aFile) {
MutexAutoLock lock(mLock);
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FILELOAD_TIME> timer;
nsCOMPtr<nsIInputStream> localInFile;
nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile,
PR_RDONLY | nsIFile::OS_READAHEAD);
NS_ENSURE_SUCCESS(rv, rv);
// Calculate how big the file is, make sure our read buffer isn't bigger
// than the file itself which is just wasting memory.
int64_t fileSize;
rv = aFile->GetFileSize(&fileSize);
NS_ENSURE_SUCCESS(rv, rv);
if (fileSize < 0 || fileSize > UINT32_MAX) {
return NS_ERROR_FAILURE;
}
uint32_t bufferSize =
std::min<uint32_t>(static_cast<uint32_t>(fileSize), MAX_BUFFER_SIZE);
// Convert to buffered stream
nsCOMPtr<nsIInputStream> in;
rv = NS_NewBufferedInputStream(getter_AddRefs(in), localInFile.forget(),
bufferSize);
NS_ENSURE_SUCCESS(rv, rv);
rv = LoadPrefixes(in);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
NS_IMETHODIMP
nsUrlClassifierPrefixSet::StoreToFile(nsIFile* aFile) {
MutexAutoLock lock(mLock);
nsCOMPtr<nsIOutputStream> localOutFile;
nsresult rv =
NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
NS_ENSURE_SUCCESS(rv, rv);
uint32_t fileSize;
// Preallocate the file storage
{
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FALLOCATE_TIME> timer;
fileSize = CalculatePreallocateSize();
// Ignore failure, the preallocation is a hint and we write out the entire
// file later on
Unused << fos->Preallocate(fileSize);
}
// Convert to buffered stream
nsCOMPtr<nsIOutputStream> out;
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
std::min(fileSize, MAX_BUFFER_SIZE));
NS_ENSURE_SUCCESS(rv, rv);
rv = WritePrefixes(out);
NS_ENSURE_SUCCESS(rv, rv);
LOG(("[%s] Storing PrefixSet successful", mName.get()));
return NS_OK;
}
nsresult nsUrlClassifierPrefixSet::LoadPrefixes(nsCOMPtr<nsIInputStream>& in) {
mCanary.Check();
MutexAutoLock lock(mLock);
mCanary.Check();
Clear();
uint32_t magic;
@ -518,6 +442,8 @@ uint32_t nsUrlClassifierPrefixSet::CalculatePreallocateSize() const {
nsresult nsUrlClassifierPrefixSet::WritePrefixes(
nsCOMPtr<nsIOutputStream>& out) const {
MutexAutoLock lock(mLock);
mCanary.Check();
// In Bug 1362761, crashes happened while reading mIndexDeltas[i].

8
toolkit/components/url-classifier/nsUrlClassifierPrefixSet.h
Просмотреть файл

@ -39,10 +39,11 @@ class nsUrlClassifierPrefixSet final : public nsIUrlClassifierPrefixSet,
NS_IMETHOD GetPrefixes(uint32_t* aCount, uint32_t** aPrefixes) override;
NS_IMETHOD Contains(uint32_t aPrefix, bool* aFound) override;
NS_IMETHOD IsEmpty(bool* aEmpty) override;
NS_IMETHOD LoadFromFile(nsIFile* aFile) override;
NS_IMETHOD StoreToFile(nsIFile* aFile) override;
nsresult GetPrefixesNative(FallibleTArray<uint32_t>& outArray);
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
uint32_t CalculatePreallocateSize() const;
size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
@ -63,9 +64,6 @@ class nsUrlClassifierPrefixSet final : public nsIUrlClassifierPrefixSet,
nsresult MakePrefixSet(const uint32_t* aArray, uint32_t aLength);
uint32_t BinSearch(uint32_t start, uint32_t end, uint32_t target) const;
bool IsEmptyInternal() const;
uint32_t CalculatePreallocateSize() const;
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
// Lock to prevent races between the url-classifier thread (which does most
// of the operations) and the main thread (which does memory reporting).

17
toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp
Просмотреть файл

@ -143,16 +143,17 @@ static void VerifyPrefixSet(PrefixStringMap& expected) {
// Verify the prefix set is written to disk.
nsCOMPtr<nsIFile> file;
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->AppendNative(GTEST_SAFEBROWSING_DIR);
file->AppendNative(GTEST_PREFIXFILE);
RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet;
load->Init(GTEST_TABLE);
RefPtr<LookupCacheV4> lookup =
new LookupCacheV4(GTEST_TABLE, NS_LITERAL_CSTRING("test"), file);
lookup->Init();
file->AppendNative(GTEST_PREFIXFILE);
lookup->LoadFromFile(file);
PrefixStringMap prefixesInFile;
load->LoadFromFile(file);
load->GetPrefixes(prefixesInFile);
lookup->GetPrefixes(prefixesInFile);
for (auto iter = expected.ConstIter(); !iter.Done(); iter.Next()) {
nsCString* expectedPrefix = iter.Data();
@ -233,7 +234,7 @@ static void testOpenLookupCache() {
}
// Tests start from here.
TEST(UrlClassifierTableUpdateV4, FixLenghtPSetFullUpdate) {
TEST(UrlClassifierTableUpdateV4, FixLengthPSetFullUpdate) {
srand(time(NULL));
_PrefixArray array;
@ -249,7 +250,7 @@ TEST(UrlClassifierTableUpdateV4, FixLenghtPSetFullUpdate) {
Clear();
}
TEST(UrlClassifierTableUpdateV4, VariableLenghtPSetFullUpdate) {
TEST(UrlClassifierTableUpdateV4, VariableLengthPSetFullUpdate) {
_PrefixArray array;
PrefixStringMap map;
nsCString sha256;

372
toolkit/components/url-classifier/tests/gtest/TestVariableLengthPrefixSet.cpp
Просмотреть файл

@ -17,11 +17,11 @@ typedef nsCString _Prefix;
typedef nsTArray<_Prefix> _PrefixArray;
// Create fullhash by appending random characters.
static nsCString* CreateFullHash(const nsACString& in) {
nsCString* out = new nsCString(in);
out->SetLength(32);
static nsCString CreateFullHash(const nsACString& in) {
nsCString out(in);
out.SetLength(32);
for (size_t i = in.Length(); i < 32; i++) {
out->SetCharAt(char(rand() % 256), i);
out.SetCharAt(char(rand() % 256), i);
}
return out;
@ -55,10 +55,9 @@ static void RandomPrefixes(uint32_t N, uint32_t MIN, uint32_t MAX,
}
}
static void CheckContent(VariableLengthPrefixSet* pset,
PrefixStringMap& expected) {
static void CheckContent(LookupCacheV4* cache, PrefixStringMap& expected) {
PrefixStringMap vlPSetMap;
pset->GetPrefixes(vlPSetMap);
cache->GetPrefixes(vlPSetMap);
for (auto iter = vlPSetMap.Iter(); !iter.Done(); iter.Next()) {
nsCString* expectedPrefix = expected.Get(iter.Key());
@ -71,15 +70,16 @@ static void CheckContent(VariableLengthPrefixSet* pset,
// This test loops through all the prefixes and converts each prefix to
// fullhash by appending random characters, each converted fullhash
// should at least match its original length in the prefixSet.
static void DoExpectedLookup(VariableLengthPrefixSet* pset,
_PrefixArray& array) {
static void DoExpectedLookup(LookupCacheV4* cache, _PrefixArray& array) {
uint32_t matchLength = 0;
for (uint32_t i = 0; i < array.Length(); i++) {
const nsCString& prefix = array[i];
UniquePtr<nsCString> fullhash(CreateFullHash(prefix));
Completion complete;
complete.Assign(CreateFullHash(prefix));
// Find match for prefix-generated full hash
pset->Matches(*fullhash, &matchLength);
bool has, confirmed;
cache->Has(complete, &has, &matchLength, &confirmed);
MOZ_ASSERT(matchLength != 0);
if (matchLength != prefix.Length()) {
@ -93,8 +93,7 @@ static void DoExpectedLookup(VariableLengthPrefixSet* pset,
continue;
}
if (0 == memcmp(fullhash->BeginReading(), array[j].BeginReading(),
matchLength)) {
if (0 == memcmp(complete.buf, array[j].BeginReading(), matchLength)) {
found = true;
break;
}
@ -104,7 +103,7 @@ static void DoExpectedLookup(VariableLengthPrefixSet* pset,
}
}
static void DoRandomLookup(VariableLengthPrefixSet* pset, uint32_t N,
static void DoRandomLookup(LookupCacheV4* cache, uint32_t N,
_PrefixArray& array) {
for (uint32_t i = 0; i < N; i++) {
// Random 32-bytes test fullhash
@ -122,8 +121,11 @@ static void DoRandomLookup(VariableLengthPrefixSet* pset, uint32_t N,
}
}
Completion complete;
complete.Assign(nsDependentCSubstring(buf, 32));
bool has, confirmed;
uint32_t matchLength = 0;
pset->Matches(nsDependentCSubstring(buf, 32), &matchLength);
cache->Has(complete, &has, &matchLength, &confirmed);
ASSERT_TRUE(expected.IsEmpty() ? !matchLength
: expected.Contains(matchLength));
@ -168,12 +170,23 @@ static void SetupPrefixMap(const _PrefixArray& array, PrefixStringMap& map) {
}
}
static already_AddRefed<LookupCacheV4> SetupLookupCache(
const nsACString& aName) {
nsCOMPtr<nsIFile> rootDir;
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(rootDir));
nsAutoCString provider("test");
RefPtr<LookupCacheV4> lookup = new LookupCacheV4(aName, provider, rootDir);
lookup->Init();
return lookup.forget();
}
// Test setting prefix set with only 4-bytes prefixes
TEST(UrlClassifierVLPrefixSet, FixedLengthSet) {
srand(time(nullptr));
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
PrefixStringMap map;
_PrefixArray array = {
@ -182,13 +195,11 @@ TEST(UrlClassifierVLPrefixSet, FixedLengthSet) {
};
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
DoExpectedLookup(cache, array);
DoRandomLookup(cache, 1000, array);
CheckContent(cache, map);
// Run random test
array.Clear();
@ -197,19 +208,16 @@ TEST(UrlClassifierVLPrefixSet, FixedLengthSet) {
RandomPrefixes(1500, 4, 4, array);
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
DoExpectedLookup(cache, array);
DoRandomLookup(cache, 1000, array);
CheckContent(cache, map);
}
// Test setting prefix set with only 5~32 bytes prefixes
TEST(UrlClassifierVLPrefixSet, VariableLengthSet) {
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
PrefixStringMap map;
_PrefixArray array = {
@ -223,13 +231,11 @@ TEST(UrlClassifierVLPrefixSet, VariableLengthSet) {
_Prefix("yankee"), _Prefix("ZuluZuluZuluZulu")};
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
DoExpectedLookup(cache, array);
DoRandomLookup(cache, 1000, array);
CheckContent(cache, map);
// Run random test
array.Clear();
@ -238,19 +244,16 @@ TEST(UrlClassifierVLPrefixSet, VariableLengthSet) {
RandomPrefixes(1500, 5, 32, array);
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
DoExpectedLookup(cache, array);
DoRandomLookup(cache, 1000, array);
CheckContent(cache, map);
}
// Test setting prefix set with both 4-bytes prefixes and 5~32 bytes prefixes
TEST(UrlClassifierVLPrefixSet, MixedPrefixSet) {
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
PrefixStringMap map;
_PrefixArray array = {_Prefix("enus"),
@ -273,13 +276,11 @@ TEST(UrlClassifierVLPrefixSet, MixedPrefixSet) {
_Prefix("Stheno, Euryale and Medusa")};
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
DoExpectedLookup(cache, array);
DoRandomLookup(cache, 1000, array);
CheckContent(cache, map);
// Run random test
array.Clear();
@ -288,19 +289,16 @@ TEST(UrlClassifierVLPrefixSet, MixedPrefixSet) {
RandomPrefixes(1500, 4, 32, array);
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
DoExpectedLookup(cache, array);
DoRandomLookup(cache, 1000, array);
CheckContent(cache, map);
}
// Test resetting prefix set
TEST(UrlClassifierVLPrefixSet, ResetPrefix) {
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
// First prefix set
_PrefixArray array1 = {
@ -313,9 +311,9 @@ TEST(UrlClassifierVLPrefixSet, ResetPrefix) {
PrefixStringMap map;
SetupPrefixMap(array1, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array1);
DoExpectedLookup(cache, array1);
}
// Second
@ -329,25 +327,28 @@ TEST(UrlClassifierVLPrefixSet, ResetPrefix) {
PrefixStringMap map;
SetupPrefixMap(array2, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array2);
DoExpectedLookup(cache, array2);
}
// Should not match any of the first prefix set
uint32_t matchLength = 0;
for (uint32_t i = 0; i < array1.Length(); i++) {
UniquePtr<nsACString> fullhash(CreateFullHash(array1[i]));
Completion complete;
complete.Assign(CreateFullHash(array1[i]));
// Find match for prefix-generated full hash
bool has, confirmed;
cache->Has(complete, &has, &matchLength, &confirmed);
pset->Matches(*fullhash, &matchLength);
ASSERT_TRUE(matchLength == 0);
}
}
// Test only set one 4-bytes prefix and one full-length prefix
TEST(UrlClassifierVLPrefixSet, TinyPrefixSet) {
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
PrefixStringMap map;
_PrefixArray array = {
@ -356,52 +357,47 @@ TEST(UrlClassifierVLPrefixSet, TinyPrefixSet) {
};
SetupPrefixMap(array, map);
pset->SetPrefixes(map);
cache->Build(map);
DoExpectedLookup(pset, array);
DoRandomLookup(pset, 1000, array);
CheckContent(pset, map);
DoExpectedLookup(cache, array);
DoRandomLookup(cache, 1000, array);
CheckContent(cache, map);
}
// Test empty prefix set and IsEmpty function
TEST(UrlClassifierVLPrefixSet, EmptyPrefixSet) {
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
bool empty;
pset->IsEmpty(&empty);
bool empty = cache->IsEmpty();
ASSERT_TRUE(empty);
PrefixStringMap map;
_PrefixArray array1;
// Lookup an empty array should never match
DoRandomLookup(pset, 100, array1);
DoRandomLookup(cache, 100, array1);
// Insert an 4-bytes prefix, then IsEmpty should return false
_PrefixArray array2 = {_Prefix("test")};
SetupPrefixMap(array2, map);
pset->SetPrefixes(map);
cache->Build(map);
pset->IsEmpty(&empty);
empty = cache->IsEmpty();
ASSERT_TRUE(!empty);
_PrefixArray array3 = {_Prefix("test variable length")};
// Insert an 5~32 bytes prefix, then IsEmpty should return false
SetupPrefixMap(array3, map);
pset->SetPrefixes(map);
cache->Build(map);
pset->IsEmpty(&empty);
empty = cache->IsEmpty();
ASSERT_TRUE(!empty);
}
// Test prefix size should only between 4~32 bytes
TEST(UrlClassifierVLPrefixSet, MinMaxPrefixSet) {
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
pset->Init(NS_LITERAL_CSTRING("test"));
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
PrefixStringMap map;
{
@ -409,7 +405,7 @@ TEST(UrlClassifierVLPrefixSet, MinMaxPrefixSet) {
_Prefix("1aaa2bbb3ccc4ddd5eee6fff7ggg8hhh")};
SetupPrefixMap(array, map);
nsresult rv = pset->SetPrefixes(map);
nsresult rv = cache->Build(map);
ASSERT_TRUE(rv == NS_OK);
}
@ -418,7 +414,7 @@ TEST(UrlClassifierVLPrefixSet, MinMaxPrefixSet) {
_PrefixArray array = {_Prefix("123")};
SetupPrefixMap(array, map);
nsresult rv = pset->SetPrefixes(map);
nsresult rv = cache->Build(map);
ASSERT_TRUE(NS_FAILED(rv));
}
@ -427,123 +423,171 @@ TEST(UrlClassifierVLPrefixSet, MinMaxPrefixSet) {
_PrefixArray array = {_Prefix("1aaa2bbb3ccc4ddd5eee6fff7ggg8hhh9")};
SetupPrefixMap(array, map);
nsresult rv = pset->SetPrefixes(map);
nsresult rv = cache->Build(map);
ASSERT_TRUE(NS_FAILED(rv));
}
}
// Test save then load prefix set with only 4-bytes prefixes
TEST(UrlClassifierVLPrefixSet, LoadSaveFixedLengthPrefixSet) {
RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet;
save->Init(NS_LITERAL_CSTRING("test-save"));
_PrefixArray array;
RandomPrefixes(10000, 4, 4, array);
PrefixStringMap map;
SetupPrefixMap(array, map);
save->SetPrefixes(map);
DoExpectedLookup(save, array);
DoRandomLookup(save, 1000, array);
CheckContent(save, map);
nsCOMPtr<nsIFile> file;
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->Append(NS_LITERAL_STRING("test.vlpset"));
_PrefixArray array;
PrefixStringMap map;
save->StoreToFile(file);
// Save
{
RefPtr<LookupCacheV4> save =
SetupLookupCache(NS_LITERAL_CSTRING("test-save"));
RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet;
load->Init(NS_LITERAL_CSTRING("test-load"));
RandomPrefixes(10000, 4, 4, array);
load->LoadFromFile(file);
SetupPrefixMap(array, map);
save->Build(map);
DoExpectedLookup(load, array);
DoExpectedLookup(save, array);
DoRandomLookup(save, 1000, array);
CheckContent(save, map);
DoRandomLookup(load, 1000, array);
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->Append(NS_LITERAL_STRING("test.vlpset"));
save->StoreToFile(file);
}
CheckContent(load, map);
// Load
{
RefPtr<LookupCacheV4> load =
SetupLookupCache(NS_LITERAL_CSTRING("test-load"));
load->LoadFromFile(file);
DoExpectedLookup(load, array);
DoRandomLookup(load, 1000, array);
CheckContent(load, map);
}
file->Remove(false);
}
// Test save then load prefix set with only 5~32 bytes prefixes
TEST(UrlClassifierVLPrefixSet, LoadSaveVariableLengthPrefixSet) {
RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet;
save->Init(NS_LITERAL_CSTRING("test-save"));
_PrefixArray array;
RandomPrefixes(10000, 5, 32, array);
PrefixStringMap map;
SetupPrefixMap(array, map);
save->SetPrefixes(map);
DoExpectedLookup(save, array);
DoRandomLookup(save, 1000, array);
CheckContent(save, map);
nsCOMPtr<nsIFile> file;
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->Append(NS_LITERAL_STRING("test.vlpset"));
_PrefixArray array;
PrefixStringMap map;
save->StoreToFile(file);
// Save
{
RefPtr<LookupCacheV4> save =
SetupLookupCache(NS_LITERAL_CSTRING("test-save"));
RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet;
load->Init(NS_LITERAL_CSTRING("test-load"));
RandomPrefixes(10000, 5, 32, array);
load->LoadFromFile(file);
SetupPrefixMap(array, map);
save->Build(map);
DoExpectedLookup(load, array);
DoExpectedLookup(save, array);
DoRandomLookup(save, 1000, array);
CheckContent(save, map);
DoRandomLookup(load, 1000, array);
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->Append(NS_LITERAL_STRING("test.vlpset"));
save->StoreToFile(file);
}
CheckContent(load, map);
// Load
{
RefPtr<LookupCacheV4> load =
SetupLookupCache(NS_LITERAL_CSTRING("test-load"));
load->LoadFromFile(file);
DoExpectedLookup(load, array);
DoRandomLookup(load, 1000, array);
CheckContent(load, map);
}
file->Remove(false);
}
// Test save then load prefix with both 4 bytes prefixes and 5~32 bytes prefixes
TEST(UrlClassifierVLPrefixSet, LoadSavePrefixSet) {
RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet;
save->Init(NS_LITERAL_CSTRING("test-save"));
// Try to simulate the real case that most prefixes are 4bytes
_PrefixArray array;
RandomPrefixes(20000, 4, 4, array);
RandomPrefixes(1000, 5, 32, array);
PrefixStringMap map;
SetupPrefixMap(array, map);
save->SetPrefixes(map);
DoExpectedLookup(save, array);
DoRandomLookup(save, 1000, array);
CheckContent(save, map);
nsCOMPtr<nsIFile> file;
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->Append(NS_LITERAL_STRING("test.vlpset"));
_PrefixArray array;
PrefixStringMap map;
save->StoreToFile(file);
// Save
{
RefPtr<LookupCacheV4> save =
SetupLookupCache(NS_LITERAL_CSTRING("test-save"));
RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet;
load->Init(NS_LITERAL_CSTRING("test-load"));
// Try to simulate the real case that most prefixes are 4bytes
RandomPrefixes(20000, 4, 4, array);
RandomPrefixes(1000, 5, 32, array);
load->LoadFromFile(file);
SetupPrefixMap(array, map);
save->Build(map);
DoExpectedLookup(load, array);
DoExpectedLookup(save, array);
DoRandomLookup(save, 1000, array);
CheckContent(save, map);
DoRandomLookup(load, 1000, array);
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->Append(NS_LITERAL_STRING("test.vlpset"));
save->StoreToFile(file);
}
CheckContent(load, map);
// Load
{
RefPtr<LookupCacheV4> load =
SetupLookupCache(NS_LITERAL_CSTRING("test-load"));
load->LoadFromFile(file);
DoExpectedLookup(load, array);
DoRandomLookup(load, 1000, array);
CheckContent(load, map);
}
file->Remove(false);
}
// This is for fixed-length prefixset
TEST(UrlClassifierVLPrefixSet, LoadSaveNoDelta) {
nsCOMPtr<nsIFile> file;
_PrefixArray array;
PrefixStringMap map;
for (uint32_t i = 0; i < 100; i++) {
// construct a tree without deltas by making the distance
// between entries larger than 16 bits
uint32_t v = ((1 << 16) + 1) * i;
nsCString* ele = array.AppendElement();
ele->AppendASCII(reinterpret_cast<const char*>(&v), 4);
}
// Save
{
RefPtr<LookupCacheV4> save =
SetupLookupCache(NS_LITERAL_CSTRING("test-save"));
SetupPrefixMap(array, map);
save->Build(map);
DoExpectedLookup(save, array);
DoRandomLookup(save, 1000, array);
CheckContent(save, map);
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->Append(NS_LITERAL_STRING("test.vlpset"));
save->StoreToFile(file);
}
// Load
{
RefPtr<LookupCacheV4> load =
SetupLookupCache(NS_LITERAL_CSTRING("test-load"));
load->LoadFromFile(file);
DoExpectedLookup(load, array);
DoRandomLookup(load, 1000, array);
CheckContent(load, map);
}
file->Remove(false);
}

62
toolkit/components/url-classifier/tests/unit/test_prefixset.js
Просмотреть файл

@ -140,40 +140,6 @@ function testReSetPrefixes() {
checkContents(pset, secondPrefixes);
}
function testLoadSaveLargeSet() {
let N = 1000;
let arr = [];
for (let i = 0; i < N; i++) {
let randInt = Math.floor(Math.random() * Math.pow(2, 32));
arr.push(randInt);
}
arr.sort((x, y) => x - y);
let pset = newPset();
pset.setPrefixes(arr, arr.length);
doExpectedLookups(pset, arr, 1);
doRandomLookups(pset, arr, 1000);
checkContents(pset, arr);
// Now try to save, restore, and redo the lookups
var file = Services.dirsvc.get("ProfLD", Ci.nsIFile);
file.append("testLarge.pset");
pset.storeToFile(file);
let psetLoaded = newPset();
psetLoaded.loadFromFile(file);
doExpectedLookups(psetLoaded, arr, 1);
doRandomLookups(psetLoaded, arr, 1000);
checkContents(psetLoaded, arr);
}
function testTinySet() {
let pset = Cc["@mozilla.org/url-classifier/prefixset;1"]
.createInstance(Ci.nsIUrlClassifierPrefixSet);
@ -190,37 +156,11 @@ function testTinySet() {
checkContents(pset, prefixes);
}
function testLoadSaveNoDelta() {
let N = 100;
let arr = [];
for (let i = 0; i < N; i++) {
// construct a tree without deltas by making the distance
// between entries larger than 16 bits
arr.push(((1 << 16) + 1) * i);
}
let pset = newPset();
pset.setPrefixes(arr, arr.length);
doExpectedLookups(pset, arr, 1);
var file = Services.dirsvc.get("ProfLD", Ci.nsIFile);
file.append("testNoDelta.pset");
pset.storeToFile(file);
pset.loadFromFile(file);
doExpectedLookups(pset, arr, 1);
}
var tests = [testBasicPset,
testSimplePset,
testReSetPrefixes,
testLoadSaveLargeSet,
testDuplicates,
testTinySet,
testLoadSaveNoDelta];
testTinySet];
function run_test() {
// None of the tests use |executeSoon| or any sort of callbacks, so we can