зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1353956 - P3. Separate file processing and prefix data processing for SafeBrowsing prefix set. r=gcp
SafeBrowsing prefix files LOAD/SAVE operations are handled in xxxPrefixSet.cpp. It would be more clear if xxxPrefixSet.cpp only processes prefix data, while LookupCacheV2/LookupCacheV4 which use prefix set process file. This patch doesn't change any behavior, testcases need to update because the LookupCache & xxxPrefixSet APIs are changed. Differential Revision: https://phabricator.services.mozilla.com/D21462 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Dimi Lee
Родитель
d253a723f7
Коммит
b8f59ed8b6
|
|
@ -651,11 +651,73 @@ nsresult LookupCacheV2::ClearPrefixes() {
|
|||
}
|
||||
|
||||
nsresult LookupCacheV2::StoreToFile(nsCOMPtr<nsIFile>& aFile) {
|
||||
return mPrefixSet->StoreToFile(aFile);
|
||||
nsCOMPtr<nsIOutputStream> localOutFile;
|
||||
nsresult rv =
|
||||
NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
|
||||
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
uint32_t fileSize;
|
||||
|
||||
// Preallocate the file storage
|
||||
{
|
||||
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
|
||||
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FALLOCATE_TIME> timer;
|
||||
|
||||
fileSize = mPrefixSet->CalculatePreallocateSize();
|
||||
|
||||
// Ignore failure, the preallocation is a hint and we write out the entire
|
||||
// file later on
|
||||
Unused << fos->Preallocate(fileSize);
|
||||
}
|
||||
|
||||
// Convert to buffered stream
|
||||
nsCOMPtr<nsIOutputStream> out;
|
||||
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
|
||||
std::min(fileSize, MAX_BUFFER_SIZE));
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = mPrefixSet->WritePrefixes(out);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
LOG(("[%s] Storing PrefixSet successful", mTableName.get()));
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
nsresult LookupCacheV2::LoadFromFile(nsCOMPtr<nsIFile>& aFile) {
|
||||
return mPrefixSet->LoadFromFile(aFile);
|
||||
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FILELOAD_TIME> timer;
|
||||
|
||||
nsCOMPtr<nsIInputStream> localInFile;
|
||||
nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile,
|
||||
PR_RDONLY | nsIFile::OS_READAHEAD);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
// Calculate how big the file is, make sure our read buffer isn't bigger
|
||||
// than the file itself which is just wasting memory.
|
||||
int64_t fileSize;
|
||||
rv = aFile->GetFileSize(&fileSize);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
if (fileSize < 0 || fileSize > UINT32_MAX) {
|
||||
return NS_ERROR_FAILURE;
|
||||
}
|
||||
|
||||
uint32_t bufferSize =
|
||||
std::min<uint32_t>(static_cast<uint32_t>(fileSize), MAX_BUFFER_SIZE);
|
||||
|
||||
// Convert to buffered stream
|
||||
nsCOMPtr<nsIInputStream> in;
|
||||
rv = NS_NewBufferedInputStream(getter_AddRefs(in), localInFile.forget(),
|
||||
bufferSize);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = mPrefixSet->LoadPrefixes(in);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
mPrimed = true;
|
||||
LOG(("[%s] Loading PrefixSet successful", mTableName.get()));
|
||||
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
size_t LookupCacheV2::SizeOfPrefixSet() const {
|
||||
|
|
|
|||
|
|
@ -231,6 +231,9 @@ class LookupCache {
|
|||
virtual nsresult Has(const Completion& aCompletion, bool* aHas,
|
||||
uint32_t* aMatchLength, bool* aConfirmed) = 0;
|
||||
|
||||
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) = 0;
|
||||
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) = 0;
|
||||
|
||||
virtual bool IsEmpty() const = 0;
|
||||
|
||||
virtual void ClearAll();
|
||||
|
|
@ -250,8 +253,6 @@ class LookupCache {
|
|||
private:
|
||||
nsresult LoadPrefixSet();
|
||||
|
||||
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) = 0;
|
||||
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) = 0;
|
||||
virtual size_t SizeOfPrefixSet() const = 0;
|
||||
|
||||
virtual int Ver() const = 0;
|
||||
|
|
@ -259,6 +260,8 @@ class LookupCache {
|
|||
protected:
|
||||
virtual ~LookupCache() {}
|
||||
|
||||
static const uint32_t MAX_BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
// Check completions in positive cache and prefix in negative cache.
|
||||
// 'aHas' and 'aConfirmed' are output parameters.
|
||||
nsresult CheckCache(const Completion& aCompletion, bool* aHas,
|
||||
|
|
@ -292,6 +295,9 @@ class LookupCacheV2 final : public LookupCache {
|
|||
virtual nsresult Has(const Completion& aCompletion, bool* aHas,
|
||||
uint32_t* aMatchLength, bool* aConfirmed) override;
|
||||
|
||||
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) override;
|
||||
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) override;
|
||||
|
||||
virtual bool IsEmpty() const override;
|
||||
|
||||
nsresult Build(AddPrefixArray& aAddPrefixes, AddCompleteArray& aAddCompletes);
|
||||
|
|
@ -314,8 +320,6 @@ class LookupCacheV2 final : public LookupCache {
|
|||
nsresult ReadCompletions();
|
||||
|
||||
virtual nsresult ClearPrefixes() override;
|
||||
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) override;
|
||||
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) override;
|
||||
virtual size_t SizeOfPrefixSet() const override;
|
||||
|
||||
private:
|
||||
|
|
|
|||
|
|
@ -164,11 +164,74 @@ nsresult LookupCacheV4::ClearPrefixes() {
|
|||
}
|
||||
|
||||
nsresult LookupCacheV4::StoreToFile(nsCOMPtr<nsIFile>& aFile) {
|
||||
return mVLPrefixSet->StoreToFile(aFile);
|
||||
NS_ENSURE_ARG_POINTER(aFile);
|
||||
|
||||
nsCOMPtr<nsIOutputStream> localOutFile;
|
||||
nsresult rv =
|
||||
NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
|
||||
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
uint32_t fileSize = 0;
|
||||
// Preallocate the file storage
|
||||
{
|
||||
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
|
||||
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FALLOCATE_TIME> timer;
|
||||
|
||||
fileSize = mVLPrefixSet->CalculatePreallocateSize();
|
||||
|
||||
Unused << fos->Preallocate(fileSize);
|
||||
}
|
||||
|
||||
// Convert to buffered stream
|
||||
nsCOMPtr<nsIOutputStream> out;
|
||||
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
|
||||
std::min(fileSize, MAX_BUFFER_SIZE));
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = mVLPrefixSet->WritePrefixes(out);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
LOG(("[%s] Storing PrefixSet successful", mTableName.get()));
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
nsresult LookupCacheV4::LoadFromFile(nsCOMPtr<nsIFile>& aFile) {
|
||||
return mVLPrefixSet->LoadFromFile(aFile);
|
||||
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FILELOAD_TIME> timer;
|
||||
|
||||
nsCOMPtr<nsIInputStream> localInFile;
|
||||
nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile,
|
||||
PR_RDONLY | nsIFile::OS_READAHEAD);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
// Calculate how big the file is, make sure our read buffer isn't bigger
|
||||
// than the file itself which is just wasting memory.
|
||||
int64_t fileSize;
|
||||
rv = aFile->GetFileSize(&fileSize);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
if (fileSize < 0 || fileSize > UINT32_MAX) {
|
||||
return NS_ERROR_FAILURE;
|
||||
}
|
||||
|
||||
uint32_t bufferSize =
|
||||
std::min<uint32_t>(static_cast<uint32_t>(fileSize), MAX_BUFFER_SIZE);
|
||||
|
||||
// Convert to buffered stream
|
||||
nsCOMPtr<nsIInputStream> in;
|
||||
rv = NS_NewBufferedInputStream(getter_AddRefs(in), localInFile.forget(),
|
||||
bufferSize);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = mVLPrefixSet->LoadPrefixes(in);
|
||||
if (NS_WARN_IF(NS_FAILED(rv))) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
mPrimed = true;
|
||||
LOG(("[%s] Loading PrefixSet successful", mTableName.get()));
|
||||
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
size_t LookupCacheV4::SizeOfPrefixSet() const {
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ class LookupCacheV4 final : public LookupCache {
|
|||
virtual nsresult Has(const Completion& aCompletion, bool* aHas,
|
||||
uint32_t* aMatchLength, bool* aConfirmed) override;
|
||||
|
||||
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) override;
|
||||
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) override;
|
||||
|
||||
virtual bool IsEmpty() const override;
|
||||
|
||||
nsresult Build(PrefixStringMap& aPrefixMap);
|
||||
|
|
@ -47,8 +50,6 @@ class LookupCacheV4 final : public LookupCache {
|
|||
|
||||
protected:
|
||||
virtual nsresult ClearPrefixes() override;
|
||||
virtual nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) override;
|
||||
virtual nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile) override;
|
||||
virtual size_t SizeOfPrefixSet() const override;
|
||||
|
||||
private:
|
||||
|
|
|
|||
|
|
@ -204,91 +204,18 @@ nsresult VariableLengthPrefixSet::IsEmpty(bool* aEmpty) const {
|
|||
return NS_OK;
|
||||
}
|
||||
|
||||
nsresult VariableLengthPrefixSet::LoadFromFile(nsCOMPtr<nsIFile>& aFile) {
|
||||
MutexAutoLock lock(mLock);
|
||||
|
||||
NS_ENSURE_ARG_POINTER(aFile);
|
||||
|
||||
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FILELOAD_TIME> timer;
|
||||
|
||||
nsCOMPtr<nsIInputStream> localInFile;
|
||||
nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile,
|
||||
PR_RDONLY | nsIFile::OS_READAHEAD);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
// Calculate how big the file is, make sure our read buffer isn't bigger
|
||||
// than the file itself which is just wasting memory.
|
||||
int64_t fileSize;
|
||||
rv = aFile->GetFileSize(&fileSize);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
if (fileSize < 0 || fileSize > UINT32_MAX) {
|
||||
return NS_ERROR_FAILURE;
|
||||
}
|
||||
|
||||
uint32_t bufferSize =
|
||||
std::min<uint32_t>(static_cast<uint32_t>(fileSize), MAX_BUFFER_SIZE);
|
||||
|
||||
// Convert to buffered stream
|
||||
nsCOMPtr<nsIInputStream> in;
|
||||
rv = NS_NewBufferedInputStream(getter_AddRefs(in), localInFile.forget(),
|
||||
bufferSize);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = mFixedPrefixSet->LoadPrefixes(in);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = LoadPrefixes(in);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
return NS_OK;
|
||||
;
|
||||
}
|
||||
|
||||
nsresult VariableLengthPrefixSet::StoreToFile(nsCOMPtr<nsIFile>& aFile) const {
|
||||
NS_ENSURE_ARG_POINTER(aFile);
|
||||
|
||||
MutexAutoLock lock(mLock);
|
||||
|
||||
nsCOMPtr<nsIOutputStream> localOutFile;
|
||||
nsresult rv =
|
||||
NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
|
||||
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
uint32_t fileSize = 0;
|
||||
// Preallocate the file storage
|
||||
{
|
||||
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
|
||||
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FALLOCATE_TIME> timer;
|
||||
|
||||
fileSize += mFixedPrefixSet->CalculatePreallocateSize();
|
||||
fileSize += CalculatePreallocateSize();
|
||||
|
||||
Unused << fos->Preallocate(fileSize);
|
||||
}
|
||||
|
||||
// Convert to buffered stream
|
||||
nsCOMPtr<nsIOutputStream> out;
|
||||
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
|
||||
std::min(fileSize, MAX_BUFFER_SIZE));
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = mFixedPrefixSet->WritePrefixes(out);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = WritePrefixes(out);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
nsresult VariableLengthPrefixSet::LoadPrefixes(nsCOMPtr<nsIInputStream>& in) {
|
||||
MutexAutoLock lock(mLock);
|
||||
|
||||
// First read prefixes from fixed-length prefix set
|
||||
nsresult rv = mFixedPrefixSet->LoadPrefixes(in);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
// Then read prefixes from variable-length prefix set
|
||||
uint32_t magic;
|
||||
uint32_t read;
|
||||
|
||||
nsresult rv =
|
||||
in->Read(reinterpret_cast<char*>(&magic), sizeof(uint32_t), &read);
|
||||
rv = in->Read(reinterpret_cast<char*>(&magic), sizeof(uint32_t), &read);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE);
|
||||
|
||||
|
|
@ -349,6 +276,10 @@ nsresult VariableLengthPrefixSet::LoadPrefixes(nsCOMPtr<nsIInputStream>& in) {
|
|||
uint32_t VariableLengthPrefixSet::CalculatePreallocateSize() const {
|
||||
uint32_t fileSize = 0;
|
||||
|
||||
// Size of fixed length prefix set.
|
||||
fileSize += mFixedPrefixSet->CalculatePreallocateSize();
|
||||
|
||||
// Size of variable length prefix set.
|
||||
// Store how many prefix string.
|
||||
fileSize += sizeof(uint32_t);
|
||||
|
||||
|
|
@ -363,10 +294,17 @@ uint32_t VariableLengthPrefixSet::CalculatePreallocateSize() const {
|
|||
|
||||
nsresult VariableLengthPrefixSet::WritePrefixes(
|
||||
nsCOMPtr<nsIOutputStream>& out) const {
|
||||
MutexAutoLock lock(mLock);
|
||||
|
||||
// First, write fixed length prefix set
|
||||
nsresult rv = mFixedPrefixSet->WritePrefixes(out);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
// Then, write variable length prefix set
|
||||
uint32_t written;
|
||||
uint32_t writelen = sizeof(uint32_t);
|
||||
uint32_t magic = PREFIXSET_VERSION_MAGIC;
|
||||
nsresult rv = out->Write(reinterpret_cast<char*>(&magic), writelen, &written);
|
||||
rv = out->Write(reinterpret_cast<char*>(&magic), writelen, &written);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE);
|
||||
|
||||
|
|
|
|||
|
|
@ -30,8 +30,10 @@ class VariableLengthPrefixSet final : public nsIMemoryReporter {
|
|||
nsresult GetFixedLengthPrefixes(FallibleTArray<uint32_t>& aPrefixes);
|
||||
nsresult Matches(const nsACString& aFullHash, uint32_t* aLength) const;
|
||||
nsresult IsEmpty(bool* aEmpty) const;
|
||||
nsresult LoadFromFile(nsCOMPtr<nsIFile>& aFile);
|
||||
nsresult StoreToFile(nsCOMPtr<nsIFile>& aFile) const;
|
||||
|
||||
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
|
||||
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
|
||||
uint32_t CalculatePreallocateSize() const;
|
||||
|
||||
size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
|
||||
|
||||
|
|
@ -47,10 +49,6 @@ class VariableLengthPrefixSet final : public nsIMemoryReporter {
|
|||
bool BinarySearch(const nsACString& aFullHash, const nsACString& aPrefixes,
|
||||
uint32_t aPrefixSize) const;
|
||||
|
||||
uint32_t CalculatePreallocateSize() const;
|
||||
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
|
||||
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
|
||||
|
||||
// Lock to prevent races between the url-classifier thread (which does most
|
||||
// of the operations) and the main thread (which does memory reporting).
|
||||
// It should be held for all operations between Init() and destruction that
|
||||
|
|
|
|||
|
|
@ -24,6 +24,4 @@ interface nsIUrlClassifierPrefixSet : nsISupports
|
|||
// Do a lookup in the PrefixSet, return whether the value is present.
|
||||
boolean contains(in unsigned long aPrefix);
|
||||
boolean isEmpty();
|
||||
void loadFromFile(in nsIFile aFile);
|
||||
void storeToFile(in nsIFile aFile);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -12,16 +12,13 @@
|
|||
#include "nsPrintfCString.h"
|
||||
#include "nsTArray.h"
|
||||
#include "nsString.h"
|
||||
#include "nsIFile.h"
|
||||
#include "nsTArray.h"
|
||||
#include "nsThreadUtils.h"
|
||||
#include "nsNetUtil.h"
|
||||
#include "nsISeekableStream.h"
|
||||
#include "nsIBufferedStreams.h"
|
||||
#include "nsIFileStreams.h"
|
||||
#include "mozilla/MemoryReporting.h"
|
||||
#include "mozilla/Telemetry.h"
|
||||
#include "mozilla/FileUtils.h"
|
||||
#include "mozilla/Logging.h"
|
||||
#include "mozilla/Unused.h"
|
||||
#include <algorithm>
|
||||
|
|
@ -345,83 +342,10 @@ nsUrlClassifierPrefixSet::IsEmpty(bool* aEmpty) {
|
|||
return NS_OK;
|
||||
}
|
||||
|
||||
NS_IMETHODIMP
|
||||
nsUrlClassifierPrefixSet::LoadFromFile(nsIFile* aFile) {
|
||||
MutexAutoLock lock(mLock);
|
||||
|
||||
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FILELOAD_TIME> timer;
|
||||
|
||||
nsCOMPtr<nsIInputStream> localInFile;
|
||||
nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile,
|
||||
PR_RDONLY | nsIFile::OS_READAHEAD);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
// Calculate how big the file is, make sure our read buffer isn't bigger
|
||||
// than the file itself which is just wasting memory.
|
||||
int64_t fileSize;
|
||||
rv = aFile->GetFileSize(&fileSize);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
if (fileSize < 0 || fileSize > UINT32_MAX) {
|
||||
return NS_ERROR_FAILURE;
|
||||
}
|
||||
|
||||
uint32_t bufferSize =
|
||||
std::min<uint32_t>(static_cast<uint32_t>(fileSize), MAX_BUFFER_SIZE);
|
||||
|
||||
// Convert to buffered stream
|
||||
nsCOMPtr<nsIInputStream> in;
|
||||
rv = NS_NewBufferedInputStream(getter_AddRefs(in), localInFile.forget(),
|
||||
bufferSize);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = LoadPrefixes(in);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
NS_IMETHODIMP
|
||||
nsUrlClassifierPrefixSet::StoreToFile(nsIFile* aFile) {
|
||||
MutexAutoLock lock(mLock);
|
||||
|
||||
nsCOMPtr<nsIOutputStream> localOutFile;
|
||||
nsresult rv =
|
||||
NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
|
||||
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
uint32_t fileSize;
|
||||
|
||||
// Preallocate the file storage
|
||||
{
|
||||
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
|
||||
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FALLOCATE_TIME> timer;
|
||||
|
||||
fileSize = CalculatePreallocateSize();
|
||||
|
||||
// Ignore failure, the preallocation is a hint and we write out the entire
|
||||
// file later on
|
||||
Unused << fos->Preallocate(fileSize);
|
||||
}
|
||||
|
||||
// Convert to buffered stream
|
||||
nsCOMPtr<nsIOutputStream> out;
|
||||
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
|
||||
std::min(fileSize, MAX_BUFFER_SIZE));
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
rv = WritePrefixes(out);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
LOG(("[%s] Storing PrefixSet successful", mName.get()));
|
||||
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
nsresult nsUrlClassifierPrefixSet::LoadPrefixes(nsCOMPtr<nsIInputStream>& in) {
|
||||
mCanary.Check();
|
||||
MutexAutoLock lock(mLock);
|
||||
|
||||
mCanary.Check();
|
||||
Clear();
|
||||
|
||||
uint32_t magic;
|
||||
|
|
@ -518,6 +442,8 @@ uint32_t nsUrlClassifierPrefixSet::CalculatePreallocateSize() const {
|
|||
|
||||
nsresult nsUrlClassifierPrefixSet::WritePrefixes(
|
||||
nsCOMPtr<nsIOutputStream>& out) const {
|
||||
MutexAutoLock lock(mLock);
|
||||
|
||||
mCanary.Check();
|
||||
|
||||
// In Bug 1362761, crashes happened while reading mIndexDeltas[i].
|
||||
|
|
|
|||
|
|
@ -39,10 +39,11 @@ class nsUrlClassifierPrefixSet final : public nsIUrlClassifierPrefixSet,
|
|||
NS_IMETHOD GetPrefixes(uint32_t* aCount, uint32_t** aPrefixes) override;
|
||||
NS_IMETHOD Contains(uint32_t aPrefix, bool* aFound) override;
|
||||
NS_IMETHOD IsEmpty(bool* aEmpty) override;
|
||||
NS_IMETHOD LoadFromFile(nsIFile* aFile) override;
|
||||
NS_IMETHOD StoreToFile(nsIFile* aFile) override;
|
||||
|
||||
nsresult GetPrefixesNative(FallibleTArray<uint32_t>& outArray);
|
||||
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
|
||||
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
|
||||
uint32_t CalculatePreallocateSize() const;
|
||||
|
||||
size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
|
||||
|
||||
|
|
@ -63,9 +64,6 @@ class nsUrlClassifierPrefixSet final : public nsIUrlClassifierPrefixSet,
|
|||
nsresult MakePrefixSet(const uint32_t* aArray, uint32_t aLength);
|
||||
uint32_t BinSearch(uint32_t start, uint32_t end, uint32_t target) const;
|
||||
bool IsEmptyInternal() const;
|
||||
uint32_t CalculatePreallocateSize() const;
|
||||
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
|
||||
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
|
||||
|
||||
// Lock to prevent races between the url-classifier thread (which does most
|
||||
// of the operations) and the main thread (which does memory reporting).
|
||||
|
|
|
|||
|
|
@ -143,16 +143,17 @@ static void VerifyPrefixSet(PrefixStringMap& expected) {
|
|||
// Verify the prefix set is written to disk.
|
||||
nsCOMPtr<nsIFile> file;
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
|
||||
|
||||
file->AppendNative(GTEST_SAFEBROWSING_DIR);
|
||||
file->AppendNative(GTEST_PREFIXFILE);
|
||||
|
||||
RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet;
|
||||
load->Init(GTEST_TABLE);
|
||||
RefPtr<LookupCacheV4> lookup =
|
||||
new LookupCacheV4(GTEST_TABLE, NS_LITERAL_CSTRING("test"), file);
|
||||
lookup->Init();
|
||||
|
||||
file->AppendNative(GTEST_PREFIXFILE);
|
||||
lookup->LoadFromFile(file);
|
||||
|
||||
PrefixStringMap prefixesInFile;
|
||||
load->LoadFromFile(file);
|
||||
load->GetPrefixes(prefixesInFile);
|
||||
lookup->GetPrefixes(prefixesInFile);
|
||||
|
||||
for (auto iter = expected.ConstIter(); !iter.Done(); iter.Next()) {
|
||||
nsCString* expectedPrefix = iter.Data();
|
||||
|
|
@ -233,7 +234,7 @@ static void testOpenLookupCache() {
|
|||
}
|
||||
|
||||
// Tests start from here.
|
||||
TEST(UrlClassifierTableUpdateV4, FixLenghtPSetFullUpdate) {
|
||||
TEST(UrlClassifierTableUpdateV4, FixLengthPSetFullUpdate) {
|
||||
srand(time(NULL));
|
||||
|
||||
_PrefixArray array;
|
||||
|
|
@ -249,7 +250,7 @@ TEST(UrlClassifierTableUpdateV4, FixLenghtPSetFullUpdate) {
|
|||
Clear();
|
||||
}
|
||||
|
||||
TEST(UrlClassifierTableUpdateV4, VariableLenghtPSetFullUpdate) {
|
||||
TEST(UrlClassifierTableUpdateV4, VariableLengthPSetFullUpdate) {
|
||||
_PrefixArray array;
|
||||
PrefixStringMap map;
|
||||
nsCString sha256;
|
||||
|
|
|
|||
|
|
@ -17,11 +17,11 @@ typedef nsCString _Prefix;
|
|||
typedef nsTArray<_Prefix> _PrefixArray;
|
||||
|
||||
// Create fullhash by appending random characters.
|
||||
static nsCString* CreateFullHash(const nsACString& in) {
|
||||
nsCString* out = new nsCString(in);
|
||||
out->SetLength(32);
|
||||
static nsCString CreateFullHash(const nsACString& in) {
|
||||
nsCString out(in);
|
||||
out.SetLength(32);
|
||||
for (size_t i = in.Length(); i < 32; i++) {
|
||||
out->SetCharAt(char(rand() % 256), i);
|
||||
out.SetCharAt(char(rand() % 256), i);
|
||||
}
|
||||
|
||||
return out;
|
||||
|
|
@ -55,10 +55,9 @@ static void RandomPrefixes(uint32_t N, uint32_t MIN, uint32_t MAX,
|
|||
}
|
||||
}
|
||||
|
||||
static void CheckContent(VariableLengthPrefixSet* pset,
|
||||
PrefixStringMap& expected) {
|
||||
static void CheckContent(LookupCacheV4* cache, PrefixStringMap& expected) {
|
||||
PrefixStringMap vlPSetMap;
|
||||
pset->GetPrefixes(vlPSetMap);
|
||||
cache->GetPrefixes(vlPSetMap);
|
||||
|
||||
for (auto iter = vlPSetMap.Iter(); !iter.Done(); iter.Next()) {
|
||||
nsCString* expectedPrefix = expected.Get(iter.Key());
|
||||
|
|
@ -71,15 +70,16 @@ static void CheckContent(VariableLengthPrefixSet* pset,
|
|||
// This test loops through all the prefixes and converts each prefix to
|
||||
// fullhash by appending random characters, each converted fullhash
|
||||
// should at least match its original length in the prefixSet.
|
||||
static void DoExpectedLookup(VariableLengthPrefixSet* pset,
|
||||
_PrefixArray& array) {
|
||||
static void DoExpectedLookup(LookupCacheV4* cache, _PrefixArray& array) {
|
||||
uint32_t matchLength = 0;
|
||||
for (uint32_t i = 0; i < array.Length(); i++) {
|
||||
const nsCString& prefix = array[i];
|
||||
UniquePtr<nsCString> fullhash(CreateFullHash(prefix));
|
||||
Completion complete;
|
||||
complete.Assign(CreateFullHash(prefix));
|
||||
|
||||
// Find match for prefix-generated full hash
|
||||
pset->Matches(*fullhash, &matchLength);
|
||||
bool has, confirmed;
|
||||
cache->Has(complete, &has, &matchLength, &confirmed);
|
||||
MOZ_ASSERT(matchLength != 0);
|
||||
|
||||
if (matchLength != prefix.Length()) {
|
||||
|
|
@ -93,8 +93,7 @@ static void DoExpectedLookup(VariableLengthPrefixSet* pset,
|
|||
continue;
|
||||
}
|
||||
|
||||
if (0 == memcmp(fullhash->BeginReading(), array[j].BeginReading(),
|
||||
matchLength)) {
|
||||
if (0 == memcmp(complete.buf, array[j].BeginReading(), matchLength)) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
|
|
@ -104,7 +103,7 @@ static void DoExpectedLookup(VariableLengthPrefixSet* pset,
|
|||
}
|
||||
}
|
||||
|
||||
static void DoRandomLookup(VariableLengthPrefixSet* pset, uint32_t N,
|
||||
static void DoRandomLookup(LookupCacheV4* cache, uint32_t N,
|
||||
_PrefixArray& array) {
|
||||
for (uint32_t i = 0; i < N; i++) {
|
||||
// Random 32-bytes test fullhash
|
||||
|
|
@ -122,8 +121,11 @@ static void DoRandomLookup(VariableLengthPrefixSet* pset, uint32_t N,
|
|||
}
|
||||
}
|
||||
|
||||
Completion complete;
|
||||
complete.Assign(nsDependentCSubstring(buf, 32));
|
||||
bool has, confirmed;
|
||||
uint32_t matchLength = 0;
|
||||
pset->Matches(nsDependentCSubstring(buf, 32), &matchLength);
|
||||
cache->Has(complete, &has, &matchLength, &confirmed);
|
||||
|
||||
ASSERT_TRUE(expected.IsEmpty() ? !matchLength
|
||||
: expected.Contains(matchLength));
|
||||
|
|
@ -168,12 +170,23 @@ static void SetupPrefixMap(const _PrefixArray& array, PrefixStringMap& map) {
|
|||
}
|
||||
}
|
||||
|
||||
static already_AddRefed<LookupCacheV4> SetupLookupCache(
|
||||
const nsACString& aName) {
|
||||
nsCOMPtr<nsIFile> rootDir;
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(rootDir));
|
||||
|
||||
nsAutoCString provider("test");
|
||||
RefPtr<LookupCacheV4> lookup = new LookupCacheV4(aName, provider, rootDir);
|
||||
lookup->Init();
|
||||
|
||||
return lookup.forget();
|
||||
}
|
||||
|
||||
// Test setting prefix set with only 4-bytes prefixes
|
||||
TEST(UrlClassifierVLPrefixSet, FixedLengthSet) {
|
||||
srand(time(nullptr));
|
||||
|
||||
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
|
||||
pset->Init(NS_LITERAL_CSTRING("test"));
|
||||
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
|
||||
|
||||
PrefixStringMap map;
|
||||
_PrefixArray array = {
|
||||
|
|
@ -182,13 +195,11 @@ TEST(UrlClassifierVLPrefixSet, FixedLengthSet) {
|
|||
};
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array);
|
||||
|
||||
DoRandomLookup(pset, 1000, array);
|
||||
|
||||
CheckContent(pset, map);
|
||||
DoExpectedLookup(cache, array);
|
||||
DoRandomLookup(cache, 1000, array);
|
||||
CheckContent(cache, map);
|
||||
|
||||
// Run random test
|
||||
array.Clear();
|
||||
|
|
@ -197,19 +208,16 @@ TEST(UrlClassifierVLPrefixSet, FixedLengthSet) {
|
|||
RandomPrefixes(1500, 4, 4, array);
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array);
|
||||
|
||||
DoRandomLookup(pset, 1000, array);
|
||||
|
||||
CheckContent(pset, map);
|
||||
DoExpectedLookup(cache, array);
|
||||
DoRandomLookup(cache, 1000, array);
|
||||
CheckContent(cache, map);
|
||||
}
|
||||
|
||||
// Test setting prefix set with only 5~32 bytes prefixes
|
||||
TEST(UrlClassifierVLPrefixSet, VariableLengthSet) {
|
||||
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
|
||||
pset->Init(NS_LITERAL_CSTRING("test"));
|
||||
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
|
||||
|
||||
PrefixStringMap map;
|
||||
_PrefixArray array = {
|
||||
|
|
@ -223,13 +231,11 @@ TEST(UrlClassifierVLPrefixSet, VariableLengthSet) {
|
|||
_Prefix("yankee"), _Prefix("ZuluZuluZuluZulu")};
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array);
|
||||
|
||||
DoRandomLookup(pset, 1000, array);
|
||||
|
||||
CheckContent(pset, map);
|
||||
DoExpectedLookup(cache, array);
|
||||
DoRandomLookup(cache, 1000, array);
|
||||
CheckContent(cache, map);
|
||||
|
||||
// Run random test
|
||||
array.Clear();
|
||||
|
|
@ -238,19 +244,16 @@ TEST(UrlClassifierVLPrefixSet, VariableLengthSet) {
|
|||
RandomPrefixes(1500, 5, 32, array);
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array);
|
||||
|
||||
DoRandomLookup(pset, 1000, array);
|
||||
|
||||
CheckContent(pset, map);
|
||||
DoExpectedLookup(cache, array);
|
||||
DoRandomLookup(cache, 1000, array);
|
||||
CheckContent(cache, map);
|
||||
}
|
||||
|
||||
// Test setting prefix set with both 4-bytes prefixes and 5~32 bytes prefixes
|
||||
TEST(UrlClassifierVLPrefixSet, MixedPrefixSet) {
|
||||
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
|
||||
pset->Init(NS_LITERAL_CSTRING("test"));
|
||||
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
|
||||
|
||||
PrefixStringMap map;
|
||||
_PrefixArray array = {_Prefix("enus"),
|
||||
|
|
@ -273,13 +276,11 @@ TEST(UrlClassifierVLPrefixSet, MixedPrefixSet) {
|
|||
_Prefix("Stheno, Euryale and Medusa")};
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array);
|
||||
|
||||
DoRandomLookup(pset, 1000, array);
|
||||
|
||||
CheckContent(pset, map);
|
||||
DoExpectedLookup(cache, array);
|
||||
DoRandomLookup(cache, 1000, array);
|
||||
CheckContent(cache, map);
|
||||
|
||||
// Run random test
|
||||
array.Clear();
|
||||
|
|
@ -288,19 +289,16 @@ TEST(UrlClassifierVLPrefixSet, MixedPrefixSet) {
|
|||
RandomPrefixes(1500, 4, 32, array);
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array);
|
||||
|
||||
DoRandomLookup(pset, 1000, array);
|
||||
|
||||
CheckContent(pset, map);
|
||||
DoExpectedLookup(cache, array);
|
||||
DoRandomLookup(cache, 1000, array);
|
||||
CheckContent(cache, map);
|
||||
}
|
||||
|
||||
// Test resetting prefix set
|
||||
TEST(UrlClassifierVLPrefixSet, ResetPrefix) {
|
||||
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
|
||||
pset->Init(NS_LITERAL_CSTRING("test"));
|
||||
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
|
||||
|
||||
// First prefix set
|
||||
_PrefixArray array1 = {
|
||||
|
|
@ -313,9 +311,9 @@ TEST(UrlClassifierVLPrefixSet, ResetPrefix) {
|
|||
PrefixStringMap map;
|
||||
|
||||
SetupPrefixMap(array1, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array1);
|
||||
DoExpectedLookup(cache, array1);
|
||||
}
|
||||
|
||||
// Second
|
||||
|
|
@ -329,25 +327,28 @@ TEST(UrlClassifierVLPrefixSet, ResetPrefix) {
|
|||
PrefixStringMap map;
|
||||
|
||||
SetupPrefixMap(array2, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array2);
|
||||
DoExpectedLookup(cache, array2);
|
||||
}
|
||||
|
||||
// Should not match any of the first prefix set
|
||||
uint32_t matchLength = 0;
|
||||
for (uint32_t i = 0; i < array1.Length(); i++) {
|
||||
UniquePtr<nsACString> fullhash(CreateFullHash(array1[i]));
|
||||
Completion complete;
|
||||
complete.Assign(CreateFullHash(array1[i]));
|
||||
|
||||
// Find match for prefix-generated full hash
|
||||
bool has, confirmed;
|
||||
cache->Has(complete, &has, &matchLength, &confirmed);
|
||||
|
||||
pset->Matches(*fullhash, &matchLength);
|
||||
ASSERT_TRUE(matchLength == 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Test only set one 4-bytes prefix and one full-length prefix
|
||||
TEST(UrlClassifierVLPrefixSet, TinyPrefixSet) {
|
||||
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
|
||||
pset->Init(NS_LITERAL_CSTRING("test"));
|
||||
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
|
||||
|
||||
PrefixStringMap map;
|
||||
_PrefixArray array = {
|
||||
|
|
@ -356,52 +357,47 @@ TEST(UrlClassifierVLPrefixSet, TinyPrefixSet) {
|
|||
};
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
DoExpectedLookup(pset, array);
|
||||
|
||||
DoRandomLookup(pset, 1000, array);
|
||||
|
||||
CheckContent(pset, map);
|
||||
DoExpectedLookup(cache, array);
|
||||
DoRandomLookup(cache, 1000, array);
|
||||
CheckContent(cache, map);
|
||||
}
|
||||
|
||||
// Test empty prefix set and IsEmpty function
|
||||
TEST(UrlClassifierVLPrefixSet, EmptyPrefixSet) {
|
||||
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
|
||||
pset->Init(NS_LITERAL_CSTRING("test"));
|
||||
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
|
||||
|
||||
bool empty;
|
||||
pset->IsEmpty(&empty);
|
||||
bool empty = cache->IsEmpty();
|
||||
ASSERT_TRUE(empty);
|
||||
|
||||
PrefixStringMap map;
|
||||
_PrefixArray array1;
|
||||
|
||||
// Lookup an empty array should never match
|
||||
DoRandomLookup(pset, 100, array1);
|
||||
DoRandomLookup(cache, 100, array1);
|
||||
|
||||
// Insert an 4-bytes prefix, then IsEmpty should return false
|
||||
_PrefixArray array2 = {_Prefix("test")};
|
||||
SetupPrefixMap(array2, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
pset->IsEmpty(&empty);
|
||||
empty = cache->IsEmpty();
|
||||
ASSERT_TRUE(!empty);
|
||||
|
||||
_PrefixArray array3 = {_Prefix("test variable length")};
|
||||
|
||||
// Insert an 5~32 bytes prefix, then IsEmpty should return false
|
||||
SetupPrefixMap(array3, map);
|
||||
pset->SetPrefixes(map);
|
||||
cache->Build(map);
|
||||
|
||||
pset->IsEmpty(&empty);
|
||||
empty = cache->IsEmpty();
|
||||
ASSERT_TRUE(!empty);
|
||||
}
|
||||
|
||||
// Test prefix size should only between 4~32 bytes
|
||||
TEST(UrlClassifierVLPrefixSet, MinMaxPrefixSet) {
|
||||
RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
|
||||
pset->Init(NS_LITERAL_CSTRING("test"));
|
||||
RefPtr<LookupCacheV4> cache = SetupLookupCache(NS_LITERAL_CSTRING("test"));
|
||||
|
||||
PrefixStringMap map;
|
||||
{
|
||||
|
|
@ -409,7 +405,7 @@ TEST(UrlClassifierVLPrefixSet, MinMaxPrefixSet) {
|
|||
_Prefix("1aaa2bbb3ccc4ddd5eee6fff7ggg8hhh")};
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
nsresult rv = pset->SetPrefixes(map);
|
||||
nsresult rv = cache->Build(map);
|
||||
ASSERT_TRUE(rv == NS_OK);
|
||||
}
|
||||
|
||||
|
|
@ -418,7 +414,7 @@ TEST(UrlClassifierVLPrefixSet, MinMaxPrefixSet) {
|
|||
_PrefixArray array = {_Prefix("123")};
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
nsresult rv = pset->SetPrefixes(map);
|
||||
nsresult rv = cache->Build(map);
|
||||
ASSERT_TRUE(NS_FAILED(rv));
|
||||
}
|
||||
|
||||
|
|
@ -427,123 +423,171 @@ TEST(UrlClassifierVLPrefixSet, MinMaxPrefixSet) {
|
|||
_PrefixArray array = {_Prefix("1aaa2bbb3ccc4ddd5eee6fff7ggg8hhh9")};
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
nsresult rv = pset->SetPrefixes(map);
|
||||
nsresult rv = cache->Build(map);
|
||||
ASSERT_TRUE(NS_FAILED(rv));
|
||||
}
|
||||
}
|
||||
|
||||
// Test save then load prefix set with only 4-bytes prefixes
|
||||
TEST(UrlClassifierVLPrefixSet, LoadSaveFixedLengthPrefixSet) {
|
||||
RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet;
|
||||
save->Init(NS_LITERAL_CSTRING("test-save"));
|
||||
|
||||
_PrefixArray array;
|
||||
RandomPrefixes(10000, 4, 4, array);
|
||||
|
||||
PrefixStringMap map;
|
||||
SetupPrefixMap(array, map);
|
||||
save->SetPrefixes(map);
|
||||
|
||||
DoExpectedLookup(save, array);
|
||||
|
||||
DoRandomLookup(save, 1000, array);
|
||||
|
||||
CheckContent(save, map);
|
||||
|
||||
nsCOMPtr<nsIFile> file;
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
|
||||
file->Append(NS_LITERAL_STRING("test.vlpset"));
|
||||
_PrefixArray array;
|
||||
PrefixStringMap map;
|
||||
|
||||
save->StoreToFile(file);
|
||||
// Save
|
||||
{
|
||||
RefPtr<LookupCacheV4> save =
|
||||
SetupLookupCache(NS_LITERAL_CSTRING("test-save"));
|
||||
|
||||
RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet;
|
||||
load->Init(NS_LITERAL_CSTRING("test-load"));
|
||||
RandomPrefixes(10000, 4, 4, array);
|
||||
|
||||
load->LoadFromFile(file);
|
||||
SetupPrefixMap(array, map);
|
||||
save->Build(map);
|
||||
|
||||
DoExpectedLookup(load, array);
|
||||
DoExpectedLookup(save, array);
|
||||
DoRandomLookup(save, 1000, array);
|
||||
CheckContent(save, map);
|
||||
|
||||
DoRandomLookup(load, 1000, array);
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
|
||||
file->Append(NS_LITERAL_STRING("test.vlpset"));
|
||||
save->StoreToFile(file);
|
||||
}
|
||||
|
||||
CheckContent(load, map);
|
||||
// Load
|
||||
{
|
||||
RefPtr<LookupCacheV4> load =
|
||||
SetupLookupCache(NS_LITERAL_CSTRING("test-load"));
|
||||
load->LoadFromFile(file);
|
||||
|
||||
DoExpectedLookup(load, array);
|
||||
DoRandomLookup(load, 1000, array);
|
||||
CheckContent(load, map);
|
||||
}
|
||||
|
||||
file->Remove(false);
|
||||
}
|
||||
|
||||
// Test save then load prefix set with only 5~32 bytes prefixes
|
||||
TEST(UrlClassifierVLPrefixSet, LoadSaveVariableLengthPrefixSet) {
|
||||
RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet;
|
||||
save->Init(NS_LITERAL_CSTRING("test-save"));
|
||||
|
||||
_PrefixArray array;
|
||||
RandomPrefixes(10000, 5, 32, array);
|
||||
|
||||
PrefixStringMap map;
|
||||
SetupPrefixMap(array, map);
|
||||
save->SetPrefixes(map);
|
||||
|
||||
DoExpectedLookup(save, array);
|
||||
|
||||
DoRandomLookup(save, 1000, array);
|
||||
|
||||
CheckContent(save, map);
|
||||
|
||||
nsCOMPtr<nsIFile> file;
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
|
||||
file->Append(NS_LITERAL_STRING("test.vlpset"));
|
||||
_PrefixArray array;
|
||||
PrefixStringMap map;
|
||||
|
||||
save->StoreToFile(file);
|
||||
// Save
|
||||
{
|
||||
RefPtr<LookupCacheV4> save =
|
||||
SetupLookupCache(NS_LITERAL_CSTRING("test-save"));
|
||||
|
||||
RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet;
|
||||
load->Init(NS_LITERAL_CSTRING("test-load"));
|
||||
RandomPrefixes(10000, 5, 32, array);
|
||||
|
||||
load->LoadFromFile(file);
|
||||
SetupPrefixMap(array, map);
|
||||
save->Build(map);
|
||||
|
||||
DoExpectedLookup(load, array);
|
||||
DoExpectedLookup(save, array);
|
||||
DoRandomLookup(save, 1000, array);
|
||||
CheckContent(save, map);
|
||||
|
||||
DoRandomLookup(load, 1000, array);
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
|
||||
file->Append(NS_LITERAL_STRING("test.vlpset"));
|
||||
save->StoreToFile(file);
|
||||
}
|
||||
|
||||
CheckContent(load, map);
|
||||
// Load
|
||||
{
|
||||
RefPtr<LookupCacheV4> load =
|
||||
SetupLookupCache(NS_LITERAL_CSTRING("test-load"));
|
||||
load->LoadFromFile(file);
|
||||
|
||||
DoExpectedLookup(load, array);
|
||||
DoRandomLookup(load, 1000, array);
|
||||
CheckContent(load, map);
|
||||
}
|
||||
|
||||
file->Remove(false);
|
||||
}
|
||||
|
||||
// Test save then load prefix with both 4 bytes prefixes and 5~32 bytes prefixes
|
||||
TEST(UrlClassifierVLPrefixSet, LoadSavePrefixSet) {
|
||||
RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet;
|
||||
save->Init(NS_LITERAL_CSTRING("test-save"));
|
||||
|
||||
// Try to simulate the real case that most prefixes are 4bytes
|
||||
_PrefixArray array;
|
||||
RandomPrefixes(20000, 4, 4, array);
|
||||
RandomPrefixes(1000, 5, 32, array);
|
||||
|
||||
PrefixStringMap map;
|
||||
SetupPrefixMap(array, map);
|
||||
save->SetPrefixes(map);
|
||||
|
||||
DoExpectedLookup(save, array);
|
||||
|
||||
DoRandomLookup(save, 1000, array);
|
||||
|
||||
CheckContent(save, map);
|
||||
|
||||
nsCOMPtr<nsIFile> file;
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
|
||||
file->Append(NS_LITERAL_STRING("test.vlpset"));
|
||||
_PrefixArray array;
|
||||
PrefixStringMap map;
|
||||
|
||||
save->StoreToFile(file);
|
||||
// Save
|
||||
{
|
||||
RefPtr<LookupCacheV4> save =
|
||||
SetupLookupCache(NS_LITERAL_CSTRING("test-save"));
|
||||
|
||||
RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet;
|
||||
load->Init(NS_LITERAL_CSTRING("test-load"));
|
||||
// Try to simulate the real case that most prefixes are 4bytes
|
||||
RandomPrefixes(20000, 4, 4, array);
|
||||
RandomPrefixes(1000, 5, 32, array);
|
||||
|
||||
load->LoadFromFile(file);
|
||||
SetupPrefixMap(array, map);
|
||||
save->Build(map);
|
||||
|
||||
DoExpectedLookup(load, array);
|
||||
DoExpectedLookup(save, array);
|
||||
DoRandomLookup(save, 1000, array);
|
||||
CheckContent(save, map);
|
||||
|
||||
DoRandomLookup(load, 1000, array);
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
|
||||
file->Append(NS_LITERAL_STRING("test.vlpset"));
|
||||
save->StoreToFile(file);
|
||||
}
|
||||
|
||||
CheckContent(load, map);
|
||||
// Load
|
||||
{
|
||||
RefPtr<LookupCacheV4> load =
|
||||
SetupLookupCache(NS_LITERAL_CSTRING("test-load"));
|
||||
load->LoadFromFile(file);
|
||||
|
||||
DoExpectedLookup(load, array);
|
||||
DoRandomLookup(load, 1000, array);
|
||||
CheckContent(load, map);
|
||||
}
|
||||
|
||||
file->Remove(false);
|
||||
}
|
||||
|
||||
// This is for fixed-length prefixset
|
||||
TEST(UrlClassifierVLPrefixSet, LoadSaveNoDelta) {
|
||||
nsCOMPtr<nsIFile> file;
|
||||
_PrefixArray array;
|
||||
PrefixStringMap map;
|
||||
|
||||
for (uint32_t i = 0; i < 100; i++) {
|
||||
// construct a tree without deltas by making the distance
|
||||
// between entries larger than 16 bits
|
||||
uint32_t v = ((1 << 16) + 1) * i;
|
||||
nsCString* ele = array.AppendElement();
|
||||
ele->AppendASCII(reinterpret_cast<const char*>(&v), 4);
|
||||
}
|
||||
|
||||
// Save
|
||||
{
|
||||
RefPtr<LookupCacheV4> save =
|
||||
SetupLookupCache(NS_LITERAL_CSTRING("test-save"));
|
||||
|
||||
SetupPrefixMap(array, map);
|
||||
save->Build(map);
|
||||
|
||||
DoExpectedLookup(save, array);
|
||||
DoRandomLookup(save, 1000, array);
|
||||
CheckContent(save, map);
|
||||
|
||||
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
|
||||
file->Append(NS_LITERAL_STRING("test.vlpset"));
|
||||
save->StoreToFile(file);
|
||||
}
|
||||
|
||||
// Load
|
||||
{
|
||||
RefPtr<LookupCacheV4> load =
|
||||
SetupLookupCache(NS_LITERAL_CSTRING("test-load"));
|
||||
load->LoadFromFile(file);
|
||||
|
||||
DoExpectedLookup(load, array);
|
||||
DoRandomLookup(load, 1000, array);
|
||||
CheckContent(load, map);
|
||||
}
|
||||
|
||||
file->Remove(false);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -140,40 +140,6 @@ function testReSetPrefixes() {
|
|||
checkContents(pset, secondPrefixes);
|
||||
}
|
||||
|
||||
function testLoadSaveLargeSet() {
|
||||
let N = 1000;
|
||||
let arr = [];
|
||||
|
||||
for (let i = 0; i < N; i++) {
|
||||
let randInt = Math.floor(Math.random() * Math.pow(2, 32));
|
||||
arr.push(randInt);
|
||||
}
|
||||
|
||||
arr.sort((x, y) => x - y);
|
||||
|
||||
let pset = newPset();
|
||||
pset.setPrefixes(arr, arr.length);
|
||||
|
||||
doExpectedLookups(pset, arr, 1);
|
||||
doRandomLookups(pset, arr, 1000);
|
||||
|
||||
checkContents(pset, arr);
|
||||
|
||||
// Now try to save, restore, and redo the lookups
|
||||
var file = Services.dirsvc.get("ProfLD", Ci.nsIFile);
|
||||
file.append("testLarge.pset");
|
||||
|
||||
pset.storeToFile(file);
|
||||
|
||||
let psetLoaded = newPset();
|
||||
psetLoaded.loadFromFile(file);
|
||||
|
||||
doExpectedLookups(psetLoaded, arr, 1);
|
||||
doRandomLookups(psetLoaded, arr, 1000);
|
||||
|
||||
checkContents(psetLoaded, arr);
|
||||
}
|
||||
|
||||
function testTinySet() {
|
||||
let pset = Cc["@mozilla.org/url-classifier/prefixset;1"]
|
||||
.createInstance(Ci.nsIUrlClassifierPrefixSet);
|
||||
|
|
@ -190,37 +156,11 @@ function testTinySet() {
|
|||
checkContents(pset, prefixes);
|
||||
}
|
||||
|
||||
function testLoadSaveNoDelta() {
|
||||
let N = 100;
|
||||
let arr = [];
|
||||
|
||||
for (let i = 0; i < N; i++) {
|
||||
// construct a tree without deltas by making the distance
|
||||
// between entries larger than 16 bits
|
||||
arr.push(((1 << 16) + 1) * i);
|
||||
}
|
||||
|
||||
let pset = newPset();
|
||||
pset.setPrefixes(arr, arr.length);
|
||||
|
||||
doExpectedLookups(pset, arr, 1);
|
||||
|
||||
var file = Services.dirsvc.get("ProfLD", Ci.nsIFile);
|
||||
file.append("testNoDelta.pset");
|
||||
|
||||
pset.storeToFile(file);
|
||||
pset.loadFromFile(file);
|
||||
|
||||
doExpectedLookups(pset, arr, 1);
|
||||
}
|
||||
|
||||
var tests = [testBasicPset,
|
||||
testSimplePset,
|
||||
testReSetPrefixes,
|
||||
testLoadSaveLargeSet,
|
||||
testDuplicates,
|
||||
testTinySet,
|
||||
testLoadSaveNoDelta];
|
||||
testTinySet];
|
||||
|
||||
function run_test() {
|
||||
// None of the tests use |executeSoon| or any sort of callbacks, so we can
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче