Bug 1297962 - Add noise data when sending v4 gethash request r=francois

MozReview-Commit-ID: GbyvX7wcg8c
* * *
[mq]: 1297962_review

MozReview-Commit-ID: 1U2T0wq778R

--HG--
extra : rebase_source : 55f852204aa29b135360e461b8c152e45fecc7b6

toolkit/components/url-classifier/Classifier.cpp

@@ -1382,31 +1382,45 @@ Classifier::ReadNoiseEntries(const Prefix& aPrefix,
                              uint32_t aCount,
                              PrefixArray* aNoiseEntries)
 {
-  // TODO : Bug 1297962, support adding noise for v4
-  LookupCacheV2 *cache =
-    LookupCache::Cast<LookupCacheV2>(GetLookupCache(aTableName));
+  FallibleTArray<uint32_t> prefixes;
+  nsresult rv;
+
+  LookupCache *cache = GetLookupCache(aTableName);
   if (!cache) {
     return NS_ERROR_FAILURE;
   }
 
-  FallibleTArray<uint32_t> prefixes;
-  nsresult rv = cache->GetPrefixes(prefixes);
+  LookupCacheV2* cacheV2 = LookupCache::Cast<LookupCacheV2>(cache);
+  if (cacheV2) {
+    rv = cacheV2->GetPrefixes(prefixes);
+  } else {
+    rv = LookupCache::Cast<LookupCacheV4>(cache)->GetFixedLengthPrefixes(prefixes);
+  }
+
   NS_ENSURE_SUCCESS(rv, rv);
 
-  size_t idx = prefixes.BinaryIndexOf(aPrefix.ToUint32());
-
-  if (idx == nsTArray<uint32_t>::NoIndex) {
+  if (prefixes.Length() == 0) {
     NS_WARNING("Could not find prefix in PrefixSet during noise lookup");
     return NS_ERROR_FAILURE;
   }
 
-  idx -= idx % aCount;
+  for (size_t i = 0; i < aCount; i++) {
+    // Pick some prefixes from cache as noise
+    // We pick a random prefix index from 0 to prefixes.Length() - 1;
+    uint32_t idx = rand() % prefixes.Length();
 
-  for (size_t i = 0; (i < aCount) && ((idx+i) < prefixes.Length()); i++) {
-    Prefix newPref;
-    newPref.FromUint32(prefixes[idx+i]);
-    if (newPref != aPrefix) {
-      aNoiseEntries->AppendElement(newPref);
+    Prefix newPrefix;
+    uint32_t hash = prefixes[idx];
+    // In the case V4 little endian, we did swapping endian when converting from char* to
+    // int, should revert endian to make sure we will send hex string correctly
+    // See https://bugzilla.mozilla.org/show_bug.cgi?id=1283007#c23
+    if (!cacheV2 && !bool(MOZ_BIG_ENDIAN)) {
+      hash = NativeEndian::swapFromBigEndian(prefixes[idx]);
+    }
+
+    newPrefix.FromUint32(hash);
+    if (newPrefix != aPrefix) {
+      aNoiseEntries->AppendElement(newPrefix);
     }
   }
 

toolkit/components/url-classifier/Classifier.h

@@ -113,6 +113,9 @@ public:
   // update intermediaries.
   nsresult SwapInNewTablesAndCleanup();
 
+  LookupCache *GetLookupCache(const nsACString& aTable,
+                              bool aForUpdate = false);
+
 private:
   void DropStores();
   void DeleteTables(nsIFile* aDirectory, const nsTArray<nsCString>& aTables);
@@ -147,9 +150,6 @@ private:
 
   nsresult UpdateCache(TableUpdate* aUpdates);
 
-  LookupCache *GetLookupCache(const nsACString& aTable,
-                              bool aForUpdate = false);
-
   LookupCache *GetLookupCacheForUpdate(const nsACString& aTable) {
     return GetLookupCache(aTable, true);
   }

toolkit/components/url-classifier/LookupCacheV4.cpp

@@ -139,6 +139,12 @@ LookupCacheV4::GetPrefixes(PrefixStringMap& aPrefixMap)
   return mVLPrefixSet->GetPrefixes(aPrefixMap);
 }
 
+nsresult
+LookupCacheV4::GetFixedLengthPrefixes(FallibleTArray<uint32_t>& aPrefixes)
+{
+  return mVLPrefixSet->GetFixedLengthPrefixes(aPrefixes);
+}
+
 nsresult
 LookupCacheV4::ClearPrefixes()
 {

toolkit/components/url-classifier/LookupCacheV4.h

@@ -38,6 +38,7 @@ public:
   nsresult Build(PrefixStringMap& aPrefixMap);
 
   nsresult GetPrefixes(PrefixStringMap& aPrefixMap);
+  nsresult GetFixedLengthPrefixes(FallibleTArray<uint32_t>& aPrefixes);
 
   // ApplyUpdate will merge data stored in aTableUpdate with prefixes in aInputMap.
   nsresult ApplyUpdate(TableUpdateV4* aTableUpdate,

toolkit/components/url-classifier/VariableLengthPrefixSet.cpp

@@ -37,7 +37,7 @@ VariableLengthPrefixSet::VariableLengthPrefixSet()
   mFixedPrefixSet = new nsUrlClassifierPrefixSet();
 }
 
-NS_IMETHODIMP
+nsresult
 VariableLengthPrefixSet::Init(const nsACString& aName)
 {
   mMemoryReportPath =
@@ -56,7 +56,7 @@ VariableLengthPrefixSet::~VariableLengthPrefixSet()
   UnregisterWeakMemoryReporter(this);
 }
 
-NS_IMETHODIMP
+nsresult
 VariableLengthPrefixSet::SetPrefixes(const PrefixStringMap& aPrefixMap)
 {
   MutexAutoLock lock(mLock);
@@ -152,10 +152,16 @@ VariableLengthPrefixSet::GetPrefixes(PrefixStringMap& aPrefixMap)
   return NS_OK;
 }
 
+nsresult
+VariableLengthPrefixSet::GetFixedLengthPrefixes(FallibleTArray<uint32_t>& aPrefixes)
+{
+  return mFixedPrefixSet->GetPrefixesNative(aPrefixes);
+}
+
 // It should never be the case that more than one hash prefixes match a given
 // full hash. However, if that happens, this method returns any one of them.
 // It does not guarantee which one of those will be returned.
-NS_IMETHODIMP
+nsresult
 VariableLengthPrefixSet::Matches(const nsACString& aFullHash, uint32_t* aLength)
 {
   MutexAutoLock lock(mLock);
@@ -189,7 +195,7 @@ VariableLengthPrefixSet::Matches(const nsACString& aFullHash, uint32_t* aLength)
   return NS_OK;
 }
 
-NS_IMETHODIMP
+nsresult
 VariableLengthPrefixSet::IsEmpty(bool* aEmpty)
 {
   MutexAutoLock lock(mLock);
@@ -202,7 +208,7 @@ VariableLengthPrefixSet::IsEmpty(bool* aEmpty)
   return NS_OK;
 }
 
-NS_IMETHODIMP
+nsresult
 VariableLengthPrefixSet::LoadFromFile(nsIFile* aFile)
 {
   MutexAutoLock lock(mLock);
@@ -241,7 +247,7 @@ VariableLengthPrefixSet::LoadFromFile(nsIFile* aFile)
   return NS_OK;;
 }
 
-NS_IMETHODIMP
+nsresult
 VariableLengthPrefixSet::StoreToFile(nsIFile* aFile)
 {
   NS_ENSURE_ARG_POINTER(aFile);

toolkit/components/url-classifier/VariableLengthPrefixSet.h

@@ -25,13 +25,14 @@ class VariableLengthPrefixSet final
 public:
   VariableLengthPrefixSet();
 
-  NS_IMETHOD Init(const nsACString& aName);
-  NS_IMETHOD SetPrefixes(const mozilla::safebrowsing::PrefixStringMap& aPrefixMap);
-  NS_IMETHOD GetPrefixes(mozilla::safebrowsing::PrefixStringMap& aPrefixMap);
-  NS_IMETHOD Matches(const nsACString& aFullHash, uint32_t* aLength);
-  NS_IMETHOD IsEmpty(bool* aEmpty);
-  NS_IMETHOD LoadFromFile(nsIFile* aFile);
-  NS_IMETHOD StoreToFile(nsIFile* aFile);
+  nsresult Init(const nsACString& aName);
+  nsresult SetPrefixes(const mozilla::safebrowsing::PrefixStringMap& aPrefixMap);
+  nsresult GetPrefixes(mozilla::safebrowsing::PrefixStringMap& aPrefixMap);
+  nsresult GetFixedLengthPrefixes(FallibleTArray<uint32_t>& aPrefixes);
+  nsresult Matches(const nsACString& aFullHash, uint32_t* aLength);
+  nsresult IsEmpty(bool* aEmpty);
+  nsresult LoadFromFile(nsIFile* aFile);
+  nsresult StoreToFile(nsIFile* aFile);
 
   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
 

toolkit/components/url-classifier/tests/gtest/Common.cpp

@@ -77,3 +77,40 @@ PrefixArrayToPrefixStringMap(const nsTArray<nsCString>& prefixArray,
   }
 }
 
+nsresult
+PrefixArrayToAddPrefixArrayV2(const nsTArray<nsCString>& prefixArray,
+                              AddPrefixArray& out)
+{
+  out.Clear();
+
+  for (size_t i = 0; i < prefixArray.Length(); i++) {
+    // Create prefix hash from string
+    Prefix hash;
+    static_assert(sizeof(hash.buf) == PREFIX_SIZE, "Prefix must be 4 bytes length");
+    memcpy(hash.buf, prefixArray[i].BeginReading(), PREFIX_SIZE);
+    MOZ_ASSERT(prefixArray[i].Length() == PREFIX_SIZE);
+
+    AddPrefix *add = out.AppendElement(fallible);
+    if (!add) {
+      return NS_ERROR_OUT_OF_MEMORY;
+    }
+
+    add->addChunk = i;
+    add->prefix = hash;
+  }
+
+  return NS_OK;
+}
+
+nsCString
+GeneratePrefix(const nsCString& aFragment, uint8_t aLength)
+{
+  Completion complete;
+  nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
+  complete.FromPlaintext(aFragment, cryptoHash);
+
+  nsCString hash;
+  hash.Assign((const char *)complete.buf, aLength);
+  return hash;
+}
+

toolkit/components/url-classifier/tests/gtest/Common.h

@@ -24,3 +24,8 @@ void ApplyUpdate(TableUpdate* update);
 void PrefixArrayToPrefixStringMap(const nsTArray<nsCString>& prefixArray,
                                   PrefixStringMap& out);
 
+nsresult PrefixArrayToAddPrefixArrayV2(const nsTArray<nsCString>& prefixArray,
+                                       AddPrefixArray& out);
+
+// Generate a hash prefix from string
+nsCString GeneratePrefix(const nsCString& aFragment, uint8_t aLength);

toolkit/components/url-classifier/tests/gtest/TestClassifier.cpp

@@ -0,0 +1,138 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "Common.h"
+#include "Classifier.h"
+#include "LookupCacheV4.h"
+
+#define GTEST_TABLE_V4 NS_LITERAL_CSTRING("gtest-malware-proto")
+#define GTEST_TABLE_V2 NS_LITERAL_CSTRING("gtest-malware-simple")
+
+typedef nsCString _Fragment;
+typedef nsTArray<nsCString> _PrefixArray;
+
+static UniquePtr<Classifier>
+GetClassifier()
+{
+  nsCOMPtr<nsIFile> file;
+  NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+
+  UniquePtr<Classifier> classifier = MakeUnique<Classifier>();
+  nsresult rv = classifier->Open(*file);
+  EXPECT_TRUE(rv == NS_OK);
+
+  return Move(classifier);
+}
+
+static nsresult
+SetupLookupCacheV4(Classifier* classifier,
+                   const _PrefixArray& aPrefixArray,
+                   const nsACString& aTable)
+{
+  LookupCacheV4* lookupCache =
+    LookupCache::Cast<LookupCacheV4>(classifier->GetLookupCache(aTable, false));
+  if (!lookupCache) {
+    return NS_ERROR_FAILURE;
+  }
+
+  PrefixStringMap map;
+  PrefixArrayToPrefixStringMap(aPrefixArray, map);
+
+  return lookupCache->Build(map);
+}
+
+static nsresult
+SetupLookupCacheV2(Classifier* classifier,
+                   const _PrefixArray& aPrefixArray,
+                   const nsACString& aTable)
+{
+  LookupCacheV2* lookupCache =
+    LookupCache::Cast<LookupCacheV2>(classifier->GetLookupCache(aTable, false));
+  if (!lookupCache) {
+    return NS_ERROR_FAILURE;
+  }
+
+  AddPrefixArray prefixes;
+  AddCompleteArray completions;
+  nsresult rv = PrefixArrayToAddPrefixArrayV2(aPrefixArray, prefixes);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+
+  EntrySort(prefixes);
+  return lookupCache->Build(prefixes, completions);
+}
+
+static void
+TestReadNoiseEntries(Classifier* classifier,
+                     const _PrefixArray& aPrefixArray,
+                     const nsCString& aTable,
+                     const nsCString& aFragment)
+{
+  Completion lookupHash;
+  nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
+  lookupHash.FromPlaintext(aFragment, cryptoHash);
+  LookupResult result;
+  result.hash.complete = lookupHash;
+
+  PrefixArray noiseEntries;
+  uint32_t noiseCount = 3;
+  nsresult rv;
+  rv = classifier->ReadNoiseEntries(result.hash.fixedLengthPrefix,
+                                    aTable, noiseCount,
+                                    &noiseEntries);
+  ASSERT_TRUE(rv == NS_OK);
+  EXPECT_TRUE(noiseEntries.Length() > 0);
+
+  for (uint32_t i = 0; i < noiseEntries.Length(); i++) {
+    // Test the noise entry should not equal the "real" hash request
+    EXPECT_NE(noiseEntries[i], result.hash.fixedLengthPrefix);
+    // Test the noise entry should exist in the cached prefix array
+    nsAutoCString partialHash;
+    partialHash.Assign(reinterpret_cast<char*>(&noiseEntries[i]), PREFIX_SIZE);
+    EXPECT_TRUE(aPrefixArray.Contains(partialHash));
+  }
+
+}
+
+TEST(Classifier, ReadNoiseEntriesV4)
+{
+  UniquePtr<Classifier> classifier(GetClassifier());
+  _PrefixArray array = { GeneratePrefix(_Fragment("bravo.com/"), 5),
+                         GeneratePrefix(_Fragment("browsing.com/"), 9),
+                         GeneratePrefix(_Fragment("gound.com/"), 4),
+                         GeneratePrefix(_Fragment("small.com/"), 4),
+                         GeneratePrefix(_Fragment("gdfad.com/"), 4),
+                         GeneratePrefix(_Fragment("afdfound.com/"), 4),
+                         GeneratePrefix(_Fragment("dffa.com/"), 4),
+                       };
+  array.Sort();
+
+  nsresult rv;
+  rv = SetupLookupCacheV4(classifier.get(), array, GTEST_TABLE_V4);
+  ASSERT_TRUE(rv == NS_OK);
+
+  TestReadNoiseEntries(classifier.get(), array, GTEST_TABLE_V4, _Fragment("gound.com/"));
+}
+
+TEST(Classifier, ReadNoiseEntriesV2)
+{
+  UniquePtr<Classifier> classifier(GetClassifier());
+  _PrefixArray array = { GeneratePrefix(_Fragment("helloworld.com/"), 4),
+                         GeneratePrefix(_Fragment("firefox.com/"), 4),
+                         GeneratePrefix(_Fragment("chrome.com/"), 4),
+                         GeneratePrefix(_Fragment("safebrowsing.com/"), 4),
+                         GeneratePrefix(_Fragment("opera.com/"), 4),
+                         GeneratePrefix(_Fragment("torbrowser.com/"), 4),
+                         GeneratePrefix(_Fragment("gfaads.com/"), 4),
+                         GeneratePrefix(_Fragment("qggdsas.com/"), 4),
+                         GeneratePrefix(_Fragment("nqtewq.com/"), 4),
+                       };
+
+  nsresult rv;
+  rv = SetupLookupCacheV2(classifier.get(), array, GTEST_TABLE_V2);
+  ASSERT_TRUE(rv == NS_OK);
+
+  TestReadNoiseEntries(classifier.get(), array, GTEST_TABLE_V2, _Fragment("helloworld.com/"));
+}

toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp

@@ -11,19 +11,6 @@
 typedef nsCString _Fragment;
 typedef nsTArray<nsCString> _PrefixArray;
 
-// Generate a hash prefix from string
-static const nsCString
-GeneratePrefix(const _Fragment& aFragment, uint8_t aLength)
-{
-  Completion complete;
-  nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
-  complete.FromPlaintext(aFragment, cryptoHash);
-
-  nsCString hash;
-  hash.Assign((const char *)complete.buf, aLength);
-  return hash;
-}
-
 static UniquePtr<LookupCacheV4>
 SetupLookupCacheV4(const _PrefixArray& prefixArray)
 {

toolkit/components/url-classifier/tests/gtest/moz.build

@@ -11,6 +11,7 @@ LOCAL_INCLUDES += [
 UNIFIED_SOURCES += [
     'Common.cpp',
     'TestChunkSet.cpp',
+    'TestClassifier.cpp',
     'TestFailUpdate.cpp',
     'TestFindFullHash.cpp',
     'TestLookupCacheV4.cpp',