From d1ed4c245a97504ca30be076c3bbdbe0035da6c4 Mon Sep 17 00:00:00 2001 From: dlee Date: Sat, 29 Jun 2019 19:05:41 +0000 Subject: [PATCH] Bug 1531354 - P5. Safe Browsing test entries are directly stored in LookupCache. r=gcp Create test entries via update introduces performance overhead. We can store them directly in LookupCache and do not save test entries to disk. Differential Revision: https://phabricator.services.mozilla.com/D34576 --HG-- extra : moz-landing-system : lando --- .../safebrowsing/content/test/head.js | 8 +- mobile/android/app/mobile.js | 2 +- .../mozilla/geckoview/ContentBlocking.java | 2 +- .../browser/robocop/testTrackingProtection.js | 39 +-------- .../tests/browser/robocop/tracking_bad.html | 2 +- modules/libpref/init/all.js | 16 ++-- .../safebrowsing/test_initial_download.py | 6 +- .../components/url-classifier/Classifier.cpp | 38 +++++++++ .../components/url-classifier/Classifier.h | 2 + .../components/url-classifier/LookupCache.cpp | 70 ++++++++++++++- .../components/url-classifier/LookupCache.h | 4 + .../url-classifier/LookupCacheV4.cpp | 2 + .../components/url-classifier/LookupCacheV4.h | 2 + .../url-classifier/SafeBrowsing.jsm | 85 ------------------- .../UrlClassifierListManager.jsm | 10 +++ .../url-classifier/nsIUrlListManager.idl | 6 ++ .../nsUrlClassifierDBService.cpp | 2 +- .../url-classifier/nsUrlClassifierUtils.cpp | 27 +++++- .../url-classifier/nsUrlClassifierUtils.h | 4 + .../tests/browser/classifierHelper.js | 29 +++---- .../tests/gtest/TestFailUpdate.cpp | 4 +- .../tests/gtest/TestFindFullHash.cpp | 4 +- .../tests/mochitest/classifierCommon.js | 26 ++---- .../test_allowlisted_annotations.html | 2 +- .../tests/mochitest/test_bug1254766.html | 5 +- .../tests/mochitest/test_cachemiss.html | 1 + .../test_classified_annotations.html | 2 +- .../tests/mochitest/test_classifier.html | 7 +- .../mochitest/test_classifier_worker.html | 3 +- .../tests/mochitest/test_classify_ping.html | 8 -- .../tests/mochitest/test_gethash.html | 1 + .../test_safebrowsing_bug1272239.html | 2 +- .../test_trackingprotection_bug1157081.html | 2 +- .../tests/unit/head_urlclassifier.js | 7 ++ .../tests/unit/test_threat_type_conversion.js | 10 ++- 35 files changed, 228 insertions(+), 212 deletions(-) diff --git a/browser/components/safebrowsing/content/test/head.js b/browser/components/safebrowsing/content/test/head.js index af243b42e185..5c7ae55d898a 100644 --- a/browser/components/safebrowsing/content/test/head.js +++ b/browser/components/safebrowsing/content/test/head.js @@ -2,7 +2,7 @@ var {XPCOMUtils} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm"); // This url must sync with the table, url in SafeBrowsing.jsm addMozEntries -const PHISH_TABLE = "test-phish-simple"; +const PHISH_TABLE = "moztest-phish-simple"; const PHISH_URL = "https://www.itisatrap.org/firefox/its-a-trap.html"; /** @@ -86,7 +86,7 @@ function waitForDBInit(callback) { }); } -Services.prefs.setCharPref("urlclassifier.malwareTable", "test-malware-simple,test-unwanted-simple,test-harmful.simple"); -Services.prefs.setCharPref("urlclassifier.phishTable", "test-phish-simple"); -Services.prefs.setCharPref("urlclassifier.blockedTable", "test-block-simple"); +Services.prefs.setCharPref("urlclassifier.malwareTable", "moztest-malware-simple,moztest-unwanted-simple,moztest-harmful-simple"); +Services.prefs.setCharPref("urlclassifier.phishTable", "moztest-phish-simple"); +Services.prefs.setCharPref("urlclassifier.blockedTable", "moztest-block-simple"); SafeBrowsing.init(); diff --git a/mobile/android/app/mobile.js b/mobile/android/app/mobile.js index 2f83f97fcea0..023f8f56bd1a 100644 --- a/mobile/android/app/mobile.js +++ b/mobile/android/app/mobile.js @@ -590,7 +590,7 @@ pref("urlclassifier.downloadAllowTable", ""); pref("urlclassifier.downloadBlockTable", ""); // The Potentially Harmful Apps list replaces the malware one on Android. -pref("urlclassifier.malwareTable", "goog-harmful-proto,goog-unwanted-proto,test-harmful-simple,test-malware-simple,test-unwanted-simple"); +pref("urlclassifier.malwareTable", "goog-harmful-proto,goog-unwanted-proto,moztest-harmful-simple,moztest-malware-simple,moztest-unwanted-simple"); // True if you always want dump() to work // diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java index 07ad4018aac2..61c77dc32e78 100644 --- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java +++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java @@ -432,7 +432,7 @@ public class ContentBlocking { } - private static final String TEST = "test-track-simple"; + private static final String TEST = "moztest-track-simple"; private static final String AD = "ads-track-digest256"; private static final String ANALYTIC = "analytics-track-digest256"; private static final String SOCIAL = "social-track-digest256"; diff --git a/mobile/android/tests/browser/robocop/testTrackingProtection.js b/mobile/android/tests/browser/robocop/testTrackingProtection.js index 0b4d81906beb..1d8af4c01af8 100644 --- a/mobile/android/tests/browser/robocop/testTrackingProtection.js +++ b/mobile/android/tests/browser/robocop/testTrackingProtection.js @@ -40,44 +40,8 @@ function promiseLoadEvent(browser, url, eventType = "load", runBeforeLoad) { // Test that the Tracking Protection is active and has the correct state when // tracking content is blocked (Bug 1063831 + Bug 1520520) -// Code is mostly stolen from: -// http://dxr.mozilla.org/mozilla-central/source/browser/base/content/test/general/browser_trackingUI.js - var TABLE = "urlclassifier.trackingTable"; -// Update tracking database -function doUpdate() { - // Add some URLs to the tracking database (to be blocked) - var testData = "tracking.example.com/"; - var testUpdate = - "n:1000\ni:test-track-simple\nad:1\n" + - "a:524:32:" + testData.length + "\n" + - testData; - - let dbService = Cc["@mozilla.org/url-classifier/dbservice;1"].getService(Ci.nsIUrlClassifierDBService); - - return new Promise((resolve, reject) => { - let listener = { - QueryInterface: ChromeUtils.generateQI([Ci.nsIUrlClassifierUpdateObserver]), - updateUrlRequested: function(url) { }, - streamFinished: function(status) { }, - updateError: function(errorCode) { - ok(false, "Couldn't update classifier."); - resolve(); - }, - updateSuccess: function(requestedTimeout) { - resolve(); - }, - }; - - dbService.beginUpdate(listener, "test-track-simple", ""); - dbService.beginStream("", ""); - dbService.updateStream(testUpdate); - dbService.finishStream(); - dbService.finishUpdate(); - }); -} - var BrowserApp = Services.wm.getMostRecentWindow("navigator:browser").BrowserApp; // Tests the tracking protection UI in private browsing. By default, tracking protection is @@ -94,8 +58,7 @@ add_task(async function test_tracking_pb() { }); // Populate and use 'test-track-simple' for tracking protection lookups - Services.prefs.setCharPref(TABLE, "test-track-simple"); - await doUpdate(); + Services.prefs.setCharPref(TABLE, "moztest-track-simple"); // Point tab to a test page NOT containing tracking elements await promiseLoadEvent(browser, "http://tracking.example.org/tests/robocop/tracking_good.html"); diff --git a/mobile/android/tests/browser/robocop/tracking_bad.html b/mobile/android/tests/browser/robocop/tracking_bad.html index 17f0e459e3d1..9ca5ccc2fc1a 100644 --- a/mobile/android/tests/browser/robocop/tracking_bad.html +++ b/mobile/android/tests/browser/robocop/tracking_bad.html @@ -7,6 +7,6 @@ - + diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index d7bff2e4a458..d688c302f495 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -5200,13 +5200,13 @@ pref("captivedetect.maxRetryCount", 5); pref("dom.mapped_arraybuffer.enabled", true); // The tables used for Safebrowsing phishing and malware checks -pref("urlclassifier.malwareTable", "goog-malware-proto,goog-unwanted-proto,test-harmful-simple,test-malware-simple,test-unwanted-simple"); +pref("urlclassifier.malwareTable", "goog-malware-proto,goog-unwanted-proto,moztest-harmful-simple,moztest-malware-simple,moztest-unwanted-simple"); #ifdef MOZILLA_OFFICIAL // In official builds, we are allowed to use Google's private phishing // list (see bug 1288840). -pref("urlclassifier.phishTable", "goog-phish-proto,test-phish-simple"); +pref("urlclassifier.phishTable", "goog-phish-proto,moztest-phish-simple"); #else -pref("urlclassifier.phishTable", "googpub-phish-proto,test-phish-simple"); +pref("urlclassifier.phishTable", "googpub-phish-proto,moztest-phish-simple"); #endif // Tables for application reputation @@ -5217,10 +5217,10 @@ pref("urlclassifier.downloadBlockTable", "goog-badbinurl-proto"); pref("urlclassifier.passwordAllowTable", "goog-passwordwhite-proto"); // Tables for anti-tracking features -pref("urlclassifier.trackingAnnotationTable", "test-track-simple,ads-track-digest256,social-track-digest256,analytics-track-digest256,content-track-digest256"); -pref("urlclassifier.trackingAnnotationWhitelistTable", "test-trackwhite-simple,mozstd-trackwhite-digest256"); -pref("urlclassifier.trackingTable", "test-track-simple,base-track-digest256"); -pref("urlclassifier.trackingWhitelistTable", "test-trackwhite-simple,mozstd-trackwhite-digest256"); +pref("urlclassifier.trackingAnnotationTable", "moztest-track-simple,ads-track-digest256,social-track-digest256,analytics-track-digest256,content-track-digest256"); +pref("urlclassifier.trackingAnnotationWhitelistTable", "moztest-trackwhite-simple,mozstd-trackwhite-digest256"); +pref("urlclassifier.trackingTable", "moztest-track-simple,base-track-digest256"); +pref("urlclassifier.trackingWhitelistTable", "moztest-trackwhite-simple,mozstd-trackwhite-digest256"); pref("urlclassifier.features.fingerprinting.blacklistTables", "base-fingerprinting-track-digest256"); pref("urlclassifier.features.fingerprinting.whitelistTables", "mozstd-trackwhite-digest256"); @@ -5309,7 +5309,7 @@ pref("browser.safebrowsing.provider.mozilla.lists.base", "moz-std"); pref("browser.safebrowsing.provider.mozilla.lists.content", "moz-full"); // The table and global pref for blocking plugin content -pref("urlclassifier.blockedTable", "test-block-simple,mozplugin-block-digest256"); +pref("urlclassifier.blockedTable", "moztest-block-simple,mozplugin-block-digest256"); // Flash blocking tables pref("urlclassifier.flashAllowTable", "allow-flashallow-digest256"); diff --git a/testing/firefox-ui/tests/functional/safebrowsing/test_initial_download.py b/testing/firefox-ui/tests/functional/safebrowsing/test_initial_download.py index 2a48b64b5835..69e1582c0b11 100644 --- a/testing/firefox-ui/tests/functional/safebrowsing/test_initial_download.py +++ b/testing/firefox-ui/tests/functional/safebrowsing/test_initial_download.py @@ -12,7 +12,7 @@ from marionette_harness import MarionetteTestCase class TestSafeBrowsingInitialDownload(PuppeteerMixin, MarionetteTestCase): v2_file_extensions = [ - 'pset', + 'vlpset', 'sbstore', ] @@ -56,6 +56,10 @@ class TestSafeBrowsingInitialDownload(PuppeteerMixin, MarionetteTestCase): for pref_name in self.prefs_download_lists: base_names = self.marionette.get_pref(pref_name).split(',') + + # moztest- lists are not saved to disk + base_names = filter(lambda x: not x.startswith('moztest-'), base_names) + for ext in my_file_extensions: files.extend(['{name}.{ext}'.format(name=f, ext=ext) for f in base_names if f and f.endswith('-proto') == is_v4]) diff --git a/toolkit/components/url-classifier/Classifier.cpp b/toolkit/components/url-classifier/Classifier.cpp index d77f8dc924b5..c4eaa8f6e89a 100644 --- a/toolkit/components/url-classifier/Classifier.cpp +++ b/toolkit/components/url-classifier/Classifier.cpp @@ -860,10 +860,15 @@ nsresult Classifier::RegenActiveTables() { mActiveTablesCache.Clear(); + // Create nsTArray foundTables; nsresult rv = ScanStoreDir(mRootStoreDirectory, foundTables); Unused << NS_WARN_IF(NS_FAILED(rv)); + // We don't have test tables on disk, add Moz built-in entries here + rv = AddMozEntries(foundTables); + Unused << NS_WARN_IF(NS_FAILED(rv)); + for (const auto& table : foundTables) { RefPtr lookupCache = GetLookupCache(table); if (!lookupCache) { @@ -886,6 +891,30 @@ nsresult Classifier::RegenActiveTables() { return NS_OK; } +nsresult Classifier::AddMozEntries(nsTArray& aTables) { + nsTArray tables = { + NS_LITERAL_CSTRING("moztest-phish-simple"), + NS_LITERAL_CSTRING("moztest-malware-simple"), + NS_LITERAL_CSTRING("moztest-unwanted-simple"), + NS_LITERAL_CSTRING("moztest-harmful-simple"), + NS_LITERAL_CSTRING("moztest-track-simple"), + NS_LITERAL_CSTRING("moztest-trackwhite-simple"), + NS_LITERAL_CSTRING("moztest-block-simple"), + }; + + for (const auto& table : tables) { + RefPtr c = GetLookupCache(table, false); + RefPtr lookupCache = LookupCache::Cast(c); + if (!lookupCache || lookupCache->IsPrimed()) { + continue; + } + + aTables.AppendElement(table); + } + + return NS_OK; +} + nsresult Classifier::ScanStoreDir(nsIFile* aDirectory, nsTArray& aTables) { nsCOMPtr entries; @@ -1176,6 +1205,12 @@ nsresult Classifier::UpdateHashStore(TableUpdateArray& aUpdates, LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get())); + // moztest- tables don't support update because they are directly created + // in LookupCache. To test updates, use tables begin with "test-" instead. + // Also, recommend using 'test-' tables while writing testcases because + // it is more like the real world scenario. + MOZ_ASSERT(!nsUrlClassifierUtils::IsMozTestTable(aTable)); + HashStore store(aTable, GetProvider(aTable), mUpdatingDirectory); if (!CheckValidUpdate(aUpdates, store.TableName())) { @@ -1277,6 +1312,9 @@ nsresult Classifier::UpdateTableV4(TableUpdateArray& aUpdates, return NS_ERROR_UC_UPDATE_SHUTDOWNING; } + // moztest- tables don't support update, see comment in UpdateHashStore. + MOZ_ASSERT(!nsUrlClassifierUtils::IsMozTestTable(aTable)); + LOG(("Classifier::UpdateTableV4(%s)", PromiseFlatCString(aTable).get())); if (!CheckValidUpdate(aUpdates, aTable)) { diff --git a/toolkit/components/url-classifier/Classifier.h b/toolkit/components/url-classifier/Classifier.h index fd9697178e34..b4a88f96dab8 100644 --- a/toolkit/components/url-classifier/Classifier.h +++ b/toolkit/components/url-classifier/Classifier.h @@ -199,6 +199,8 @@ class Classifier { // Used by worker thread and update thread to abort current operation. bool ShouldAbort() const; + nsresult AddMozEntries(nsTArray& aTables); + // Root dir of the Local profile. nsCOMPtr mCacheDirectory; // Main directory where to store the databases. diff --git a/toolkit/components/url-classifier/LookupCache.cpp b/toolkit/components/url-classifier/LookupCache.cpp index 10c0f7cf276e..7fbda94af295 100644 --- a/toolkit/components/url-classifier/LookupCache.cpp +++ b/toolkit/components/url-classifier/LookupCache.cpp @@ -177,10 +177,18 @@ LookupCache::LookupCache(const nsACString& aTableName, nsresult LookupCache::Open() { LOG(("Loading PrefixSet for %s", mTableName.get())); - nsresult rv = LoadPrefixSet(); - NS_ENSURE_SUCCESS(rv, rv); + nsresult rv; + if (nsUrlClassifierUtils::IsMozTestTable(mTableName)) { + // For built-in test table, we don't load it from disk, + // test entries are directly added in memory. + rv = LoadMozEntries(); + } else { + rv = LoadPrefixSet(); + } - return NS_OK; + Unused << NS_WARN_IF(NS_FAILED(rv)); + + return rv; } nsresult LookupCache::Init() { @@ -863,5 +871,61 @@ nsCString LookupCacheV2::GetPrefixSetSuffix() const { return NS_LITERAL_CSTRING(".vlpset"); } +// Support creating built-in entries for phsihing, malware, unwanted, harmful, +// tracking/tracking whitelist and flash block tables. +// +nsresult LookupCacheV2::LoadMozEntries() { + // We already have the entries, return + if (!IsEmpty() || IsPrimed()) { + return NS_OK; + } + + nsTArray entries; + + if (mTableName.EqualsLiteral("moztest-phish-simple")) { + // Entries for phishing table + entries.AppendElement( + NS_LITERAL_CSTRING("itisatrap.org/firefox/its-a-trap.html")); + } else if (mTableName.EqualsLiteral("moztest-malware-simple")) { + // Entries for malware table + entries.AppendElement( + NS_LITERAL_CSTRING("itisatrap.org/firefox/its-an-attack.html")); + } else if (mTableName.EqualsLiteral("moztest-unwanted-simple")) { + // Entries for unwanted table + entries.AppendElement( + NS_LITERAL_CSTRING("itisatrap.org/firefox/unwanted.html")); + } else if (mTableName.EqualsLiteral("moztest-harmful-simple")) { + // Entries for harmfule tables + entries.AppendElement( + NS_LITERAL_CSTRING("itisatrap.org/firefox/harmful.html")); + } else if (mTableName.EqualsLiteral("moztest-track-simple")) { + // Entries for tracking table + entries.AppendElement(NS_LITERAL_CSTRING("trackertest.org/")); + entries.AppendElement(NS_LITERAL_CSTRING("itisatracker.org/")); + } else if (mTableName.EqualsLiteral("moztest-trackwhite-simple")) { + // Entries for tracking whitelist table + entries.AppendElement( + NS_LITERAL_CSTRING("itisatrap.org/?resource=itisatracker.org")); + } else if (mTableName.EqualsLiteral("moztest-block-simple")) { + // Entries for flash block table + entries.AppendElement( + NS_LITERAL_CSTRING("itisatrap.org/firefox/blocked.html")); + } else { + MOZ_ASSERT_UNREACHABLE(); + } + + AddPrefixArray prefix; + AddCompleteArray completes; + for (const auto& entry : entries) { + AddComplete add; + if (NS_FAILED(add.complete.FromPlaintext(entry))) { + continue; + } + completes.AppendElement(add, fallible); + } + + return Build(prefix, completes); +} + } // namespace safebrowsing } // namespace mozilla diff --git a/toolkit/components/url-classifier/LookupCache.h b/toolkit/components/url-classifier/LookupCache.h index c9ef56e6e834..ef6d76e7a407 100644 --- a/toolkit/components/url-classifier/LookupCache.h +++ b/toolkit/components/url-classifier/LookupCache.h @@ -240,6 +240,8 @@ class LookupCache { virtual void ClearAll(); + virtual nsresult LoadMozEntries() = 0; + template static T* Cast(LookupCache* aThat) { return ((aThat && T::VER == aThat->Ver()) ? reinterpret_cast(aThat) @@ -315,6 +317,8 @@ class LookupCacheV2 final : public LookupCache { const MissPrefixArray& aMissPrefixes, int64_t aExpirySec = 0); + virtual nsresult LoadMozEntries() override; + static const int VER; static const uint32_t VLPSET_MAGIC; static const uint32_t VLPSET_VERSION; diff --git a/toolkit/components/url-classifier/LookupCacheV4.cpp b/toolkit/components/url-classifier/LookupCacheV4.cpp index fb60808ed3b4..9e9414334673 100644 --- a/toolkit/components/url-classifier/LookupCacheV4.cpp +++ b/toolkit/components/url-classifier/LookupCacheV4.cpp @@ -570,5 +570,7 @@ bool VLPrefixSet::GetSmallestPrefix(nsACString& aOutString) const { return pick != nullptr; } +nsresult LookupCacheV4::LoadMozEntries() { return NS_ERROR_NOT_IMPLEMENTED; } + } // namespace safebrowsing } // namespace mozilla diff --git a/toolkit/components/url-classifier/LookupCacheV4.h b/toolkit/components/url-classifier/LookupCacheV4.h index d6164ba96d4a..536e9674fb42 100644 --- a/toolkit/components/url-classifier/LookupCacheV4.h +++ b/toolkit/components/url-classifier/LookupCacheV4.h @@ -39,6 +39,8 @@ class LookupCacheV4 final : public LookupCache { nsresult WriteMetadata(RefPtr aTableUpdate); nsresult LoadMetadata(nsACString& aState, nsACString& aChecksum); + virtual nsresult LoadMozEntries() override; + static const int VER; static const uint32_t MAX_METADATA_VALUE_LENGTH; static const uint32_t VLPSET_MAGIC; diff --git a/toolkit/components/url-classifier/SafeBrowsing.jsm b/toolkit/components/url-classifier/SafeBrowsing.jsm index e2ad2c95da47..08335b4fcde5 100644 --- a/toolkit/components/url-classifier/SafeBrowsing.jsm +++ b/toolkit/components/url-classifier/SafeBrowsing.jsm @@ -172,7 +172,6 @@ var SafeBrowsing = { Services.prefs.addObserver("plugins.flashBlock.enabled", this); this.readPrefs(); - this.addMozEntries(); this.controlUpdateChecking(); this.initialized = true; @@ -424,88 +423,4 @@ var SafeBrowsing = { listManager.maybeToggleUpdateChecking(); }, - - - addMozEntries() { - // Add test entries to the DB. - // XXX bug 779008 - this could be done by DB itself? - const phishURL = "itisatrap.org/firefox/its-a-trap.html"; - const malwareURL = "itisatrap.org/firefox/its-an-attack.html"; - const unwantedURL = "itisatrap.org/firefox/unwanted.html"; - const harmfulURL = "itisatrap.org/firefox/harmful.html"; - const trackerURLs = [ - "trackertest.org/", - "itisatracker.org/", - ]; - const whitelistURL = "itisatrap.org/?resource=itisatracker.org"; - const blockedURL = "itisatrap.org/firefox/blocked.html"; - - let update = "n:1000\ni:test-malware-simple\nad:1\n" + - "a:1:32:" + malwareURL.length + "\n" + - malwareURL + "\n"; - update += "n:1000\ni:test-phish-simple\nad:1\n" + - "a:1:32:" + phishURL.length + "\n" + - phishURL + "\n"; - update += "n:1000\ni:test-unwanted-simple\nad:1\n" + - "a:1:32:" + unwantedURL.length + "\n" + - unwantedURL + "\n"; - update += "n:1000\ni:test-harmful-simple\nad:1\n" + - "a:1:32:" + harmfulURL.length + "\n" + - harmfulURL + "\n"; - update += "n:1000\ni:test-track-simple\n" + - "ad:" + trackerURLs.length + "\n"; - trackerURLs.forEach((trackerURL, i) => { - update += "a:" + (i + 1) + ":32:" + trackerURL.length + "\n" + - trackerURL + "\n"; - }); - update += "n:1000\ni:test-trackwhite-simple\nad:1\n" + - "a:1:32:" + whitelistURL.length + "\n" + - whitelistURL; - update += "n:1000\ni:test-block-simple\nad:1\n" + - "a:1:32:" + blockedURL.length + "\n" + - blockedURL; - log("addMozEntries:", update); - - let db = Cc["@mozilla.org/url-classifier/dbservice;1"]. - getService(Ci.nsIUrlClassifierDBService); - - // nsIUrlClassifierUpdateObserver - let dummyListener = { - updateUrlRequested() { }, - streamFinished() { }, - // We notify observers when we're done in order to be able to make perf - // test results more consistent - updateError() { - Services.obs.notifyObservers(db, "mozentries-update-finished", "error"); - }, - updateSuccess() { - Services.obs.notifyObservers(db, "mozentries-update-finished", "success"); - }, - }; - - try { - let tables = "test-malware-simple,test-phish-simple,test-unwanted-simple,test-harmful-simple,test-track-simple,test-trackwhite-simple,test-block-simple"; - db.beginUpdate(dummyListener, tables, ""); - db.beginStream("", ""); - db.updateStream(update); - db.finishStream(); - db.finishUpdate(); - } catch (ex) { - // beginUpdate will throw harmlessly if there's an existing update in progress, ignore failures. - log("addMozEntries failed!", ex); - Services.obs.notifyObservers(db, "mozentries-update-finished", "exception"); - } - }, - - addMozEntriesFinishedPromise: new Promise(resolve => { - let finished = (subject, topic, data) => { - Services.obs.removeObserver(finished, "mozentries-update-finished"); - if (data == "error") { - Cu.reportError("addMozEntries failed to update the db!"); - } - resolve(); - }; - Services.obs.addObserver(finished, "mozentries-update-finished"); - }), - }; diff --git a/toolkit/components/url-classifier/UrlClassifierListManager.jsm b/toolkit/components/url-classifier/UrlClassifierListManager.jsm index 01fbcc373ca5..d10d25958897 100644 --- a/toolkit/components/url-classifier/UrlClassifierListManager.jsm +++ b/toolkit/components/url-classifier/UrlClassifierListManager.jsm @@ -63,6 +63,10 @@ this.PROT_ListManager = function PROT_ListManager() { // success, update failure, download error). this.updateCheckers_ = {}; this.requestBackoffs_ = {}; + + // This is only used by testcases to ensure SafeBrowsing.jsm is inited + this.registered = false; + this.dbService_ = Cc["@mozilla.org/url-classifier/dbservice;1"] .getService(Ci.nsIUrlClassifierDBService); @@ -81,6 +85,8 @@ PROT_ListManager.prototype.registerTable = function(tableName, providerName, updateUrl, gethashUrl) { + this.registered = true; + this.tablesData[tableName] = {}; if (!updateUrl) { log("Can't register table " + tableName + " without updateUrl"); @@ -178,6 +184,10 @@ PROT_ListManager.prototype.enableUpdate = function(tableName) { } }; +PROT_ListManager.prototype.isRegistered = function() { + return this.registered; +}; + /** * Returns true if any table associated with the updateUrl requires updates. * @param updateUrl - the updateUrl diff --git a/toolkit/components/url-classifier/nsIUrlListManager.idl b/toolkit/components/url-classifier/nsIUrlListManager.idl index a92d1debd43e..6bf01020ee0d 100644 --- a/toolkit/components/url-classifier/nsIUrlListManager.idl +++ b/toolkit/components/url-classifier/nsIUrlListManager.idl @@ -85,4 +85,10 @@ interface nsIUrlListManager : nsISupports * are not in back-off mode. */ uint64_t getBackOffTime(in ACString provider); + + /** + * Return true if someone registers a table, this is used by testcase + * to figure out it SafeBrowsing.jsm is initialized. + */ + boolean isRegistered(); }; diff --git a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp index 4713b5f528e5..ddbfd5944444 100644 --- a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp +++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp @@ -1228,7 +1228,7 @@ nsUrlClassifierLookupCallback::LookupComplete( // has registered the table. In the second case we should not call // complete. if ((!gethashUrl.IsEmpty() || - StringBeginsWith(result->mTableName, NS_LITERAL_CSTRING("test"))) && + nsUrlClassifierUtils::IsTestTable(result->mTableName)) && mDBService->GetCompleter(result->mTableName, getter_AddRefs(completer))) { // Bug 1323953 - Send the first 4 bytes for completion no matter how diff --git a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp index b635c1ffb7e9..59d887991388 100644 --- a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp +++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp @@ -306,9 +306,12 @@ static const struct { {"goog-passwordwhite-proto", CSD_WHITELIST}, // 8 // For testing purpose. - {"test-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 - {"test-unwanted-proto", UNWANTED_SOFTWARE}, // 3 - {"test-passwordwhite-proto", CSD_WHITELIST}, // 8 + {"moztest-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 + {"test-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 + {"moztest-unwanted-proto", UNWANTED_SOFTWARE}, // 3 + {"test-unwanted-proto", UNWANTED_SOFTWARE}, // 3 + {"moztest-passwordwhite-proto", CSD_WHITELIST}, // 8 + {"test-passwordwhite-proto", CSD_WHITELIST}, // 8 }; NS_IMETHODIMP @@ -344,7 +347,8 @@ nsUrlClassifierUtils::GetProvider(const nsACString& aTableName, nsACString& aProvider) { MutexAutoLock lock(mProviderDictLock); nsCString* provider = nullptr; - if (StringBeginsWith(aTableName, NS_LITERAL_CSTRING("test"))) { + + if (IsTestTable(aTableName)) { aProvider = NS_LITERAL_CSTRING(TESTING_TABLE_PROVIDER_NAME); } else if (mProviderDict.Get(aTableName, &provider)) { aProvider = provider ? *provider : EmptyCString(); @@ -1087,3 +1091,18 @@ bool nsUrlClassifierUtils::SpecialEncode(const nsACString& url, bool nsUrlClassifierUtils::ShouldURLEscape(const unsigned char c) const { return c <= 32 || c == '%' || c >= 127; } + +// moztest- tables are built-in created in LookupCache, they contain hardcoded +// url entries in it. moztest tables don't support updates. +// static +bool nsUrlClassifierUtils::IsMozTestTable(const nsACString& aTableName) { + return StringBeginsWith(aTableName, NS_LITERAL_CSTRING("moztest-")); +} + +// test- tables are used by testcases and can add custom test entries +// through update API. +// static +bool nsUrlClassifierUtils::IsTestTable(const nsACString& aTableName) { + return IsMozTestTable(aTableName) || + StringBeginsWith(aTableName, NS_LITERAL_CSTRING("test")); +} diff --git a/toolkit/components/url-classifier/nsUrlClassifierUtils.h b/toolkit/components/url-classifier/nsUrlClassifierUtils.h index 0b781a64e37b..4b5989b1aaad 100644 --- a/toolkit/components/url-classifier/nsUrlClassifierUtils.h +++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.h @@ -42,6 +42,10 @@ class nsUrlClassifierUtils final : public nsIUrlClassifierUtils, void CanonicalNum(const nsACString& num, uint32_t bytes, bool allowOctal, nsACString& _retval); + static bool IsMozTestTable(const nsACString& aTableName); + + static bool IsTestTable(const nsACString& aTableName); + private: nsUrlClassifierUtils(); ~nsUrlClassifierUtils(); diff --git a/toolkit/components/url-classifier/tests/browser/classifierHelper.js b/toolkit/components/url-classifier/tests/browser/classifierHelper.js index 2ea56d722c98..0f631e925b1b 100644 --- a/toolkit/components/url-classifier/tests/browser/classifierHelper.js +++ b/toolkit/components/url-classifier/tests/browser/classifierHelper.js @@ -9,6 +9,8 @@ let dbService = Cc["@mozilla.org/url-classifier/dbservice;1"] .getService(Ci.nsIUrlClassifierDBService); +let listmanager = Cc["@mozilla.org/url-classifier/listmanager;1"]. + getService(Ci.nsIUrlListManager); if (typeof(classifierHelper) == "undefined") { var classifierHelper = {}; @@ -34,27 +36,16 @@ classifierHelper._updatesToCleanup = []; // after the event had already been notified, we lookup entries to see if // they are already added to database. classifierHelper.waitForInit = function() { - // This url must sync with the table, url in SafeBrowsing.jsm addMozEntries - const table = "test-phish-simple"; - const url = "http://itisatrap.org/firefox/its-a-trap.html"; - let principal = Services.scriptSecurityManager.createCodebasePrincipal( - Services.io.newURI(url), {}); - return new Promise(function(resolve, reject) { - Services.obs.addObserver(function() { - resolve(); - }, "mozentries-update-finished"); + function checkForInit() { + if (listmanager.isRegistered()) { + resolve(); + } else { + setTimeout(() => { checkForInit(); }, 1000); + } + } - let listener = { - QueryInterface: ChromeUtils.generateQI(["nsIUrlClassifierUpdateObserver"]), - - handleEvent(value) { - if (value === table) { - resolve(); - } - }, - }; - dbService.lookup(principal, table, listener); + checkForInit(); }); }; diff --git a/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp b/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp index 59cc40c0aa70..d108af6cb679 100644 --- a/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp +++ b/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp @@ -7,7 +7,7 @@ using namespace mozilla; using namespace mozilla::safebrowsing; -static const char* kFilesInV2[] = {".pset", ".sbstore"}; +static const char* kFilesInV2[] = {".vlpset", ".sbstore"}; static const char* kFilesInV4[] = {".vlpset", ".metadata"}; #define V2_TABLE "gtest-malware-simple" @@ -45,7 +45,7 @@ TEST(UrlClassifierFailUpdate, CheckTableReset) ApplyUpdate(update); - // A successful V2 update should create .pset & .sbstore files + // A successful V2 update should create .vlpset & .sbstore files CheckFileExist(V2_TABLE, kFilesInV2, true); } diff --git a/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp b/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp index e9c170b5fad4..5fc1d1410fdc 100644 --- a/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp +++ b/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp @@ -17,8 +17,8 @@ TEST(UrlClassifierFindFullHash, Request) nsUrlClassifierUtils* urlUtil = nsUrlClassifierUtils::GetInstance(); nsTArray listNames; - listNames.AppendElement("test-phish-proto"); - listNames.AppendElement("test-unwanted-proto"); + listNames.AppendElement("moztest-phish-proto"); + listNames.AppendElement("moztest-unwanted-proto"); nsCString listStates[] = {nsCString("sta\x00te1", 7), nsCString("sta\x00te2", 7)}; diff --git a/toolkit/components/url-classifier/tests/mochitest/classifierCommon.js b/toolkit/components/url-classifier/tests/mochitest/classifierCommon.js index cb4022d7a5e4..99d1be1e3fb3 100644 --- a/toolkit/components/url-classifier/tests/mochitest/classifierCommon.js +++ b/toolkit/components/url-classifier/tests/mochitest/classifierCommon.js @@ -7,6 +7,8 @@ const {Services} = ChromeUtils.import("resource://gre/modules/Services.jsm"); var dbService = Cc["@mozilla.org/url-classifier/dbservice;1"] .getService(Ci.nsIUrlClassifierDBService); +var listmanager = Cc["@mozilla.org/url-classifier/listmanager;1"]. + getService(Ci.nsIUrlListManager); var timer; function setTimeout(callback, delay) { @@ -56,27 +58,11 @@ function doReload() { // after the event had already been notified, we lookup entries to see if // they are already added to database. function waitForInit() { - Services.obs.addObserver(function() { + if (listmanager.isRegistered()) { sendAsyncMessage("safeBrowsingInited"); - }, "mozentries-update-finished"); - - // This url must sync with the table, url in SafeBrowsing.jsm addMozEntries - const table = "test-phish-simple"; - const url = "http://itisatrap.org/firefox/its-a-trap.html"; - - let principal = Services.scriptSecurityManager.createCodebasePrincipal( - Services.io.newURI(url), {}); - - let listener = { - QueryInterface: ChromeUtils.generateQI(["nsIUrlClassifierUpdateObserver"]), - - handleEvent(value) { - if (value === table) { - sendAsyncMessage("safeBrowsingInited"); - } - }, - }; - dbService.lookup(principal, table, listener); + } else { + setTimeout(() => { waitForInit(); }, 1000); + } } addMessageListener("doUpdate", ({ testUpdate }) => { diff --git a/toolkit/components/url-classifier/tests/mochitest/test_allowlisted_annotations.html b/toolkit/components/url-classifier/tests/mochitest/test_allowlisted_annotations.html index 5feab7887422..e75797a42026 100644 --- a/toolkit/components/url-classifier/tests/mochitest/test_allowlisted_annotations.html +++ b/toolkit/components/url-classifier/tests/mochitest/test_allowlisted_annotations.html @@ -33,7 +33,7 @@ function clearPermissions() { } SpecialPowers.pushPrefEnv( - {"set": [["urlclassifier.trackingTable", "test-track-simple"], + {"set": [["urlclassifier.trackingTable", "moztest-track-simple"], ["privacy.trackingprotection.enabled", true], ["channelclassifier.allowlist_example", true]]}, test); diff --git a/toolkit/components/url-classifier/tests/mochitest/test_bug1254766.html b/toolkit/components/url-classifier/tests/mochitest/test_bug1254766.html index f4bed277ea5c..10e0c79b120b 100644 --- a/toolkit/components/url-classifier/tests/mochitest/test_bug1254766.html +++ b/toolkit/components/url-classifier/tests/mochitest/test_bug1254766.html @@ -15,11 +15,11 @@ diff --git a/toolkit/components/url-classifier/tests/mochitest/test_gethash.html b/toolkit/components/url-classifier/tests/mochitest/test_gethash.html index 71a1e6a4c95f..4afe11204c6d 100644 --- a/toolkit/components/url-classifier/tests/mochitest/test_gethash.html +++ b/toolkit/components/url-classifier/tests/mochitest/test_gethash.html @@ -107,6 +107,7 @@ SimpleTest.waitForExplicitFinish(); // directly and bypass classifier check SpecialPowers.pushPrefEnv({"set": [ ["browser.safebrowsing.malware.enabled", true], + ["urlclassifier.malwareTable", "test-malware-simple,test-unwanted-simple"], ["network.predictor.enabled", false], ["urlclassifier.gethash.timeout_ms", 30000], ]}, runTest); diff --git a/toolkit/components/url-classifier/tests/mochitest/test_safebrowsing_bug1272239.html b/toolkit/components/url-classifier/tests/mochitest/test_safebrowsing_bug1272239.html index de6ae41529d0..475145591dca 100644 --- a/toolkit/components/url-classifier/tests/mochitest/test_safebrowsing_bug1272239.html +++ b/toolkit/components/url-classifier/tests/mochitest/test_safebrowsing_bug1272239.html @@ -67,7 +67,7 @@ for (let list of lists) { continue; // For lists having a provider, it should have a correct gethash url - // For lists without a provider, for example, test-malware-simple, it should not + // For lists without a provider, for example, moztest-malware-simple, it should not // have a gethash url. var url = listmanager.getGethashUrl(list); var index = listsWithProvider.indexOf(list); diff --git a/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_bug1157081.html b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_bug1157081.html index 438f9b0c2400..06a84bd5db12 100644 --- a/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_bug1157081.html +++ b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_bug1157081.html @@ -61,7 +61,7 @@ function checkLoads(aWindow, aBlocked) { } SpecialPowers.pushPrefEnv( - {"set": [["urlclassifier.trackingTable", "test-track-simple"], + {"set": [["urlclassifier.trackingTable", "moztest-track-simple"], ["privacy.trackingprotection.enabled", true], ["browser.safebrowsing.malware.enabled", false], ["browser.safebrowsing.phishing.enabled", false], diff --git a/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js b/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js index cc713e4a8937..ccfc4193cc6b 100644 --- a/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js +++ b/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js @@ -23,6 +23,13 @@ Services.prefs.setBoolPref("browser.safebrowsing.blockedURIs.enabled", true); Services.prefs.setBoolPref("browser.safebrowsing.phishing.enabled", true); Services.prefs.setBoolPref("browser.safebrowsing.provider.test.disableBackoff", true); +// Add testing tables, we don't use moztest-* here because it doesn't support update +Services.prefs.setCharPref("urlclassifier.phishTable", "test-phish-simple"); +Services.prefs.setCharPref("urlclassifier.malwareTable", "test-harmful-simple,test-malware-simple,test-unwanted-simple"); +Services.prefs.setCharPref("urlclassifier.blockedTable", "test-block-simple"); +Services.prefs.setCharPref("urlclassifier.trackingTable", "test-track-simple"); +Services.prefs.setCharPref("urlclassifier.trackingWhitelistTable", "test-trackwhite-simple"); + // Enable all completions for tests Services.prefs.setCharPref("urlclassifier.disallow_completions", ""); diff --git a/toolkit/components/url-classifier/tests/unit/test_threat_type_conversion.js b/toolkit/components/url-classifier/tests/unit/test_threat_type_conversion.js index f383f930ecd7..f8f324cec095 100644 --- a/toolkit/components/url-classifier/tests/unit/test_threat_type_conversion.js +++ b/toolkit/components/url-classifier/tests/unit/test_threat_type_conversion.js @@ -9,6 +9,9 @@ function run_test() { equal(urlUtils.convertListNameToThreatType("goog-unwanted-proto"), 3); equal(urlUtils.convertListNameToThreatType("goog-harmful-proto"), 4); equal(urlUtils.convertListNameToThreatType("goog-phish-proto"), 5); + equal(urlUtils.convertListNameToThreatType("goog-badbinurl-proto"), 7); + equal(urlUtils.convertListNameToThreatType("goog-passwordwhite-proto"), 8); + equal(urlUtils.convertListNameToThreatType("goog-downloadwhite-proto"), 9); try { urlUtils.convertListNameToThreatType("bad-list-name"); @@ -22,10 +25,13 @@ function run_test() { // Test threat type to list name conversion. equal(urlUtils.convertThreatTypeToListNames(1), "goog-malware-proto"); - equal(urlUtils.convertThreatTypeToListNames(2), "googpub-phish-proto,test-phish-proto"); - equal(urlUtils.convertThreatTypeToListNames(3), "goog-unwanted-proto,test-unwanted-proto"); + equal(urlUtils.convertThreatTypeToListNames(2), "googpub-phish-proto,moztest-phish-proto,test-phish-proto"); + equal(urlUtils.convertThreatTypeToListNames(3), "goog-unwanted-proto,moztest-unwanted-proto,test-unwanted-proto"); equal(urlUtils.convertThreatTypeToListNames(4), "goog-harmful-proto"); equal(urlUtils.convertThreatTypeToListNames(5), "goog-phish-proto"); + equal(urlUtils.convertThreatTypeToListNames(7), "goog-badbinurl-proto"); + equal(urlUtils.convertThreatTypeToListNames(8), "goog-passwordwhite-proto,moztest-passwordwhite-proto,test-passwordwhite-proto"); + equal(urlUtils.convertThreatTypeToListNames(9), "goog-downloadwhite-proto"); try { urlUtils.convertThreatTypeToListNames(0);