try landing new safebrowsing protocol again. b=387196, r=tony, r=vlad (for new fixes)

browser/app/profile/firefox.js

@@ -493,7 +493,7 @@ pref("browser.safebrowsing.enabled", true);
 pref("browser.safebrowsing.remoteLookups", false);
 
 // Non-enhanced mode (local url lists) URL list to check for updates
-pref("browser.safebrowsing.provider.0.updateURL", "http://sb.google.com/safebrowsing/update?client={moz:client}&appver={moz:version}&");
+pref("browser.safebrowsing.provider.0.updateURL", "http://sb.google.com/safebrowsing/downloads?client={moz:client}&appver={moz:version}&pver=2.0");
 
 pref("browser.safebrowsing.dataProvider", 0);
 

browser/components/safebrowsing/content/list-warden.js

@@ -188,82 +188,51 @@ function MultiTableQuerier(url, whiteTables, blackTables, callback) {
   this.debugZone = "multitablequerier";
   this.url_ = url;
 
-  this.whiteTables_ = whiteTables;
-  this.blackTables_ = blackTables;
-  this.whiteIdx_ = 0;
-  this.blackIdx_ = 0;
+  this.whiteTables_ = {};
+  for (var i = 0; i < whiteTables.length; i++) {
+    this.whiteTables_[whiteTables[i]] = true;
+  }
+
+  this.blackTables_ = {};
+  for (var i = 0; i < blackTables.length; i++) {
+    this.blackTables_[blackTables[i]] = true;
+  }
 
   this.callback_ = callback;
   this.listManager_ = Cc["@mozilla.org/url-classifier/listmanager;1"]
                       .getService(Ci.nsIUrlListManager);
 }
 
-/**
- * We first query the white tables in succession.  If any contain
- * the url, we stop.  If none contain the url, we query the black tables
- * in succession.  If any contain the url, we call callback and
- * stop.  If none of the black tables contain the url, then we just stop
- * (i.e., it's not black url).
- */
 MultiTableQuerier.prototype.run = function() {
-  var whiteTable = this.whiteTables_[this.whiteIdx_];
-  var blackTable = this.blackTables_[this.blackIdx_];
-  if (whiteTable) {
-    //G_Debug(this, "Looking in whitetable: " + whiteTable);
-    ++this.whiteIdx_;
-    this.listManager_.safeExists(whiteTable, this.url_,
-                                 BindToObject(this.whiteTableCallback_,
-                                              this));
-  } else if (blackTable) {
-    //G_Debug(this, "Looking in blacktable: " + blackTable);
-    ++this.blackIdx_;
-    this.listManager_.safeExists(blackTable, this.url_,
-                                 BindToObject(this.blackTableCallback_,
-                                              this));
-  } else {
-    // No tables left to check, so we quit.
-    G_Debug(this, "Not found in any tables: " + this.url_);
+  /* ask the dbservice for all the tables to which this URL belongs */
+  this.listManager_.safeLookup(this.url_,
+                               BindToObject(this.lookupCallback_, this));
+}
+
+MultiTableQuerier.prototype.lookupCallback_ = function(result) {
+  if (result == "") {
     this.callback_(PROT_ListWarden.NOT_FOUND);
-
-    // Break circular ref to callback.
-    this.callback_ = null;
-    this.listManager_ = null;
+    return;
   }
-}
 
-/**
- * After checking a white table, we return here.  If the url is found,
- * we can stop.  Otherwise, we call run again.
- */
-MultiTableQuerier.prototype.whiteTableCallback_ = function(isFound) {
-  //G_Debug(this, "whiteTableCallback_: " + isFound);
-  if (!isFound)
-    this.run();
-  else {
-    G_Debug(this, "Found in whitelist: " + this.url_)
-    this.callback_(PROT_ListWarden.IN_WHITELIST);
+  var tableNames = result.split(",");
 
-    // Break circular ref to callback.
-    this.callback_ = null;
-    this.listManager_ = null;
+  /* Check the whitelists */
+  for (var i = 0; i < tableNames.length; i++) {
+    if (tableNames[i] && this.whiteTables_[tableNames[i]]) {
+      this.callback_(PROT_ListWarden.IN_WHITELIST);
+      return;
+    }
   }
-}
 
-/**
- * After checking a black table, we return here.  If the url is found,
- * we can call the callback and stop.  Otherwise, we call run again.
- */
-MultiTableQuerier.prototype.blackTableCallback_ = function(isFound) {
-  //G_Debug(this, "blackTableCallback_: " + isFound);
-  if (!isFound) {
-    this.run();
-  } else {
-    // In the blacklist, must be an evil url.
-    G_Debug(this, "Found in blacklist: " + this.url_)
-    this.callback_(PROT_ListWarden.IN_BLACKLIST);
-
-    // Break circular ref to callback.
-    this.callback_ = null;
-    this.listManager_ = null;
+  /* Check the blacklists */
+  for (var i = 0; i < tableNames.length; i++) {
+    if (tableNames[i] && this.blackTables_[tableNames[i]]) {
+      this.callback_(PROT_ListWarden.IN_BLACKLIST);
+      return;
+    }
   }
+
+  /* Not in any lists we know about */
+  this.callback_(PROT_ListWarden.NOT_FOUND);
 }

browser/components/safebrowsing/content/sb-loader.js

@@ -79,10 +79,8 @@ var safebrowsing = {
     // Register tables
     // XXX: move table names to a pref that we originally will download
     // from the provider (need to workout protocol details)
-    phishWarden.registerWhiteTable("goog-white-domain");
-    phishWarden.registerWhiteTable("goog-white-url");
-    phishWarden.registerBlackTable("goog-black-url");
-    phishWarden.registerBlackTable("goog-black-enchash");
+    phishWarden.registerWhiteTable("goog-white-exp");
+    phishWarden.registerBlackTable("goog-phish-sha128");
 
     // Download/update lists if we're in non-enhanced mode
     phishWarden.maybeToggleUpdateChecking();

toolkit/components/build/Makefile.in

@@ -129,6 +129,7 @@ endif
 
 ifdef MOZ_URL_CLASSIFIER
 SHARED_LIBRARY_LIBS += ../url-classifier/src/$(LIB_PREFIX)urlclassifier_s.$(LIB_SUFFIX)
+EXTRA_DSO_LDOPTS += $(ZLIB_LIBS)
 endif
 
 ifdef MOZ_FEEDS

toolkit/components/url-classifier/content/listmanager.js

@@ -38,49 +38,24 @@
 
 // A class that manages lists, namely white and black lists for
 // phishing or malware protection. The ListManager knows how to fetch,
-// update, and store lists, and knows the "kind" of list each is (is
-// it a whitelist? a blacklist? etc). However it doesn't know how the
-// lists are serialized or deserialized (the wireformat classes know
-// this) nor the specific format of each list. For example, the list
-// could be a map of domains to "1" if the domain is phishy. Or it
-// could be a map of hosts to regular expressions to match, who knows?
-// Answer: the trtable knows. List are serialized/deserialized by the
-// wireformat reader from/to trtables, and queried by the listmanager.
+// update, and store lists.
 //
 // There is a single listmanager for the whole application.
 //
-// The listmanager is used only in privacy mode; in advanced protection
-// mode a remote server is queried.
-//
-// How to add a new table:
-// 1) get it up on the server
-// 2) add it to tablesKnown
-// 3) if it is not a known table type (trtable.js), add an implementation
-//    for it in trtable.js
-// 4) add a check for it in the phishwarden's isXY() method, for example
-//    isBlackURL()
-//
-// TODO: obviously the way this works could use a lot of improvement. In
-//       particular adding a list should just be a matter of adding
-//       its name to the listmanager and an implementation to trtable
-//       (or not if a talbe of that type exists). The format and semantics
-//       of the list comprise its name, so the listmanager should easily
-//       be able to figure out what to do with what list (i.e., no
-//       need for step 4).
 // TODO more comprehensive update tests, for example add unittest check 
 //      that the listmanagers tables are properly written on updates
 
-/**
- * The base pref name for where we keep table version numbers.
- * We add append the table name to this and set the value to
- * the version.  E.g., tableversion.goog-black-enchash may have
- * a value of 1.1234.
- */
-const kTableVersionPrefPrefix = "urlclassifier.tableversion.";
-
 // How frequently we check for updates (30 minutes)
 const kUpdateInterval = 30 * 60 * 1000;
 
+function QueryAdapter(callback) {
+  this.callback_ = callback;
+};
+
+QueryAdapter.prototype.handleResponse = function(value) {
+  this.callback_.handleEvent(value);
+}
+
 /**
  * A ListManager keeps track of black and white lists and knows
  * how to update them.
@@ -96,28 +71,7 @@ function PROT_ListManager() {
 
   this.updateserverURL_ = null;
 
-  // The lists we know about and the parses we can use to read
-  // them. Default all to the earlies possible version (1.-1); this
-  // version will get updated when successfully read from disk or
-  // fetch updates.
-  this.tablesKnown_ = {};
   this.isTesting_ = false;
-  
-  if (this.isTesting_) {
-    // populate with some tables for unittesting
-    this.tablesKnown_ = {
-      // A major version of zero means local, so don't ask for updates       
-      "test1-foo-domain" : new PROT_VersionParser("test1-foo-domain", 0, -1),
-      "test2-foo-domain" : new PROT_VersionParser("test2-foo-domain", 0, -1),
-      "test-white-domain" : 
-        new PROT_VersionParser("test-white-domain", 0, -1, true /* require mac*/),
-      "test-mac-domain" :
-        new PROT_VersionParser("test-mac-domain", 0, -1, true /* require mac */)
-    };
-    
-    // expose the object for unittesting
-    this.wrappedJSObject = this;
-  }
 
   this.tablesData = {};
 
@@ -133,6 +87,9 @@ function PROT_ListManager() {
                                    10*60*1000 /* error time, 10min */,
                                    60*60*1000 /* backoff interval, 60min */,
                                    6*60*60*1000 /* max backoff, 6hr */);
+
+  this.dbService_ = Cc["@mozilla.org/url-classifier/dbservice;1"]
+                   .getService(Ci.nsIUrlClassifierDBService);
 }
 
 /**
@@ -163,7 +120,6 @@ PROT_ListManager.prototype.setUpdateUrl = function(url) {
     // Remove old tables which probably aren't valid for the new provider.
     for (var name in this.tablesData) {
       delete this.tablesData[name];
-      delete this.tablesKnown_[name];
     }
   }
 }
@@ -188,11 +144,8 @@ PROT_ListManager.prototype.setKeyUrl = function(url) {
  */
 PROT_ListManager.prototype.registerTable = function(tableName, 
                                                     opt_requireMac) {
-  var table = new PROT_VersionParser(tableName, 1, -1, opt_requireMac);
-  if (!table)
-    return false;
-  this.tablesKnown_[tableName] = table;
-  this.tablesData[tableName] = newUrlClassifierTable(tableName);
+  this.tablesData[tableName] = {};
+  this.tablesData[tableName].needsUpdate = false;
 
   return true;
 }
@@ -203,7 +156,7 @@ PROT_ListManager.prototype.registerTable = function(tableName,
  */
 PROT_ListManager.prototype.enableUpdate = function(tableName) {
   var changed = false;
-  var table = this.tablesKnown_[tableName];
+  var table = this.tablesData[tableName];
   if (table) {
     G_Debug(this, "Enabling table updates for " + tableName);
     table.needsUpdate = true;
@@ -220,7 +173,7 @@ PROT_ListManager.prototype.enableUpdate = function(tableName) {
  */
 PROT_ListManager.prototype.disableUpdate = function(tableName) {
   var changed = false;
-  var table = this.tablesKnown_[tableName];
+  var table = this.tablesData[tableName];
   if (table) {
     G_Debug(this, "Disabling table updates for " + tableName);
     table.needsUpdate = false;
@@ -235,14 +188,9 @@ PROT_ListManager.prototype.disableUpdate = function(tableName) {
  * Determine if we have some tables that need updating.
  */
 PROT_ListManager.prototype.requireTableUpdates = function() {
-  for (var type in this.tablesKnown_) {
-    // All tables with a major of 0 are internal tables that we never
-    // update remotely.
-    if (this.tablesKnown_[type].major == 0)
-      continue;
-     
+  for (var type in this.tablesData) {
     // Tables that need updating even if other tables dont require it
-    if (this.tablesKnown_[type].needsUpdate)
+    if (this.tablesData[type].needsUpdate)
       return true;
   }
 
@@ -263,6 +211,22 @@ PROT_ListManager.prototype.maybeStartManagingUpdates = function() {
   this.maybeToggleUpdateChecking();
 }
 
+PROT_ListManager.prototype.kickoffUpdate_ = function (tableData)
+{
+  this.startingUpdate_ = false;
+  // If the user has never downloaded tables, do the check now.
+  // If the user has tables, add a fuzz of a few minutes.
+  var initialUpdateDelay = 3000;
+  if (tableData != "") {
+    // Add a fuzz of 0-5 minutes.
+    initialUpdateDelay += Math.floor(Math.random() * (5 * 60 * 1000));
+  }
+
+  this.currentUpdateChecker_ =
+    new G_Alarm(BindToObject(this.checkForUpdates, this),
+                initialUpdateDelay);
+}
+
 /**
  * Determine if we have any tables that require updating.  Different
  * Wardens may call us with new tables that need to be updated.
@@ -281,26 +245,10 @@ PROT_ListManager.prototype.maybeToggleUpdateChecking = function() {
 
     // Multiple warden can ask us to reenable updates at the same time, but we
     // really just need to schedule a single update.
-    if (!this.currentUpdateChecker_) {
-      // If the user has never downloaded tables, do the check now.
-      // If the user has tables, add a fuzz of a few minutes.
-      this.loadTableVersions_();
-      var hasTables = false;
-      for (var table in this.tablesKnown_) {
-        if (this.tablesKnown_[table].minor != -1) {
-          hasTables = true;
-          break;
-        }
-      }
-
-      var initialUpdateDelay = 3000;
-      if (hasTables) {
-        // Add a fuzz of 0-5 minutes.
-        initialUpdateDelay += Math.floor(Math.random() * (5 * 60 * 1000));
-      }
-      this.currentUpdateChecker_ =
-        new G_Alarm(BindToObject(this.checkForUpdates, this),
-                    initialUpdateDelay);
+    if (!this.currentUpdateChecker && !this.startingUpdate_) {
+      this.startingUpdate_ = true;
+      // check the current state of tables in the database
+      this.dbService_.getTables(BindToObject(this.kickoffUpdate_, this));
     }
   } else {
     G_Debug(this, "Stopping managing lists (if currently active)");
@@ -363,116 +311,19 @@ PROT_ListManager.prototype.stopUpdateChecker = function() {
  *        value in the table corresponding to key.  If the table name does not
  *        exist, we return false, too.
  */
-PROT_ListManager.prototype.safeExists = function(table, key, callback) {
+PROT_ListManager.prototype.safeLookup = function(key, callback) {
   try {
-    G_Debug(this, "safeExists: " + table + ", " + key);
-    var map = this.tablesData[table];
-    map.exists(key, callback);
+    G_Debug(this, "safeLookup: " + key);
+    var cb = new QueryAdapter(callback);
+    this.dbService_.lookup(key,
+                           BindToObject(cb.handleResponse, cb),
+                           true);
   } catch(e) {
-    G_Debug(this, "safeExists masked failure for " + table + ", key " + key + ": " + e);
-    callback.handleEvent(false);
+    G_Debug(this, "safeLookup masked failure for key " + key + ": " + e);
+    callback.handleEvent("");
   }
 }
 
-/**
- * We store table versions in user prefs.  This method pulls the values out of
- * the user prefs and into the tablesKnown objects.
- */
-PROT_ListManager.prototype.loadTableVersions_ = function() {
-  // Pull values out of prefs.
-  var prefBase = kTableVersionPrefPrefix;
-  for (var table in this.tablesKnown_) {
-    var version = this.prefs_.getPref(prefBase + table, "1.-1");
-    G_Debug(this, "loadTableVersion " + table + ": " + version);
-    var tokens = version.split(".");
-    G_Assert(this, tokens.length == 2, "invalid version number");
-    
-    this.tablesKnown_[table].major = tokens[0];
-    this.tablesKnown_[table].minor = tokens[1];
-  }
-}
-
-/**
- * Callback from db update service.  As new tables are added to the db,
- * this callback is fired so we can update the version number.
- * @param versionString String containing the table update response from the
- *        server
- */
-PROT_ListManager.prototype.setTableVersion_ = function(versionString) {
-  G_Debug(this, "Got version string: " + versionString);
-  var versionParser = new PROT_VersionParser("");
-  if (versionParser.fromString(versionString)) {
-    var tableName = versionParser.type;
-    var versionNumber = versionParser.versionString();
-    var prefBase = kTableVersionPrefPrefix;
-
-    this.prefs_.setPref(prefBase + tableName, versionNumber);
-    
-    if (!this.tablesKnown_[tableName]) {
-      this.tablesKnown_[tableName] = versionParser;
-    } else {
-      this.tablesKnown_[tableName].ImportVersion(versionParser);
-    }
-    
-    if (!this.tablesData[tableName])
-      this.tablesData[tableName] = newUrlClassifierTable(tableName);
-  }
-
-  // Since this is called from the update server, it means there was
-  // a successful http request.  Make sure to notify the request backoff
-  // object.
-  this.requestBackoff_.noteServerResponse(200 /* ok */);
-}
-
-/**
- * Prepares a URL to fetch upates from. Format is a squence of 
- * type:major:minor, fields
- * 
- * @param url The base URL to which query parameters are appended; assumes
- *            already has a trailing ?
- * @returns the URL that we should request the table update from.
- */
-PROT_ListManager.prototype.getRequestURL_ = function(url) {
-  url += "version=";
-  var firstElement = true;
-  var requestMac = false;
-
-  for (var type in this.tablesKnown_) {
-    // All tables with a major of 0 are internal tables that we never
-    // update remotely.
-    if (this.tablesKnown_[type].major == 0)
-      continue;
-
-    // Check if the table needs updating
-    if (this.tablesKnown_[type].needsUpdate == false)
-      continue;
-
-    if (!firstElement) {
-      url += ","
-    } else {
-      firstElement = false;
-    }
-    url += type + ":" + this.tablesKnown_[type].toUrl();
-
-    if (this.tablesKnown_[type].requireMac)
-      requestMac = true;
-  }
-
-  // Request a mac only if at least one of the tables to be updated requires
-  // it
-  if (requestMac) {
-    // Add the wrapped key for requesting macs
-    if (!this.urlCrypto_)
-      this.urlCrypto_ = new PROT_UrlCrypto();
-
-    url += "&wrkey=" +
-      encodeURIComponent(this.urlCrypto_.getManager().getWrappedKey());
-  }
-
-  G_Debug(this, "getRequestURL returning: " + url);
-  return url;
-}
-
 /**
  * Updates our internal tables from the update server
  *
@@ -492,56 +343,87 @@ PROT_ListManager.prototype.checkForUpdates = function() {
   if (!this.requestBackoff_.canMakeRequest())
     return false;
 
-  // Check to make sure our tables still exist (maybe the db got corrupted or
-  // the user deleted the file).  If not, we need to reset the table version
-  // before sending the update check.
-  var tableNames = [];
-  for (var tableName in this.tablesKnown_) {
-    tableNames.push(tableName);
-  }
-  var dbService = Cc["@mozilla.org/url-classifier/dbservice;1"]
-                  .getService(Ci.nsIUrlClassifierDBService);
-  dbService.checkTables(tableNames.join(","),
-                        BindToObject(this.makeUpdateRequest_, this));
+  // Grab the current state of the tables from the database
+  this.dbService_.getTables(BindToObject(this.makeUpdateRequest_, this));
   return true;
 }
 
 /**
  * Method that fires the actual HTTP update request.
  * First we reset any tables that have disappeared.
- * @param tableNames String comma separated list of tables that
- *   don't exist
+ * @param tableData List of table data already in the database, in the form
+ *        tablename;<chunk ranges>\n
  */
-PROT_ListManager.prototype.makeUpdateRequest_ = function(tableNames) {
-  // Clear prefs that track table version if they no longer exist in the db.
-  var tables = tableNames.split(",");
-  for (var i = 0; i < tables.length; ++i) {
-    G_Debug(this, "Table |" + tables[i] + "| no longer exists, clearing pref.");
-    this.prefs_.clearPref(kTableVersionPrefPrefix + tables[i]);
+PROT_ListManager.prototype.makeUpdateRequest_ = function(tableData) {
+  var tableNames = {};
+  for (var tableName in this.tablesData) {
+    tableNames[tableName] = true;
   }
 
-  // Ok, now reload the table version.
-  this.loadTableVersions_();
+  var request = "";
+
+  // For each table already in the database, include the chunk data from
+  // the database
+  var lines = tableData.split("\n");
+  for (var i = 0; i < lines.length; i++) {
+    var fields = lines[i].split(";");
+    if (tableNames[fields[0]]) {
+      request += lines[i] + "\n";
+      delete tableNames[fields[0]];
+    }
+  }
+
+  // For each requested table that didn't have chunk data in the database,
+  // request it fresh
+  for (var tableName in tableNames) {
+    request += tableName + ";\n";
+  }
 
   G_Debug(this, 'checkForUpdates: scheduling request..');
-  var url = this.getRequestURL_(this.updateserverURL_);
   var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"]
                  .getService(Ci.nsIUrlClassifierStreamUpdater);
   try {
-    streamer.updateUrl = url;
+    streamer.updateUrl = this.updateserverURL_;
   } catch (e) {
     G_Debug(this, 'invalid url');
     return;
   }
 
-  if (!streamer.downloadUpdates(BindToObject(this.setTableVersion_, this),
+  if (!streamer.downloadUpdates(request,
+                                BindToObject(this.updateSuccess_, this),
+                                BindToObject(this.updateError_, this),
                                 BindToObject(this.downloadError_, this))) {
     G_Debug(this, "pending update, wait until later");
   }
 }
 
 /**
- * Callback function if there's a download error.
+ * Callback function if the update request succeeded.
+ * @param waitForUpdate String The number of seconds that the client should
+ *        wait before requesting again.
+ */
+PROT_ListManager.prototype.updateSuccess_ = function(waitForUpdate) {
+  G_Debug(this, "update success: " + waitForUpdate);
+  if (waitForUpdate) {
+    var delay = parseInt(waitForUpdate, 10);
+    // As long as the delay is something sane (5 minutes or more), update
+    // our delay time for requesting updates
+    if (delay >= (5 * 60) && this.updateChecker_)
+      this.updateChecker_.setDelay(delay * 1000);
+  }
+}
+
+/**
+ * Callback function if the update request succeeded.
+ * @param result String The error code of the failure
+ */
+PROT_ListManager.prototype.updateError_ = function(result) {
+  G_Debug(this, "update error: " + result);
+  // XXX: there was some trouble applying the updates.
+}
+
+/**
+ * Callback function when the download failed
  * @param status String http status or an empty string if connection refused.
  */
 PROT_ListManager.prototype.downloadError_ = function(status) {
@@ -568,17 +450,3 @@ PROT_ListManager.prototype.QueryInterface = function(iid) {
   Components.returnCode = Components.results.NS_ERROR_NO_INTERFACE;
   return null;
 }
-
-// A simple factory function that creates nsIUrlClassifierTable instances based
-// on a name.  The name is a string of the format
-// provider_name-semantic_type-table_type.  For example, goog-white-enchash
-// or goog-black-url.
-function newUrlClassifierTable(name) {
-  G_Debug("protfactory", "Creating a new nsIUrlClassifierTable: " + name);
-  var tokens = name.split('-');
-  var type = tokens[2];
-  var table = Cc['@mozilla.org/url-classifier/table;1?type=' + type]
-                .createInstance(Ci.nsIUrlClassifierTable);
-  table.name = name;
-  return table;
-}

toolkit/components/url-classifier/content/moz/alarm.js

@@ -134,6 +134,10 @@ G_Alarm.prototype.notify = function(timer) {
   return ret;
 }
 
+G_Alarm.prototype.setDelay = function(delay) {
+  this.timer_.delay = delay;
+}
+
 /**
  * XPCOM cruft
  */

toolkit/components/url-classifier/public/Makefile.in

@@ -10,7 +10,6 @@ XPIDL_MODULE = url-classifier
 
 XPIDLSRCS = nsIUrlClassifierDBService.idl \
             nsIUrlClassifierStreamUpdater.idl \
-            nsIUrlClassifierTable.idl \
             nsIUrlClassifierUtils.idl \
             nsIUrlListManager.idl \
             $(NULL)

toolkit/components/url-classifier/public/nsIUrlClassifierDBService.idl

@@ -49,32 +49,34 @@ interface nsIUrlClassifierCallback : nsISupports {
  * It provides async methods for querying and updating the database.  As the
  * methods complete, they call the callback function.
  */
-[scriptable, uuid(211d5360-4af6-4a1d-99c1-926d35861eaf)]
+[scriptable, uuid(10928bf5-e18d-4086-854b-6c4006f2b009)]
 interface nsIUrlClassifierDBService : nsISupports
 {
   /**
-   * Looks up a key in the database.  After it finds a value, it calls
-   * callback with the value as the first param.  If the key is not in
-   * the db or the table does not exist, the callback is called with
-   * an empty string parameter.
+   * Looks up a key in the database.
+   *
+   * @param key: The URL to search for.  This URL will be canonicalized
+   *        by the service.
+   * @param c: The callback will be called with a comma-separated list
+   *        of tables to which the key belongs.
+   * @param needsProxy: Should be true if the callback needs to be called
+   *        in the main thread, false if the callback is threadsafe.
    */
-  void exists(in ACString tableName, in ACString key,
-              in nsIUrlClassifierCallback c);
+  void lookup(in ACString spec,
+              in nsIUrlClassifierCallback c,
+              in boolean needsProxy);
 
   /**
-   * Checks to see if the tables exist.  tableNames is a comma separated list
-   * of table names to check.  The callback is called with a comma separated
-   * list of tables that no longer exist (either the db is corrupted or the
-   * user deleted the file).
+   * Lists the tables along with which chunks are available in each table.
+   * This list is in the format of the request body:
+   *   tablename;chunkdata\n
+   *   tablename2;chunkdata2\n
+   *
+   * For example:
+   *   goog-phish-regexp;a:10,14,30-40s:56,67
+   *   goog-white-regexp;a:1-3,5
    */
-  void checkTables(in ACString tableNames, in nsIUrlClassifierCallback c);
-
-  /**
-   * Updates the table in the background.  Calls callback after each table
-   * completes processing with the new table line as the parameter.  This
-   * allows us to keep track of the table version in our main thread.
-   */
-  void updateTables(in ACString updateString, in nsIUrlClassifierCallback c);
+  void getTables(in nsIUrlClassifierCallback c);
 
   ////////////////////////////////////////////////////////////////////////////
   // Incremental update methods.  These are named to match similar methods
@@ -89,10 +91,12 @@ interface nsIUrlClassifierDBService : nsISupports
   // interface, but it's tricky because of XPCOM proxies.
 
   /**
-   * Finish an incremental update.  This commits any pending tables and
-   * calls the callback for each completed table.
+   * Finish an incremental update.  Calls successCallback with the
+   * requested delay before the next update, or failureCallback with a
+   * result code.
    */
-  void finish(in nsIUrlClassifierCallback c);
+  void finish(in nsIUrlClassifierCallback successCallback,
+              in nsIUrlClassifierCallback failureCallback);
 
   /**
    * Cancel an incremental update.  This rolls back and pending changes.

toolkit/components/url-classifier/public/nsIUrlClassifierStreamUpdater.idl

@@ -44,7 +44,7 @@
  * downloading the whole update and then updating the sqlite database, we
  * update tables as the data is streaming in.
  */
-[scriptable, uuid(d9277fa4-7d51-4175-bd4e-546c080a83bf)]
+[scriptable, uuid(adf0dfaa-ce91-4cf2-ab15-f5810408e2ec)]
 interface nsIUrlClassifierStreamUpdater : nsISupports
 {
   /**
@@ -56,11 +56,14 @@ interface nsIUrlClassifierStreamUpdater : nsISupports
    * Try to download updates from updateUrl.  Only one instance of this
    * runs at a time, so we return false if another instance is already
    * running.
-   * @param aTableCallback Called once for each table that we successfully
-   *     download with the table header as the parameter.
-   * @param aErrorCallback Called if we get an http error or a connection
-   *   refused.
+   * @param aRequestBody The body for the request.
+   * @param aSuccessCallback Called after a successful update.
+   * @param aUpdateErrorCallback Called for problems applying the update
+   * @param aDownloadErrorCallback Called if we get an http error or a
+   *        connection refused error.
    */
-  boolean downloadUpdates(in nsIUrlClassifierCallback aTableCallback,
-                          in nsIUrlClassifierCallback aErrorCallback);
+  boolean downloadUpdates(in ACString aRequestBody,
+                          in nsIUrlClassifierCallback aSuccessCallback,
+                          in nsIUrlClassifierCallback aUpdateErrorCallback,
+                          in nsIUrlClassifierCallback aDownloadErrorCallback);
 };

toolkit/components/url-classifier/public/nsIUrlClassifierUtils.idl

@@ -39,27 +39,18 @@
  * Some utility methods used by the url classifier.
  */
 
-[scriptable, uuid(89ea43b0-a23f-4db2-8d23-6d90dc55f67a)]
+interface nsIURI;
+
+[scriptable, uuid(e4f0e59c-b922-48b0-a7b6-1735c1f96fed)]
 interface nsIUrlClassifierUtils : nsISupports
 {
   /**
-   * Canonicalize a URL.  DON'T USE THIS DIRECTLY.  Use
-   * PROT_EnchashDecrypter.prototype.getCanonicalUrl instead.  This method
-   * url-decodes a string, but it doesn't normalize the hostname.  The method
-   * in EnchashDecrypter first calls this method, then normalizes the hostname.
+   * Get the lookup string for a given URI.  This normalizes the hostname,
+   * url-decodes the string, and strips off the protocol.
    *
-   * @param url String to canonicalize
+   * @param uri URI to get the lookup key for.
    *
-   * @returns String containing the canonicalized url (maximally url-decoded,
-   *          then specially url-encoded)
+   * @returns String containing the canonicalized URI.
    */
-  ACString canonicalizeURL(in ACString url);
-
-  /**
-   * When canonicalizing hostnames, the final step is to url escape everything that
-   * is not alphanumeric or hyphen or dot.  The existing methods (escape,
-   * encodeURIComponent and encodeURI are close, but not exactly what we want
-   * so we write our own function to do this.
-   */
-  ACString escapeHostname(in ACString hostname);
+  ACString getKeyForURI(in nsIURI uri);
 };

toolkit/components/url-classifier/public/nsIUrlListManager.idl

@@ -39,16 +39,17 @@
 #include "nsISupports.idl"
 
 /**
- * Interface for a class that manages updates of multiple nsIUrlClassifierTables.
+ * Interface for a class that manages updates of the url classifier database.
  */
 
 // Interface for JS function callbacks
-[scriptable, function, uuid(ba913c5c-13d6-41eb-83c1-de2f4165a516)]
+[scriptable, function, uuid(fa4caf12-d057-4e7e-81e9-ce066ceee90b)]
 interface nsIUrlListManagerCallback : nsISupports {
-  void handleEvent(in boolean value);
+  void handleEvent(in ACString value);
 };
 
-[scriptable, uuid(d39982d6-da4f-4a27-8d91-f9c7b179aa33)]
+
+[scriptable, uuid(874d6c95-fb8b-4f89-b36d-85fe267ab356)]
 interface nsIUrlListManager : nsISupports
 {
     /**
@@ -82,10 +83,12 @@ interface nsIUrlListManager : nsISupports
     void disableUpdate(in ACString tableName);
 
     /**
-     * Lookup a key in a table.  Should not raise exceptions.  Calls
-     * the callback function with a single parameter: true if the key
-     * is in the table, false if it isn't.
+     * Lookup a key.  Should not raise exceptions.  Calls the callback
+     * function with a comma-separated list of tables to which the key
+     * belongs.
      */
-    void safeExists(in ACString tableName, in ACString key,
+    void safeLookup(in ACString key,
                     in nsIUrlListManagerCallback cb);
+
+    void checkForUpdates();
 };

toolkit/components/url-classifier/src/Makefile.in

@@ -16,6 +16,7 @@ REQUIRES = necko \
            storage \
            string \
            xpcom \
+           $(ZLIB_REQUIRES) \
            $(NULL)
 
 CPPSRCS = \
@@ -25,14 +26,16 @@ CPPSRCS = \
           $(NULL)
 
 LOCAL_INCLUDES = \
-                 -I$(srcdir)/../../build
+                 -I$(srcdir)/../../build \
                  $(NULL)
 
 
 # Same as JS components that are run through the pre-processor.
-EXTRA_PP_COMPONENTS = nsUrlClassifierTable.js \
-                      nsUrlClassifierLib.js \
+EXTRA_PP_COMPONENTS = nsUrlClassifierLib.js \
                       nsUrlClassifierListManager.js \
                       $(NULL)
 
 include $(topsrcdir)/config/rules.mk
+
+export:: $(topsrcdir)/security/nss/lib/freebl/sha512.c
+	$(INSTALL) $^ .

toolkit/components/url-classifier/src/nsUrlClassifierListManager.js

@@ -39,7 +39,6 @@ const Cc = Components.classes;
 const Ci = Components.interfaces;
 
 #include ../content/listmanager.js
-#include ../content/wireformat.js
 
 var modScope = this;
 function Init() {

toolkit/components/url-classifier/src/nsUrlClassifierStreamUpdater.cpp

@@ -37,10 +37,14 @@
  * ***** END LICENSE BLOCK ***** */
 
 #include "nsCRT.h"
+#include "nsIHttpChannel.h"
 #include "nsIObserverService.h"
+#include "nsIStringStream.h"
+#include "nsIUploadChannel.h"
 #include "nsIURI.h"
 #include "nsIUrlClassifierDBService.h"
 #include "nsStreamUtils.h"
+#include "nsStringStream.h"
 #include "nsToolkitCompsCID.h"
 #include "nsUrlClassifierStreamUpdater.h"
 #include "prlog.h"
@@ -63,8 +67,9 @@ class nsUrlClassifierStreamUpdater;
 class TableUpdateListener : public nsIStreamListener
 {
 public:
-  TableUpdateListener(nsIUrlClassifierCallback *aTableCallback,
-                      nsIUrlClassifierCallback *aErrorCallback,
+  TableUpdateListener(nsIUrlClassifierCallback *aSuccessCallback,
+                      nsIUrlClassifierCallback *aUpdateErrorCallback,
+                      nsIUrlClassifierCallback *aDownloadErrorCallback,
                       nsUrlClassifierStreamUpdater* aStreamUpdater);
   nsCOMPtr<nsIUrlClassifierDBService> mDBService;
 
@@ -76,20 +81,23 @@ private:
   ~TableUpdateListener() {}
 
   // Callback when table updates complete.
-  nsCOMPtr<nsIUrlClassifierCallback> mTableCallback;
-  nsCOMPtr<nsIUrlClassifierCallback> mErrorCallback;
+  nsCOMPtr<nsIUrlClassifierCallback> mSuccessCallback;
+  nsCOMPtr<nsIUrlClassifierCallback> mUpdateErrorCallback;
+  nsCOMPtr<nsIUrlClassifierCallback> mDownloadErrorCallback;
 
   // Reference to the stream updater that created this.
   nsUrlClassifierStreamUpdater *mStreamUpdater;
 };
 
 TableUpdateListener::TableUpdateListener(
-                                nsIUrlClassifierCallback *aTableCallback,
-                                nsIUrlClassifierCallback *aErrorCallback,
+                                nsIUrlClassifierCallback *aSuccessCallback,
+                                nsIUrlClassifierCallback *aUpdateErrorCallback,
+                                nsIUrlClassifierCallback *aDownloadErrorCallback,
                                 nsUrlClassifierStreamUpdater* aStreamUpdater)
 {
-  mTableCallback = aTableCallback;
-  mErrorCallback = aErrorCallback;
+  mSuccessCallback = aSuccessCallback;
+  mDownloadErrorCallback = aDownloadErrorCallback;
+  mUpdateErrorCallback = aUpdateErrorCallback;
   mStreamUpdater = aStreamUpdater;
 }
 
@@ -110,10 +118,13 @@ TableUpdateListener::OnStartRequest(nsIRequest *request, nsISupports* context)
   nsresult status;
   rv = httpChannel->GetStatus(&status);
   NS_ENSURE_SUCCESS(rv, rv);
+
+  LOG(("OnStartRequest (status %d)", status));
+
   if (NS_ERROR_CONNECTION_REFUSED == status ||
       NS_ERROR_NET_TIMEOUT == status) {
     // Assume that we're overloading the server and trigger backoff.
-    mErrorCallback->HandleEvent(nsCString());
+    mDownloadErrorCallback->HandleEvent(nsCString());
     return NS_ERROR_ABORT;
   }
 
@@ -151,7 +162,7 @@ TableUpdateListener::OnDataAvailable(nsIRequest *request,
 
     nsCAutoString strStatus;
     strStatus.AppendInt(status);
-    mErrorCallback->HandleEvent(strStatus);
+    mDownloadErrorCallback->HandleEvent(strStatus);
     return NS_ERROR_ABORT;
   }
 
@@ -180,7 +191,7 @@ TableUpdateListener::OnStopRequest(nsIRequest *request, nsISupports* context,
   // If we got the whole stream, call Finish to commit the changes.
   // Otherwise, call Cancel to rollback the changes.
   if (NS_SUCCEEDED(aStatus))
-    mDBService->Finish(mTableCallback);
+    mDBService->Finish(mSuccessCallback, mUpdateErrorCallback);
   else
     mDBService->CancelStream();
 
@@ -235,6 +246,8 @@ nsUrlClassifierStreamUpdater::GetUpdateUrl(nsACString & aUpdateUrl)
 NS_IMETHODIMP
 nsUrlClassifierStreamUpdater::SetUpdateUrl(const nsACString & aUpdateUrl)
 {
+  LOG(("Update URL is %s\n", PromiseFlatCString(aUpdateUrl).get()));
+
   nsresult rv = NS_NewURI(getter_AddRefs(mUpdateUrl), aUpdateUrl);
   NS_ENSURE_SUCCESS(rv, rv);
 
@@ -243,8 +256,10 @@ nsUrlClassifierStreamUpdater::SetUpdateUrl(const nsACString & aUpdateUrl)
 
 NS_IMETHODIMP
 nsUrlClassifierStreamUpdater::DownloadUpdates(
-                                nsIUrlClassifierCallback *aTableCallback,
-                                nsIUrlClassifierCallback *aErrorCallback,
+                                const nsACString &aRequestBody,
+                                nsIUrlClassifierCallback *aSuccessCallback,
+                                nsIUrlClassifierCallback *aUpdateErrorCallback,
+                                nsIUrlClassifierCallback *aDownloadErrorCallback,
                                 PRBool *_retval)
 {
   if (mIsUpdating) {
@@ -276,8 +291,12 @@ nsUrlClassifierStreamUpdater::DownloadUpdates(
   rv = NS_NewChannel(getter_AddRefs(mChannel), mUpdateUrl);
   NS_ENSURE_SUCCESS(rv, rv);
 
+  rv = AddRequestBody(aRequestBody);
+  NS_ENSURE_SUCCESS(rv, rv);
+
   // Bind to a different callback each time we invoke this method.
-  mListener = new TableUpdateListener(aTableCallback, aErrorCallback, this);
+  mListener = new TableUpdateListener(aSuccessCallback, aUpdateErrorCallback,
+                                      aDownloadErrorCallback, this);
 
   // Make the request
   rv = mChannel->AsyncOpen(mListener.get(), nsnull);
@@ -289,6 +308,35 @@ nsUrlClassifierStreamUpdater::DownloadUpdates(
   return NS_OK;
 }
 
+nsresult
+nsUrlClassifierStreamUpdater::AddRequestBody(const nsACString &aRequestBody)
+{
+  nsresult rv;
+  nsCOMPtr<nsIStringInputStream> strStream =
+    do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = strStream->SetData(aRequestBody.BeginReading(),
+                          aRequestBody.Length());
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsIUploadChannel> uploadChannel = do_QueryInterface(mChannel, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = uploadChannel->SetUploadStream(strStream,
+                                      NS_LITERAL_CSTRING("text/plain"),
+                                      -1);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = httpChannel->SetRequestMethod(NS_LITERAL_CSTRING("POST"));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return NS_OK;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // nsIObserver implementation
 

toolkit/components/url-classifier/src/nsUrlClassifierStreamUpdater.h

@@ -71,6 +71,8 @@ private:
   // Disallow copy constructor
   nsUrlClassifierStreamUpdater(nsUrlClassifierStreamUpdater&);
 
+  nsresult AddRequestBody(const nsACString &aRequestBody);
+
   PRBool mIsUpdating;
   PRBool mInitialized;
   nsCOMPtr<nsIURI> mUpdateUrl;

toolkit/components/url-classifier/src/nsUrlClassifierUtils.cpp

@@ -36,7 +36,11 @@
 
 #include "nsEscape.h"
 #include "nsString.h"
+#include "nsIURI.h"
+#include "nsNetUtil.h"
 #include "nsUrlClassifierUtils.h"
+#include "nsVoidArray.h"
+#include "prprf.h"
 
 static char int_to_hex_digit(PRInt32 i)
 {
@@ -44,6 +48,58 @@ static char int_to_hex_digit(PRInt32 i)
   return static_cast<char>(((i < 10) ? (i + '0') : ((i - 10) + 'A')));
 }
 
+static PRBool
+IsDecimal(const nsACString & num)
+{
+  for (PRUint32 i = 0; i < num.Length(); i++) {
+    if (!isdigit(num[i])) {
+      return PR_FALSE;
+    }
+  }
+
+  return PR_TRUE;
+}
+
+static PRBool
+IsHex(const nsACString & num)
+{
+  if (num.Length() < 3) {
+    return PR_FALSE;
+  }
+
+  if (num[0] != '0' || !(num[1] == 'x' || num[1] == 'X')) {
+    return PR_FALSE;
+  }
+
+  for (PRUint32 i = 2; i < num.Length(); i++) {
+    if (!isxdigit(num[i])) {
+      return PR_FALSE;
+    }
+  }
+
+  return PR_TRUE;
+}
+
+static PRBool
+IsOctal(const nsACString & num)
+{
+  if (num.Length() < 2) {
+    return PR_FALSE;
+  }
+
+  if (num[0] != '0') {
+    return PR_FALSE;
+  }
+
+  for (PRUint32 i = 1; i < num.Length(); i++) {
+    if (!isdigit(num[i]) || num[i] == '8' || num[i] == '9') {
+      return PR_FALSE;
+    }
+  }
+
+  return PR_TRUE;
+}
+
 nsUrlClassifierUtils::nsUrlClassifierUtils() : mEscapeCharmap(nsnull)
 {
 }
@@ -64,54 +120,252 @@ NS_IMPL_ISUPPORTS1(nsUrlClassifierUtils, nsIUrlClassifierUtils)
 /////////////////////////////////////////////////////////////////////////////
 // nsIUrlClassifierUtils
 
-/* ACString canonicalizeURL (in ACString url); */
 NS_IMETHODIMP
-nsUrlClassifierUtils::CanonicalizeURL(const nsACString & url, nsACString & _retval)
+nsUrlClassifierUtils::GetKeyForURI(nsIURI * uri, nsACString & _retval)
 {
-  nsCAutoString decodedUrl(url);
-  nsCAutoString temp;
-  while (NS_UnescapeURL(decodedUrl.get(), decodedUrl.Length(), 0, temp)) {
-    decodedUrl.Assign(temp);
-    temp.Truncate();
-  }
-  SpecialEncode(decodedUrl, _retval);
-  return NS_OK;
-}
+  nsCOMPtr<nsIURI> innerURI = NS_GetInnermostURI(uri);
+  if (!innerURI)
+    innerURI = uri;
+
+  nsCAutoString host;
+  innerURI->GetAsciiHost(host);
+
+  nsresult rv = CanonicalizeHostname(host, _retval);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCAutoString path;
+  rv = innerURI->GetPath(path);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // strip out anchors and query parameters
+  PRInt32 ref = path.FindChar('#');
+  if (ref != kNotFound)
+    path.SetLength(ref);
+
+  ref = path.FindChar('?');
+  if (ref != kNotFound)
+    path.SetLength(ref);
+
+  nsCAutoString temp;
+  rv = CanonicalizePath(path, temp);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  _retval.Append(temp);
 
-NS_IMETHODIMP
-nsUrlClassifierUtils::EscapeHostname(const nsACString & hostname,
-                                     nsACString & _retval)
-{
-  const char* curChar = hostname.BeginReading();
-  const char* end = hostname.EndReading();
-  while (curChar != end) {
-    unsigned char c = static_cast<unsigned char>(*curChar);
-    if (mEscapeCharmap->Contains(c)) {
-      _retval.Append('%');
-      _retval.Append(int_to_hex_digit(c / 16));
-      _retval.Append(int_to_hex_digit(c % 16));
-    } else {
-      _retval.Append(*curChar);
-    }
-    ++curChar;
-  }
-  
   return NS_OK;
 }
 
 /////////////////////////////////////////////////////////////////////////////
 // non-interface methods
 
+nsresult
+nsUrlClassifierUtils::CanonicalizeHostname(const nsACString & hostname,
+                                           nsACString & _retval)
+{
+  nsCAutoString unescaped;
+  if (!NS_UnescapeURL(PromiseFlatCString(hostname).get(),
+                      PromiseFlatCString(hostname).Length(),
+                      0, unescaped)) {
+    unescaped.Assign(hostname);
+  }
+
+  nsCAutoString cleaned;
+  CleanupHostname(unescaped, cleaned);
+
+  nsCAutoString temp;
+  ParseIPAddress(cleaned, temp);
+  if (!temp.IsEmpty()) {
+    cleaned.Assign(temp);
+  }
+
+  ToLowerCase(cleaned);
+  SpecialEncode(cleaned, PR_FALSE, _retval);
+
+  return NS_OK;
+}
+
+
+nsresult
+nsUrlClassifierUtils::CanonicalizePath(const nsACString & path,
+                                       nsACString & _retval)
+{
+  _retval.Truncate();
+
+  nsCAutoString decodedPath(path);
+  nsCAutoString temp;
+  while (NS_UnescapeURL(decodedPath.get(), decodedPath.Length(), 0, temp)) {
+    decodedPath.Assign(temp);
+    temp.Truncate();
+  }
+
+  SpecialEncode(decodedPath, PR_TRUE, _retval);
+  // XXX: lowercase the path?
+
+  return NS_OK;
+}
+
+void
+nsUrlClassifierUtils::CleanupHostname(const nsACString & hostname,
+                                      nsACString & _retval)
+{
+  _retval.Truncate();
+
+  const char* curChar = hostname.BeginReading();
+  const char* end = hostname.EndReading();
+  char lastChar = '\0';
+  while (curChar != end) {
+    unsigned char c = static_cast<unsigned char>(*curChar);
+    if (c == '.' && (lastChar == '\0' || lastChar == '.')) {
+      // skip
+    } else {
+      _retval.Append(*curChar);
+    }
+    lastChar = c;
+    ++curChar;
+  }
+
+  // cut off trailing dots
+  while (_retval[_retval.Length() - 1] == '.') {
+    _retval.SetLength(_retval.Length() - 1);
+  }
+}
+
+void
+nsUrlClassifierUtils::ParseIPAddress(const nsACString & host,
+                                     nsACString & _retval)
+{
+  _retval.Truncate();
+  nsACString::const_iterator iter, end;
+  host.BeginReading(iter);
+  host.EndReading(end);
+
+  if (host.Length() <= 15) {
+    // The Windows resolver allows a 4-part dotted decimal IP address to
+    // have a space followed by any old rubbish, so long as the total length
+    // of the string doesn't get above 15 characters. So, "10.192.95.89 xy"
+    // is resolved to 10.192.95.89.
+    // If the string length is greater than 15 characters, e.g.
+    // "10.192.95.89 xy.wildcard.example.com", it will be resolved through
+    // DNS.
+
+    if (FindCharInReadable(' ', iter, end)) {
+      end = iter;
+    }
+  }
+
+  for (host.BeginReading(iter); iter != end; iter++) {
+    if (!(isxdigit(*iter) || *iter == 'x' || *iter == 'X' || *iter == '.')) {
+      // not an IP
+      return;
+    }
+  }
+
+  host.BeginReading(iter);
+  nsCStringArray parts;
+  parts.ParseString(PromiseFlatCString(Substring(iter, end)).get(), ".");
+  if (parts.Count() > 4) {
+    return;
+  }
+
+  // If any potentially-octal numbers (start with 0 but not hex) have
+  // non-octal digits, no part of the ip can be in octal
+  // XXX: this came from the old javascript implementation, is it really
+  // supposed to be like this?
+  PRBool allowOctal = PR_TRUE;
+  for (PRInt32 i = 0; i < parts.Count(); i++) {
+    const nsCString& part = *parts[i];
+    if (part[0] == '0') {
+      for (PRUint32 j = 1; j < part.Length(); j++) {
+        if (part[j] == 'x') {
+          break;
+        }
+        if (part[j] == '8' || part[j] == '9') {
+          allowOctal = PR_FALSE;
+          break;
+        }
+      }
+    }
+  }
+
+  for (PRInt32 i = 0; i < parts.Count(); i++) {
+    nsCAutoString canonical;
+
+    if (i == parts.Count() - 1) {
+      CanonicalNum(*parts[i], 5 - parts.Count(), allowOctal, canonical);
+    } else {
+      CanonicalNum(*parts[i], 1, allowOctal, canonical);
+    }
+
+    if (canonical.IsEmpty()) {
+      _retval.Truncate();
+      return;
+    }
+
+    if (_retval.IsEmpty()) {
+      _retval.Assign(canonical);
+    } else {
+      _retval.Append('.');
+      _retval.Append(canonical);
+    }
+  }
+  return;
+}
+
+void
+nsUrlClassifierUtils::CanonicalNum(const nsACString& num,
+                                   PRUint32 bytes,
+                                   PRBool allowOctal,
+                                   nsACString& _retval)
+{
+  _retval.Truncate();
+
+  if (num.Length() < 1) {
+    return;
+  }
+
+  PRUint32 val;
+  if (allowOctal && IsOctal(num)) {
+    if (PR_sscanf(PromiseFlatCString(num).get(), "%o", &val) != 1) {
+      return;
+    }
+  } else if (IsDecimal(num)) {
+    if (PR_sscanf(PromiseFlatCString(num).get(), "%u", &val) != 1) {
+      return;
+    }
+  } else if (IsHex(num)) {
+  if (PR_sscanf(PromiseFlatCString(num).get(), num[1] == 'X' ? "0X%x" : "0x%x",
+                &val) != 1) {
+      return;
+    }
+  } else {
+    return;
+  }
+
+  while (bytes--) {
+    char buf[20];
+    PR_snprintf(buf, sizeof(buf), "%u", val & 0xff);
+    if (_retval.IsEmpty()) {
+      _retval.Assign(buf);
+    } else {
+      _retval = nsDependentCString(buf) + NS_LITERAL_CSTRING(".") + _retval;
+    }
+    val >>= 8;
+  }
+}
+
 // This function will encode all "special" characters in typical url
-// encoding, that is %hh where h is a valid hex digit.  See the comment in
-// the header file for details.
+// encoding, that is %hh where h is a valid hex digit.  It will also fold
+// any duplicated slashes.
 PRBool
-nsUrlClassifierUtils::SpecialEncode(const nsACString & url, nsACString & _retval)
+nsUrlClassifierUtils::SpecialEncode(const nsACString & url,
+                                    PRBool foldSlashes,
+                                    nsACString & _retval)
 {
   PRBool changed = PR_FALSE;
   const char* curChar = url.BeginReading();
   const char* end = url.EndReading();
 
+  unsigned char lastChar = '\0';
   while (curChar != end) {
     unsigned char c = static_cast<unsigned char>(*curChar);
     if (ShouldURLEscape(c)) {
@@ -125,9 +379,12 @@ nsUrlClassifierUtils::SpecialEncode(const nsACString & url, nsACString & _retval
       _retval.Append(int_to_hex_digit(c % 16));
 
       changed = PR_TRUE;
+    } else if (foldSlashes && (c == '/' && lastChar == '/')) {
+      // skip
     } else {
       _retval.Append(*curChar);
     }
+    lastChar = c;
     curChar++;
   }
   return changed;

toolkit/components/url-classifier/src/nsUrlClassifierUtils.h

@@ -77,19 +77,30 @@ public:
   nsUrlClassifierUtils();
   ~nsUrlClassifierUtils() {}
 
-  nsresult Init();
-
   NS_DECL_ISUPPORTS
   NS_DECL_NSIURLCLASSIFIERUTILS
 
+  nsresult Init();
+
+  nsresult CanonicalizeHostname(const nsACString & hostname,
+                                nsACString & _retval);
+  nsresult CanonicalizePath(const nsACString & url, nsACString & _retval);
+
   // This function will encode all "special" characters in typical url encoding,
   // that is %hh where h is a valid hex digit.  The characters which are encoded
   // by this function are any ascii characters under 32(control characters and
   // space), 37(%), and anything 127 or above (special characters).  Url is the
   // string to encode, ret is the encoded string.  Function returns true if
   // ret != url.
-  PRBool SpecialEncode(const nsACString & url, nsACString & _retval);
+  PRBool SpecialEncode(const nsACString & url,
+                       PRBool foldSlashes,
+                       nsACString & _retval);
 
+  void ParseIPAddress(const nsACString & host, nsACString & _retval);
+  void CanonicalNum(const nsACString & num,
+                    PRUint32 bytes,
+                    PRBool allowOctal,
+                    nsACString & _retval);
 private:
   // Disallow copy constructor
   nsUrlClassifierUtils(const nsUrlClassifierUtils&);
@@ -97,6 +108,8 @@ private:
   // Function to tell if we should encode a character.
   PRBool ShouldURLEscape(const unsigned char c) const;
 
+  void CleanupHostname(const nsACString & host, nsACString & _retval);
+
   nsAutoPtr<Charmap> mEscapeCharmap;
 };
 

toolkit/components/url-classifier/tests/Makefile.in

@@ -52,15 +52,11 @@ REQUIRES = \
 	string \
 	url-classifier \
 	xpcom \
+	necko \
 	$(NULL)
 
-# mochitests
-_TEST_FILES = \
-	test_enchash-decrypter.xhtml \
-	$(NULL)
-
-libs:: $(_TEST_FILES)
-	$(INSTALL) $^ $(DEPTH)/_tests/testing/mochitest/tests/$(relativesrcdir)
+# xpcshell tests
+XPCSHELL_TESTS=unit
 
 # simple c++ tests (no xpcom)
 CPPSRCS = \
@@ -76,6 +72,7 @@ LOCAL_INCLUDES = \
 
 LIBS = \
 	../src/$(LIB_PREFIX)urlclassifier_s.$(LIB_SUFFIX) \
+	$(MOZ_COMPONENT_LIBS) \
 	$(XPCOM_LIBS) \
 	$(NSPR_LIBS) \
 	$(NULL)

toolkit/components/url-classifier/tests/TestUrlClassifierUtils.cpp

@@ -39,6 +39,8 @@
 #include "nsEscape.h"
 #include "nsString.h"
 #include "nsUrlClassifierUtils.h"
+#include "nsNetUtil.h"
+#include "stdlib.h"
 
 static int gTotalTests = 0;
 static int gPassedTests = 0;
@@ -117,8 +119,8 @@ void TestEncodeHelper(const char* in, const char* expected)
 {
   nsCString out, strIn(in), strExp(expected);
   nsUrlClassifierUtils utils;
-  
-  utils.SpecialEncode(strIn, out);
+
+  utils.SpecialEncode(strIn, PR_TRUE, out);
   CheckEquals(strExp, out);
 }
 
@@ -136,7 +138,7 @@ void TestEnc()
   }
   nsUrlClassifierUtils utils;
   nsCString out;
-  utils.SpecialEncode(noenc, out);
+  utils.SpecialEncode(noenc, PR_FALSE, out);
   CheckEquals(noenc, out);
 
   // Test that all the chars that we should encode [0,32],37,[127,255] are
@@ -151,16 +153,18 @@ void TestEnc()
   }
   
   out.Truncate();
-  utils.SpecialEncode(yesAsString, out);
+  utils.SpecialEncode(yesAsString, PR_FALSE, out);
   CheckEquals(yesExpectedString, out);
+
+  TestEncodeHelper("blah//blah", "blah/blah");
 }
 
 void TestCanonicalizeHelper(const char* in, const char* expected)
 {
   nsCString out, strIn(in), strExp(expected);
   nsUrlClassifierUtils utils;
-  
-  utils.CanonicalizeURL(strIn, out);
+
+  utils.CanonicalizePath(strIn, out);
   CheckEquals(strExp, out);
 }
 
@@ -177,19 +181,142 @@ void TestCanonicalize()
       "~a!b@c#d$e%25f^00&11*22(33)44_55+");
 
   TestCanonicalizeHelper("", "");
-  TestCanonicalizeHelper("http://www.google.com", "http://www.google.com");
-  TestCanonicalizeHelper("http://%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/",
-                         "http://168.188.99.26/.secure/www.ebay.com/");
-  TestCanonicalizeHelper("http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",
-                         "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/");
+  TestCanonicalizeHelper("%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/",
+                         "168.188.99.26/.secure/www.ebay.com/");
+  TestCanonicalizeHelper("195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",
+                         "195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/");
+}
+
+void TestParseIPAddressHelper(const char *in, const char *expected)
+{
+  nsCString out, strIn(in), strExp(expected);
+  nsUrlClassifierUtils utils;
+  utils.Init();
+
+  utils.ParseIPAddress(strIn, out);
+  CheckEquals(strExp, out);
+}
+
+void TestParseIPAddress()
+{
+  TestParseIPAddressHelper("123.123.0.0.1", "");
+  TestParseIPAddressHelper("255.0.0.1", "255.0.0.1");
+  TestParseIPAddressHelper("12.0x12.01234", "12.18.2.156");
+  TestParseIPAddressHelper("276.2.3", "20.2.0.3");
+  TestParseIPAddressHelper("012.034.01.055", "10.28.1.45");
+  TestParseIPAddressHelper("0x12.0x43.0x44.0x01", "18.67.68.1");
+  TestParseIPAddressHelper("167838211", "10.1.2.3");
+  TestParseIPAddressHelper("3279880203", "195.127.0.11");
+  TestParseIPAddressHelper("0x12434401", "18.67.68.1");
+  TestParseIPAddressHelper("413960661", "24.172.137.213");
+  TestParseIPAddressHelper("03053104725", "24.172.137.213");
+  TestParseIPAddressHelper("030.0254.0x89d5", "24.172.137.213");
+  TestParseIPAddressHelper("1.234.4.0377", "1.234.4.255");
+  TestParseIPAddressHelper("1.2.3.00x0", "");
+  TestParseIPAddressHelper("10.192.95.89 xy", "10.192.95.89");
+  TestParseIPAddressHelper("10.192.95.89 xyz", "");
+  TestParseIPAddressHelper("1.2.3.0x0", "1.2.3.0");
+  TestParseIPAddressHelper("1.2.3.4", "1.2.3.4");
+}
+
+void TestCanonicalNumHelper(const char *in, PRUint32 bytes,
+                            bool allowOctal, const char *expected)
+{
+  nsCString out, strIn(in), strExp(expected);
+  nsUrlClassifierUtils utils;
+  utils.Init();
+
+  utils.CanonicalNum(strIn, bytes, allowOctal, out);
+  CheckEquals(strExp, out);
+}
+
+void TestCanonicalNum()
+{
+  TestCanonicalNumHelper("", 1, true, "");
+  TestCanonicalNumHelper("10", 0, true, "");
+  TestCanonicalNumHelper("45", 1, true, "45");
+  TestCanonicalNumHelper("0x10", 1, true, "16");
+  TestCanonicalNumHelper("367", 2, true, "1.111");
+  TestCanonicalNumHelper("012345", 3, true, "0.20.229");
+  TestCanonicalNumHelper("0173", 1, true, "123");
+  TestCanonicalNumHelper("09", 1, false, "9");
+  TestCanonicalNumHelper("0x120x34", 2, true, "");
+  TestCanonicalNumHelper("0x12fc", 2, true, "18.252");
+  TestCanonicalNumHelper("3279880203", 4, true, "195.127.0.11");
+  TestCanonicalNumHelper("0x0000059", 1, true, "89");
+  TestCanonicalNumHelper("0x00000059", 1, true, "89");
+  TestCanonicalNumHelper("0x0000067", 1, true, "103");
+}
+
+void TestHostnameHelper(const char *in, const char *expected)
+{
+  nsCString out, strIn(in), strExp(expected);
+  nsUrlClassifierUtils utils;
+  utils.Init();
+
+  utils.CanonicalizeHostname(strIn, out);
+  CheckEquals(strExp, out);
+}
+
+void TestHostname()
+{
+  TestHostnameHelper("abcd123;[]", "abcd123;[]");
+  TestHostnameHelper("abc.123", "abc.123");
+  TestHostnameHelper("abc..123", "abc.123");
+  TestHostnameHelper("trailing.", "trailing");
+  TestHostnameHelper("i love trailing dots....", "i%20love%20trailing%20dots");
+  TestHostnameHelper(".leading", "leading");
+  TestHostnameHelper("..leading", "leading");
+  TestHostnameHelper(".dots.", "dots");
+  TestHostnameHelper(".both.", "both");
+  TestHostnameHelper(".both..", "both");
+  TestHostnameHelper("..both.", "both");
+  TestHostnameHelper("..both..", "both");
+  TestHostnameHelper("..a.b.c.d..", "a.b.c.d");
+  TestHostnameHelper("..127.0.0.1..", "127.0.0.1");
+  TestHostnameHelper("asdf!@#$a", "asdf!@#$a");
+  TestHostnameHelper("AB CD 12354", "ab%20cd%2012354");
+  TestHostnameHelper("\1\2\3\4\112\177", "%01%02%03%04j%7F");
+  TestHostnameHelper("<>.AS/-+", "<>.as/-+");
+
+}
+
+void TestLongHostname()
+{
+  static const int kTestSize = 1024 * 150;
+  char *str = static_cast<char*>(malloc(kTestSize + 1));
+  memset(str, 'x', kTestSize);
+  str[kTestSize] = '\0';
+
+  nsUrlClassifierUtils utils;
+  utils.Init();
+
+  nsCAutoString out;
+  nsDependentCString in(str);
+  PRIntervalTime clockStart = PR_IntervalNow();
+  utils.CanonicalizeHostname(in, out);
+  PRIntervalTime clockEnd = PR_IntervalNow();
+
+  CheckEquals(in, out);
+
+  printf("CanonicalizeHostname on long string (%dms)\n",
+         PR_IntervalToMilliseconds(clockEnd - clockStart));
 }
 
 int main(int argc, char **argv)
 {
+  NS_LogInit();
+
   TestUnescape();
   TestEnc();
   TestCanonicalize();
+  TestCanonicalNum();
+  TestParseIPAddress();
+  TestHostname();
+  TestLongHostname();
+
   printf("%d of %d tests passed\n", gPassedTests, gTotalTests);
   // Non-zero return status signals test failure to build system.
+
   return (gPassedTests != gTotalTests);
 }