Bug 340603: getting table updates consumes large amounts of memory

patch: enable streaming updates
r+a=darin

browser/app/profile/firefox.js

@@ -472,9 +472,7 @@ pref("browser.safebrowsing.enabled", true);
 pref("browser.safebrowsing.remoteLookups", false);
 
 // Non-enhanced mode (local url lists) URL list to check for updates
-pref("urlclassifier.provider.0.updateURL", "http://sb.google.com/safebrowsing/update?");
-// XXXtony: Move to this pref location once we update url-classifier.
-//pref("browser.safebrowsing.provider.0.updateURL", "http://sb.google.com/safebrowsing/update?");
+pref("browser.safebrowsing.provider.0.updateURL", "http://sb.google.com/safebrowsing/update?");
 
 pref("browser.safebrowsing.dataProvider", 0);
 

browser/components/safebrowsing/content/list-warden.js

@@ -59,6 +59,10 @@ function PROT_ListWarden() {
                       .getService(Ci.nsIUrlListManager);
   this.listManager_ = listManager;
 
+  // If we add support for changing local data providers, we need to add a
+  // pref observer that sets the update url accordingly.
+  this.listManager_.setUpdateUrl(gDataProvider.getUpdateURL());
+
   // Once we register tables, their respective names will be listed here.
   this.blackTables_ = [];
   this.whiteTables_ = [];

toolkit/components/url-classifier/content/listmanager.js

@@ -78,13 +78,6 @@
  */
 const kTableVersionPrefPrefix = "urlclassifier.tableversion.";
 
-/**
- * Pref name for the update server.
- * TODO: Add support for multiple providers.
- */
-const kUpdateServerUrl = "urlclassifier.provider.0.updateURL";
-
-
 /**
  * A ListManager keeps track of black and white lists and knows
  * how to update them.
@@ -96,10 +89,9 @@ function PROT_ListManager() {
   G_debugService.enableZone(this.debugZone);
 
   this.currentUpdateChecker_ = null;   // set when we toggle updates
-  this.rpcPending_ = false;
   this.prefs_ = new G_Preferences();
 
-  this.updateserverURL_ = this.prefs_.getPref(kUpdateServerUrl, null);
+  this.updateserverURL_ = null;
 
   // The lists we know about and the parses we can use to read
   // them. Default all to the earlies possible version (1.-1); this
@@ -145,6 +137,28 @@ PROT_ListManager.prototype.shutdown_ = function() {
   }
 }
 
+/**
+ * Set the url we check for updates.  If the new url is valid and different,
+ * update our table list.
+ * 
+ * After setting the update url, the caller is responsible for registering
+ * tables and then toggling update checking.  All the code for this logic is
+ * currently in browser/components/safebrowsing.  Maybe it should be part of
+ * the listmanger?
+ */
+PROT_ListManager.prototype.setUpdateUrl = function(url) {
+  G_Debug(this, "Set update url: " + url);
+  if (url != this.updateserverURL_) {
+    this.updateserverURL_ = url;
+    
+    // Remove old tables which probably aren't valid for the new provider.
+    for (var name in this.tablesData) {
+      delete this.tablesData[name];
+      delete this.tablesKnown_[name];
+    }
+  }
+}
+
 /**
  * Register a new table table
  * @param tableName - the name of the table
@@ -403,52 +417,24 @@ PROT_ListManager.prototype.checkForUpdates = function() {
   // Allow new updates to be scheduled from maybeToggleUpdateChecking()
   this.currentUpdateChecker_ = null;
 
-  if (this.rpcPending_) {
-    G_Debug(this, 'checkForUpdates: old callback is still pending...');
-    return false;
-  }
-
   if (!this.updateserverURL_) {
     G_Debug(this, 'checkForUpdates: no update server url');
     return false;
   }
   this.loadTableVersions_();
+
   G_Debug(this, 'checkForUpdates: scheduling request..');
-  this.rpcPending_ = true;
-  this.xmlFetcher_ = new PROT_XMLFetcher();
-  this.xmlFetcher_.get(this.getRequestURL_(this.updateserverURL_),
-                       BindToObject(this.rpcDone, this));
-  return true;
-}
-
-/**
- * A callback that is executed when the XMLHTTP request is finished
- *
- * @param data String containing the returned data
- */
-PROT_ListManager.prototype.rpcDone = function(data) {
-  G_Debug(this, "Called rpcDone");
-  this.rpcPending_ = false;
-  this.xmlFetcher_ = null;
-
-  if (!data || !data.length) {
-    G_Debug(this, "No data. Returning");
-    return;
+  var url = this.getRequestURL_(this.updateserverURL_);
+  var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"]
+                 .getService(Ci.nsIUrlClassifierStreamUpdater);
+  try {
+    streamer.updateUrl = url;
+  } catch (e) {
+    G_Debug(this, 'invalid url');
+    return false;
   }
 
-  // Only use the the data if the mac matches.
-  data = this.checkMac_(data);
-
-  if (data.length == 0) {
-    return;
-  }
-
-  var dbUpdateSrv = Cc["@mozilla.org/url-classifier/dbservice;1"]
-                    .getService(Ci.nsIUrlClassifierDBService);
-
-  // Update the tables on disk in a background thread.  Multiple updates
-  // will be queue up.
-  dbUpdateSrv.updateTables(data, BindToObject(this.setTableVersion_, this));
+  return streamer.downloadUpdates(BindToObject(this.setTableVersion_, this));
 }
 
 /**

toolkit/components/url-classifier/public/nsIUrlListManager.idl

@@ -48,9 +48,14 @@ interface nsIUrlListManagerCallback : nsISupports {
   void handleEvent(in boolean value);
 };
 
-[scriptable, uuid(e1a80418-1bf9-4bd7-a40d-94d549c24955)]
+[scriptable, uuid(914b3a54-47a8-4cb0-b9df-c89064f6bb34)]
 interface nsIUrlListManager : nsISupports
 {
+    /**
+     * Set the URL we check for updates.
+     */
+    void setUpdateUrl(in ACString url);
+
     /**
      * Add a table to the list of tables we are managing.  The name is a
      * string of the format provider_name-semantic_type-table_type.  For