Bug 1192924: Check for updated system add-ons and download and install them. r=rhelmer

This performs the update check for system add-ons. It runs as part of the daily
add-on update checks similar to hotfix checks. Currently no URL is set so builds
won't actually start checking yet.

I've taken a few shortcuts here by only staging updates and needing a restart to
install as well as always downloading updates rather than using existing local
copies. At least the latter probably needs fixing before turning this on but
it makes more sense to iterate on those in tree.

--HG--
rename : toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app1/features/system1@tests.mozilla.org.xpi => toolkit/mozapps/extensions/test/xpcshell/data/system_addons/system1_1.xpi
rename : toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app3/features/system1@tests.mozilla.org.xpi => toolkit/mozapps/extensions/test/xpcshell/data/system_addons/system1_1_badcert.xpi
rename : toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app2/features/system1@tests.mozilla.org.xpi => toolkit/mozapps/extensions/test/xpcshell/data/system_addons/system1_2.xpi
rename : toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app1/features/system2@tests.mozilla.org.xpi => toolkit/mozapps/extensions/test/xpcshell/data/system_addons/system2_1.xpi
rename : toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app2/features/system3@tests.mozilla.org.xpi => toolkit/mozapps/extensions/test/xpcshell/data/system_addons/system3_1.xpi
extra : commitid : Ex9NNNduxhK
extra : rebase_source : 99088404019f53a8a76105c66edce404f2c7e454

testing/profiles/prefs_general.js

@@ -95,6 +95,7 @@ user_pref("extensions.update.url", "http://%(server)s/extensions-dummy/updateURL
 user_pref("extensions.update.background.url", "http://%(server)s/extensions-dummy/updateBackgroundURL");
 user_pref("extensions.blocklist.url", "http://%(server)s/extensions-dummy/blocklistURL");
 user_pref("extensions.hotfix.url", "http://%(server)s/extensions-dummy/hotfixURL");
+user_pref("extensions.systemAddon.update.url", "http://%(server)s/dummy-system-addons.xml");
 // Turn off extension updates so they don't bother tests
 user_pref("extensions.update.enabled", false);
 // Make sure opening about:addons won't hit the network

testing/talos/talos/config.py

@@ -140,6 +140,8 @@ DEFAULTS = dict(
             'http://127.0.0.1/plugins-dummy/updateCheckURL',
         'media.gmp-manager.url':
             'http://127.0.0.1/gmpmanager-dummy/update.xml',
+        'extensions.systemAddon.update.url':
+            'http://127.0.0.1/dummy-system-addons.xml',
         'media.navigator.enabled': True,
         'media.peerconnection.enabled': True,
         'media.navigator.permission.disabled': True,

testing/xpcshell/head.js

@@ -1505,6 +1505,7 @@ try {
       .getService(Components.interfaces.nsIPrefBranch);
 
     prefs.setCharPref("media.gmp-manager.url.override", "http://%(server)s/dummy-gmp-manager.xml");
+    prefs.setCharPref("extensions.systemAddon.update.url", "http://%(server)s/dummy-system-addons.xml");
     prefs.setCharPref("browser.selfsupport.url", "https://%(server)s/selfsupport-dummy/");
     prefs.setCharPref("toolkit.telemetry.server", "https://%(server)s/telemetry-dummy");
     prefs.setCharPref("browser.search.geoip.url", "https://%(server)s/geoip-dummy");

toolkit/mozapps/extensions/AddonManager.jsm

@@ -859,6 +859,13 @@ var AddonManagerInternal = {
     logger.debug(`Provider finished startup: ${providerName(aProvider)}`);
   },
 
+  _getProviderByName(aName) {
+    for (let provider of this.providers) {
+      if (providerName(provider) == aName)
+        return provider;
+    }
+  },
+
   /**
    * Initializes the AddonManager, loading any known providers and initializing
    * them.
@@ -1464,9 +1471,9 @@ var AddonManagerInternal = {
     let buPromise = Task.spawn(function* backgroundUpdateTask() {
       let hotfixID = this.hotfixID;
 
-      let checkHotfix = hotfixID &&
-                        Services.prefs.getBoolPref(PREF_APP_UPDATE_ENABLED) &&
-                        Services.prefs.getBoolPref(PREF_APP_UPDATE_AUTO);
+      let appUpdateEnabled = Services.prefs.getBoolPref(PREF_APP_UPDATE_ENABLED) &&
+                             Services.prefs.getBoolPref(PREF_APP_UPDATE_AUTO);
+      let checkHotfix = hotfixID && appUpdateEnabled;
 
       logger.debug("Background update check beginning");
 
@@ -1613,6 +1620,15 @@ var AddonManagerInternal = {
         }
       }
 
+      if (appUpdateEnabled) {
+        try {
+          yield AddonManagerInternal._getProviderByName("XPIProvider").updateSystemAddons();
+        }
+        catch (e) {
+          logger.warn("Failed to update system addons", e);
+        }
+      }
+
       logger.debug("Background update check complete");
       Services.obs.notifyObservers(null,
                                    "addons-background-update-complete",

toolkit/mozapps/extensions/internal/XPIProvider.jsm

@@ -42,6 +42,10 @@ XPCOMUtils.defineLazyModuleGetter(this, "BrowserToolboxProcess",
                                   "resource:///modules/devtools/client/framework/ToolboxProcess.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ConsoleAPI",
                                   "resource://gre/modules/devtools/shared/Console.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "ProductAddonChecker",
+                                  "resource://gre/modules/addons/ProductAddonChecker.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "UpdateUtils",
+                                  "resource://gre/modules/UpdateUtils.jsm");
 
 XPCOMUtils.defineLazyServiceGetter(this, "Blocklist",
                                    "@mozilla.org/extensions/blocklist;1",
@@ -99,6 +103,7 @@ const PREF_BRANCH_INSTALLED_ADDON     = "extensions.installedDistroAddon.";
 const PREF_SHOWN_SELECTION_UI         = "extensions.shownSelectionUI";
 const PREF_INTERPOSITION_ENABLED      = "extensions.interposition.enabled";
 const PREF_SYSTEM_ADDON_SET           = "extensions.systemAddonSet";
+const PREF_SYSTEM_ADDON_UPDATE_URL    = "extensions.systemAddon.update.url";
 
 const PREF_EM_MIN_COMPAT_APP_VERSION      = "extensions.minCompatibleAppVersion";
 const PREF_EM_MIN_COMPAT_PLATFORM_VERSION = "extensions.minCompatiblePlatformVersion";
@@ -310,6 +315,25 @@ LAZY_OBJECTS.forEach(name => {
 });
 
 
+// Behaves like Promise.all except waits for all promises to resolve/reject
+// before resolving/rejecting itself
+function waitForAllPromises(promises) {
+  return new Promise((resolve, reject) => {
+    let shouldReject = false;
+    let rejectValue = null;
+
+    let newPromises = [
+      for (p of promises)
+        p.catch(value => {
+          shouldReject = true;
+          rejectValue = value;
+        })
+    ]
+    Promise.all(newPromises)
+           .then((results) => shouldReject ? reject(rejectValue) : resolve(results));
+  });
+}
+
 function findMatchingStaticBlocklistItem(aAddon) {
   for (let item of STATIC_BLOCKLIST_PATTERNS) {
     if ("creator" in item && typeof item.creator == "string") {
@@ -2759,6 +2783,91 @@ this.XPIProvider = {
                                !XPIDatabase.writeAddonsList());
   },
 
+  updateSystemAddons: Task.async(function XPI_updateSystemAddons() {
+    // Download the list of system add-ons
+    let url = Preferences.get(PREF_SYSTEM_ADDON_UPDATE_URL, null);
+    if (!url)
+      return;
+
+    url = UpdateUtils.formatUpdateURL(url);
+
+    logger.info(`Starting system add-on update check from ${url}.`);
+    let addonList = yield ProductAddonChecker.getProductAddonList(url);
+
+    // If there was no list then do nothing.
+    if (!addonList) {
+      logger.info("No system add-ons list was returned.");
+      return;
+    }
+
+    addonList = [for (spec of addonList) { spec, path: null, addon: null }];
+
+    // Bug 1204159: If this matches the current set in the profile or app locations
+    // then just switch to those
+
+    let systemAddonLocation = XPIProvider.installLocationsByName[KEY_APP_SYSTEM_ADDONS];
+
+    // Download all the add-ons
+    // Bug 1204158: If we already have some of these locally then just use those
+    let downloadAddon = Task.async(function*(item) {
+      try {
+        item.path = yield ProductAddonChecker.downloadAddon(item.spec);
+        item.addon = yield loadManifestFromFile(nsIFile(item.path), systemAddonLocation);
+      }
+      catch (e) {
+        logger.error(`Failed to download system add-on ${item.spec.id}`, e);
+      }
+    });
+    yield Promise.all([for (item of addonList) downloadAddon(item)]);
+
+    // The download promises all resolve regardless, now check if they all
+    // succeeded
+    let validateAddon = (item) => {
+      if (item.spec.id != item.addon.id) {
+        logger.warn(`Downloaded system add-on expected to be ${item.spec.id} but was ${item.addon.id}.`);
+        return false;
+      }
+
+      if (item.spec.version != item.addon.version) {
+        logger.warn(`Expected system add-on ${item.spec.id} to be version ${item.version} but was ${item.addon.version}.`);
+        return false;
+      }
+
+      if (!systemAddonLocation.isValidAddon(item.addon))
+        return false;
+
+      return true;
+    }
+
+    try {
+      if (!addonList.every(item => item.path && item.addon && validateAddon(item))) {
+        throw new Error("Rejecting updated system add-on set that either could not " +
+                        "be downloaded or contained unusable add-ons.");
+      }
+
+      // Install into the install location
+      logger.info("Installing new system add-on set");
+      yield systemAddonLocation.installAddonSet([for (item of addonList) item.addon]);
+
+      // Bug 1204156: Switch to the new system add-ons without requiring a restart
+    }
+    finally {
+      // Delete the temporary files
+      logger.info("Deleting temporary files");
+      for (let item of addonList) {
+        // If this item downloaded delete the temporary file.
+        if (item.path) {
+          try {
+            yield OS.File.remove(item.path);
+          }
+          catch (e) {
+            logger.warn(`Failed to remove temporary file ${item.path}.`, e);
+          }
+        }
+      }
+    }
+  }),
+
   /**
    * Verifies that all installed add-ons are still correctly signed.
    */
@@ -7356,40 +7465,110 @@ Object.assign(SystemAddonInstallLocation.prototype, {
     return this._directory != null;
   },
 
+  isValidAddon: function(aAddon) {
+    if (aAddon.appDisabled) {
+      logger.warn(`System add-on ${aAddon.id} isn't compatible with the application.`);
+      return false;
+    }
+
+    if (aAddon.unpack) {
+      logger.warn(`System add-on ${aAddon.id} isn't a packed add-on.`);
+      return false;
+    }
+
+    if (!aAddon.bootstrap) {
+      logger.warn(`System add-on ${aAddon.id} isn't restartless.`);
+      return false;
+    }
+
+    return true;
+  },
+
   /**
    * Tests whether the loaded add-on information matches what is expected.
    */
   isValid: function(aAddons) {
     for (let id of Object.keys(this._addonSet.addons)) {
       if (!aAddons.has(id)) {
-        logger.warn("Expected add-on " + id + " is missing from the system add-on location.");
+        logger.warn(`Expected add-on ${id} is missing from the system add-on location.`);
         return false;
       }
 
       let addon = aAddons.get(id);
-      if (addon.appDisabled) {
-        logger.warn("System add-on " + id + " isn't compatible with the application.");
-        return false;
-      }
-
-      if (addon.unpack) {
-        logger.warn("System add-on " + id + " isn't a packed add-on.");
-        return false;
-      }
-
-      if (!addon.bootstrap) {
-        logger.warn("System add-on " + id + " isn't restartless.");
-        return false;
-      }
-
       if (addon.version != this._addonSet.addons[id].version) {
-        logger.warn("System add-on " + id + " wasn't the correct version.");
+        logger.warn(`Expected system add-on ${id} to be version ${this._addonSet.addons[id].version} but was ${addon.version}.`);
         return false;
       }
+
+      if (!this.isValidAddon(addon))
+        return false;
     }
 
     return true;
   },
+
+  /**
+   * Installs a new set of system add-ons into the location and updates the
+   * add-on set in prefs. We wait to switch state until a restart.
+   */
+  installAddonSet: Task.async(function(aAddons) {
+    // Make sure the base dir exists
+    yield OS.File.makeDir(this._baseDir.path, { ignoreExisting: true });
+
+    let newDir = this._baseDir.clone();
+
+    let uuidGen = Cc["@mozilla.org/uuid-generator;1"].
+                  getService(Ci.nsIUUIDGenerator);
+    newDir.append("blank");
+
+    while (true) {
+      newDir.leafName = uuidGen.generateUUID().toString();
+
+      try {
+        yield OS.File.makeDir(newDir.path, { ignoreExisting: false });
+        break;
+      }
+      catch (e) {
+        // Directory already exists, pick another
+      }
+    }
+
+    let copyAddon = Task.async(function*(addon) {
+      let target = OS.Path.join(newDir.path, addon.id + ".xpi");
+      logger.info(`Copying ${addon.id} from ${addon._sourceBundle.path} to ${target}.`);
+      try {
+        yield OS.File.copy(addon._sourceBundle.path, target);
+      }
+      catch (e) {
+        logger.error(`Failed to copy ${addon.id} from ${addon._sourceBundle.path} to ${target}.`, e);
+        throw e;
+      }
+      addon._sourceBundle = new nsIFile(target);
+    });
+
+    try {
+      yield waitForAllPromises([for (addon of aAddons) copyAddon(addon)]);
+    }
+    catch (e) {
+      try {
+        yield OS.File.removeDir(newDir.path, { ignorePermissions: true });
+      }
+      catch (e) {
+        logger.warn(`Failed to remove new system add-on directory ${newDir.path}.`, e);
+      }
+      throw e;
+    }
+
+    // All add-ons in position, create the new state and store it in prefs
+    let state = { schema: 1, directory: newDir.leafName, addons: {} };
+    for (let addon of aAddons) {
+      state.addons[addon.id] = {
+        version: addon.version
+      }
+    }
+
+    this._saveAddonSet(state);
+  }),
 });
 
 #ifdef XP_WIN

toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app1/features/system_1_1/bootstrap.js

@@ -1,18 +0,0 @@
-Components.utils.import("resource://gre/modules/Services.jsm");
-
-const ID = "system1@tests.mozilla.org";
-const VERSION = "1.0";
-
-function install(data, reason) {
-}
-
-function startup(data, reason) {
-  Services.prefs.setCharPref("bootstraptest." + ID + ".active_version", VERSION);
-}
-
-function shutdown(data, reason) {
-  Services.prefs.clearUserPref("bootstraptest." + ID + ".active_version");
-}
-
-function uninstall(data, reason) {
-}

toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app1/features/system_1_1/install.rdf

@@ -1,23 +0,0 @@
-<?xml version="1.0"?>
-
-<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
-
-  <Description about="urn:mozilla:install-manifest">
-    <em:id>system1@tests.mozilla.org</em:id>
-    <em:version>1.0</em:version>
-    <em:bootstrap>true</em:bootstrap>
-
-    <!-- Front End MetaData -->
-    <em:name>System Add-on 1</em:name>
-
-    <em:targetApplication>
-      <Description>
-        <em:id>xpcshell@tests.mozilla.org</em:id>
-        <em:minVersion>1</em:minVersion>
-        <em:maxVersion>5</em:maxVersion>
-      </Description>
-    </em:targetApplication>
-
-  </Description>
-</RDF>

toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app1/features/system_2_1/bootstrap.js

@@ -1,18 +0,0 @@
-Components.utils.import("resource://gre/modules/Services.jsm");
-
-const ID = "system2@tests.mozilla.org";
-const VERSION = "1.0";
-
-function install(data, reason) {
-}
-
-function startup(data, reason) {
-  Services.prefs.setCharPref("bootstraptest." + ID + ".active_version", VERSION);
-}
-
-function shutdown(data, reason) {
-  Services.prefs.clearUserPref("bootstraptest." + ID + ".active_version");
-}
-
-function uninstall(data, reason) {
-}

toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app1/features/system_2_1/install.rdf

@@ -1,23 +0,0 @@
-<?xml version="1.0"?>
-
-<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
-
-  <Description about="urn:mozilla:install-manifest">
-    <em:id>system2@tests.mozilla.org</em:id>
-    <em:version>1.0</em:version>
-    <em:bootstrap>true</em:bootstrap>
-
-    <!-- Front End MetaData -->
-    <em:name>System Add-on 2</em:name>
-
-    <em:targetApplication>
-      <Description>
-        <em:id>xpcshell@tests.mozilla.org</em:id>
-        <em:minVersion>1</em:minVersion>
-        <em:maxVersion>5</em:maxVersion>
-      </Description>
-    </em:targetApplication>
-
-  </Description>
-</RDF>

toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app2/features/system_1_2/bootstrap.js

@@ -1,18 +0,0 @@
-Components.utils.import("resource://gre/modules/Services.jsm");
-
-const ID = "system1@tests.mozilla.org";
-const VERSION = "2.0";
-
-function install(data, reason) {
-}
-
-function startup(data, reason) {
-  Services.prefs.setCharPref("bootstraptest." + ID + ".active_version", VERSION);
-}
-
-function shutdown(data, reason) {
-  Services.prefs.clearUserPref("bootstraptest." + ID + ".active_version");
-}
-
-function uninstall(data, reason) {
-}

toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app2/features/system_1_2/install.rdf

@@ -1,23 +0,0 @@
-<?xml version="1.0"?>
-
-<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
-
-  <Description about="urn:mozilla:install-manifest">
-    <em:id>system1@tests.mozilla.org</em:id>
-    <em:version>2.0</em:version>
-    <em:bootstrap>true</em:bootstrap>
-
-    <!-- Front End MetaData -->
-    <em:name>System Add-on 1</em:name>
-
-    <em:targetApplication>
-      <Description>
-        <em:id>xpcshell@tests.mozilla.org</em:id>
-        <em:minVersion>1</em:minVersion>
-        <em:maxVersion>5</em:maxVersion>
-      </Description>
-    </em:targetApplication>
-
-  </Description>
-</RDF>

toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app2/features/system_3_1/bootstrap.js

@@ -1,18 +0,0 @@
-Components.utils.import("resource://gre/modules/Services.jsm");
-
-const ID = "system3@tests.mozilla.org";
-const VERSION = "1.0";
-
-function install(data, reason) {
-}
-
-function startup(data, reason) {
-  Services.prefs.setCharPref("bootstraptest." + ID + ".active_version", VERSION);
-}
-
-function shutdown(data, reason) {
-  Services.prefs.clearUserPref("bootstraptest." + ID + ".active_version");
-}
-
-function uninstall(data, reason) {
-}

toolkit/mozapps/extensions/test/xpcshell/data/system_addons/app2/features/system_3_1/install.rdf

@@ -1,23 +0,0 @@
-<?xml version="1.0"?>
-
-<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
-
-  <Description about="urn:mozilla:install-manifest">
-    <em:id>system3@tests.mozilla.org</em:id>
-    <em:version>1.0</em:version>
-    <em:bootstrap>true</em:bootstrap>
-
-    <!-- Front End MetaData -->
-    <em:name>System Add-on 3</em:name>
-
-    <em:targetApplication>
-      <Description>
-        <em:id>xpcshell@tests.mozilla.org</em:id>
-        <em:minVersion>1</em:minVersion>
-        <em:maxVersion>5</em:maxVersion>
-      </Description>
-    </em:targetApplication>
-
-  </Description>
-</RDF>

toolkit/mozapps/extensions/test/xpcshell/test_system_reset.js

@@ -5,10 +5,22 @@ const PREF_SYSTEM_ADDON_SET = "extensions.systemAddonSet";
 // Enable signature checks for these tests
 Services.prefs.setBoolPref(PREF_XPI_SIGNATURES_REQUIRED, true);
 
-const featureDir = gProfD.clone();
-featureDir.append("features");
+const featureDir = FileUtils.getDir("ProfD", ["features"]);
 
-const distroDir = do_get_file("data/system_addons/app0");
+// Build the test sets
+let dir = FileUtils.getDir("ProfD", ["sysfeatures", "app1", "features"], true);
+do_get_file("data/system_addons/system1_1.xpi").copyTo(dir, "system1@tests.mozilla.org.xpi");
+do_get_file("data/system_addons/system2_1.xpi").copyTo(dir, "system2@tests.mozilla.org.xpi");
+
+dir = FileUtils.getDir("ProfD", ["sysfeatures", "app2", "features"], true);
+do_get_file("data/system_addons/system1_2.xpi").copyTo(dir, "system1@tests.mozilla.org.xpi");
+do_get_file("data/system_addons/system3_1.xpi").copyTo(dir, "system3@tests.mozilla.org.xpi");
+
+dir = FileUtils.getDir("ProfD", ["sysfeatures", "app3", "features"], true);
+do_get_file("data/system_addons/system1_1_badcert.xpi").copyTo(dir, "system1@tests.mozilla.org.xpi");
+do_get_file("data/system_addons/system3_1.xpi").copyTo(dir, "system3@tests.mozilla.org.xpi");
+
+const distroDir = FileUtils.getDir("ProfD", ["sysfeatures", "app0"], true);
 registerDirectory("XREAppDist", distroDir);
 
 createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "0");
@@ -136,10 +148,10 @@ add_task(function* test_updated() {
   featureDir.append(dirname);
 
   // Copy in the system add-ons
-  let file = do_get_file("data/system_addons/app1/features/system2@tests.mozilla.org.xpi");
-  file.copyTo(featureDir, file.leafName);
-  file = do_get_file("data/system_addons/app2/features/system3@tests.mozilla.org.xpi");
-  file.copyTo(featureDir, file.leafName);
+  let file = do_get_file("data/system_addons/system2_1.xpi");
+  file.copyTo(featureDir, "system2@tests.mozilla.org.xpi");
+  file = do_get_file("data/system_addons/system3_1.xpi");
+  file.copyTo(featureDir, "system3@tests.mozilla.org.xpi");
 
   // Inject it into the system set
   let addonSet = {
@@ -166,8 +178,8 @@ add_task(function* test_updated() {
 // An additional add-on in the directory should be ignored
 add_task(function* test_skips_additional() {
   // Copy in the system add-ons
-  let file = do_get_file("data/system_addons/app1/features/system1@tests.mozilla.org.xpi");
-  file.copyTo(featureDir, file.leafName);
+  let file = do_get_file("data/system_addons/system1_1.xpi");
+  file.copyTo(featureDir, "system1@tests.mozilla.org.xpi");
 
   startupManager(false);
 
@@ -191,8 +203,8 @@ add_task(function* test_revert() {
 
 // Putting it back will make the set work again
 add_task(function* test_reuse() {
-  let file = do_get_file("data/system_addons/app1/features/system2@tests.mozilla.org.xpi");
-  file.copyTo(featureDir, file.leafName);
+  let file = do_get_file("data/system_addons/system2_1.xpi");
+  file.copyTo(featureDir, "system2@tests.mozilla.org.xpi");
 
   startupManager(false);
 
@@ -214,8 +226,8 @@ add_task(function* test_corrupt_pref() {
 
 // An add-on with a bad certificate should cause us to use the default set
 add_task(function* test_bad_profile_cert() {
-  let file = do_get_file("data/system_addons/app3/features/system1@tests.mozilla.org.xpi");
-  file.copyTo(featureDir, file.leafName);
+  let file = do_get_file("data/system_addons/system1_1_badcert.xpi");
+  file.copyTo(featureDir, "system1@tests.mozilla.org.xpi");
 
   // Inject it into the system set
   let addonSet = {

toolkit/mozapps/extensions/test/xpcshell/test_system_update.js

@@ -0,0 +1,469 @@
+// Tests that we reset to the default system add-ons correctly when switching
+// application versions
+const PREF_SYSTEM_ADDON_SET           = "extensions.systemAddonSet";
+const PREF_SYSTEM_ADDON_UPDATE_URL    = "extensions.systemAddon.update.url";
+const PREF_XPI_STATE                  = "extensions.xpiState";
+const PREF_APP_UPDATE_ENABLED         = "app.update.enabled";
+
+Components.utils.import("resource://testing-common/httpd.js");
+const { computeHash } = Components.utils.import("resource://gre/modules/addons/ProductAddonChecker.jsm");
+
+// Enable signature checks for these tests
+//Services.prefs.setBoolPref(PREF_XPI_SIGNATURES_REQUIRED, true);
+
+const featureDir = FileUtils.getDir("ProfD", ["features"]);
+
+// Build the test sets
+let dir = FileUtils.getDir("ProfD", ["features", "prefilled"], true);
+do_get_file("data/system_addons/system2_2.xpi").copyTo(dir, "system2@tests.mozilla.org.xpi");
+do_get_file("data/system_addons/system3_2.xpi").copyTo(dir, "system3@tests.mozilla.org.xpi");
+
+// Mark these in the past so the startup file scan notices when files have changed properly
+FileUtils.getFile("ProfD", ["features", "prefilled", "system2@tests.mozilla.org.xpi"]).lastModifiedTime -= 10000;
+FileUtils.getFile("ProfD", ["features", "prefilled", "system3@tests.mozilla.org.xpi"]).lastModifiedTime -= 10000;
+
+const prefilledSet = {
+  schema: 1,
+  directory: dir.leafName,
+  addons: {
+    "system2@tests.mozilla.org": {
+      version: "2.0"
+    },
+    "system3@tests.mozilla.org": {
+      version: "2.0"
+    },
+  }
+};
+
+dir = FileUtils.getDir("ProfD", ["sysfeatures", "hidden", "features"], true);
+do_get_file("data/system_addons/system1_1.xpi").copyTo(dir, "system1@tests.mozilla.org.xpi");
+do_get_file("data/system_addons/system2_1.xpi").copyTo(dir, "system2@tests.mozilla.org.xpi");
+
+dir = FileUtils.getDir("ProfD", ["sysfeatures", "prefilled", "features"], true);
+do_get_file("data/system_addons/system2_2.xpi").copyTo(dir, "system2@tests.mozilla.org.xpi");
+do_get_file("data/system_addons/system3_2.xpi").copyTo(dir, "system3@tests.mozilla.org.xpi");
+
+const distroDir = FileUtils.getDir("ProfD", ["sysfeatures", "empty"], true);
+registerDirectory("XREAppDist", distroDir);
+
+createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "2");
+
+let testserver = new HttpServer();
+testserver.registerDirectory("/data/", do_get_file("data/system_addons"));
+testserver.start();
+let root = testserver.identity.primaryScheme + "://" +
+           testserver.identity.primaryHost + ":" +
+           testserver.identity.primaryPort + "/data/"
+Services.prefs.setCharPref(PREF_SYSTEM_ADDON_UPDATE_URL, root + "update.xml");
+
+function makeUUID() {
+  let uuidGen = AM_Cc["@mozilla.org/uuid-generator;1"].
+                getService(AM_Ci.nsIUUIDGenerator);
+  return uuidGen.generateUUID().toString();
+}
+
+function* serve_update(xml, perform_update) {
+  testserver.registerPathHandler("/data/update.xml", (request, response) => {
+    response.write(xml);
+  });
+
+  try {
+    yield perform_update();
+  }
+  finally {
+    testserver.registerPathHandler("/data/update.xml", null);
+  }
+}
+
+// Runs an update check making it use the passed in xml string. Uses the direct
+// call to the update function so we get rejections on failure.
+function* install_system_addons(xml) {
+  do_print("Triggering system add-on update check.");
+
+  yield serve_update(xml, function*() {
+    let { XPIProvider } = Components.utils.import("resource://gre/modules/addons/XPIProvider.jsm");
+    yield XPIProvider.updateSystemAddons();
+  });
+}
+
+// Runs a full add-on update check which will in some cases do a system add-on
+// update check. Always succeeds.
+function* update_all_addons(xml) {
+  do_print("Triggering full add-on update check.");
+
+  yield serve_update(xml, function() {
+    return new Promise(resolve => {
+      Services.obs.addObserver(function() {
+        Services.obs.removeObserver(arguments.callee, "addons-background-update-complete");
+
+        resolve();
+      }, "addons-background-update-complete", false);
+
+      // Trigger the background update timer handler
+      gInternalManager.notify(null);
+    });
+  });
+}
+
+// Builds an update.xml file for an update check based on the data passed.
+function* build_xml(addons) {
+  let xml = `<?xml version="1.0" encoding="UTF-8"?>\n\n<updates>\n`;
+  if (addons) {
+    xml += `  <addons>\n`;
+    for (let addon of addons) {
+      xml += `    <addon id="${addon.id}" URL="${root + addon.path}" version="${addon.version}"`;
+      if (addon.size)
+        xml += ` size="${addon.size}"`;
+      if (addon.hashFunction)
+        xml += ` hashFunction="${addon.hashFunction}"`;
+      if (addon.hashValue)
+        xml += ` hashValue="${addon.hashValue}"`;
+      xml += `/>\n`;
+    }
+    xml += `  </addons>\n`;
+  }
+  xml += `</updates>\n`;
+
+  return xml;
+}
+
+function* check_installed(inProfile, ...versions) {
+  for (let i = 0; i < versions.length; i++) {
+    let id = "system" + (i + 1) + "@tests.mozilla.org";
+    let addon = yield promiseAddonByID(id);
+
+    if (versions[i]) {
+      // Add-on should be installed
+      do_check_neq(addon, null);
+      do_check_eq(addon.version, versions[i]);
+      do_check_true(addon.isActive);
+      do_check_false(addon.foreignInstall);
+
+      // Verify the add-ons file is in the right place
+      let uri = addon.getResourceURI(null);
+      do_check_true(uri instanceof AM_Ci.nsIFileURL);
+
+      let file = uri.file.parent;
+      if (inProfile) {
+        file = file.parent;
+        do_check_eq(file.leafName, "features");
+        file = file.parent;
+        do_check_eq(file.path, gProfD.path);
+      }
+      else {
+        do_check_eq(file.leafName, "features");
+        file = file.parent;
+        do_check_eq(file.path, distroDir.path);
+      }
+
+      //do_check_eq(addon.signedState, AddonManager.SIGNEDSTATE_SYSTEM);
+
+      // Verify the add-on actually started
+      let installed = Services.prefs.getCharPref("bootstraptest." + id + ".active_version");
+      do_check_eq(installed, versions[i]);
+    }
+    else {
+      if (inProfile) {
+        // Add-on should not be installed
+        do_check_eq(addon, null);
+      }
+      else {
+        // Either add-on should not be installed or it shouldn't be active
+        do_check_true(!addon || !addon.isActive);
+      }
+
+      try {
+        Services.prefs.getCharPref("bootstraptest." + id + ".active_version");
+        do_throw("Expected pref to be missing");
+      }
+      catch (e) {
+      }
+    }
+  }
+}
+
+
+/**
+ * Defines the set of initial conditions to run each test against. Each should
+ * define the following properties:
+ *
+ * setup:        A task to setup the profile into the initial state.
+ * initialState: The initial expected system add-on state after setup has run.
+ */
+const TEST_CONDITIONS = {
+  // Runs tests with no updated or default system add-ons initially installed
+  blank: {
+    setup: function*() {
+      Services.prefs.clearUserPref(PREF_SYSTEM_ADDON_SET);
+      distroDir.leafName = "empty";
+    },
+    initialState: [false, null, null, null, null, null],
+  },
+
+  // Runs tests with default system add-ons installed
+  withAppSet: {
+    setup: function*() {
+      Services.prefs.clearUserPref(PREF_SYSTEM_ADDON_SET);
+      distroDir.leafName = "prefilled";
+    },
+    initialState: [false, null, "2.0", "2.0", null, null],
+  },
+
+  // Runs tests with updated system add-ons installed
+  withProfileSet: {
+    setup: function*() {
+      Services.prefs.setCharPref(PREF_SYSTEM_ADDON_SET, JSON.stringify(prefilledSet));
+      distroDir.leafName = "empty";
+    },
+    initialState: [true, null, "2.0", "2.0", null, null],
+  },
+
+  // Runs tests with both default and updated system add-ons installed
+  withBothSets: {
+    setup: function*() {
+      Services.prefs.setCharPref(PREF_SYSTEM_ADDON_SET, JSON.stringify(prefilledSet));
+      distroDir.leafName = "hidden";
+    },
+    initialState: [true, null, "2.0", "2.0", null, null],
+  },
+};
+
+
+/**
+ * The tests to run. Each test must define an updateList or test. The following
+ * properties are used:
+ *
+ * updateList: The set of add-ons the server should respond with.
+ * test:       A function to run to perform the update check (replaces
+ *             updateList)
+ * fails:      An optional property, if true the update check is expected to
+ *             fail.
+ * finalState: An optional property, the expected final state of system add-ons,
+ *             if missing the test condition's initialState is used.
+ */
+const TESTS = {
+  // Test that an error response does nothing
+  error: {
+    test: function*() {
+      try {
+        yield install_system_addons("foobar");
+        do_throw("Expected to fail the update check");
+      }
+      catch (e) {
+        do_check_true(true, "Expected to fail the update check");
+      }
+    },
+  },
+
+  // Test that a blank response does nothing
+  blank: {
+    updateList: null,
+  },
+
+  // Test that an empty list updates to an empty set of system add-ons
+  empty: {
+    updateList: [],
+    finalState: [true, null, null, null, null, null]
+  },
+
+  // Tests that a new set of system add-ons gets installed
+  newset: {
+    updateList: [
+      { id: "system4@tests.mozilla.org", version: "1.0", path: "system4_1.xpi" },
+      { id: "system5@tests.mozilla.org", version: "1.0", path: "system5_1.xpi" }
+    ],
+    finalState: [true, null, null, null, "1.0", "1.0"]
+  },
+
+  // Tests that an upgraded set of system add-ons gets installed
+  upgrades: {
+    updateList: [
+      { id: "system2@tests.mozilla.org", version: "3.0", path: "system2_3.xpi" },
+      { id: "system3@tests.mozilla.org", version: "3.0", path: "system3_3.xpi" }
+    ],
+    finalState: [true, null, "3.0", "3.0", null, null]
+  },
+
+  // Tests that a set of system add-ons, some new, some existing gets installed
+  overlapping: {
+    updateList: [
+      { id: "system1@tests.mozilla.org", version: "1.0", path: "system1_2.xpi" },
+      { id: "system2@tests.mozilla.org", version: "1.0", path: "system2_2.xpi" },
+      { id: "system3@tests.mozilla.org", version: "1.0", path: "system3_3.xpi" },
+      { id: "system4@tests.mozilla.org", version: "1.0", path: "system4_1.xpi" }
+    ],
+    finalState: [true, "2.0", "2.0", "3.0", "1.0", null]
+  },
+
+  // Specifying an incorrect version should stop us updating anything
+  badVersion: {
+    fails: true,
+    updateList: [
+      { id: "system2@tests.mozilla.org", version: "4.0", path: "system2_3.xpi" },
+      { id: "system3@tests.mozilla.org", version: "3.0", path: "system3_3.xpi" }
+    ],
+  },
+
+  // Specifying an invalid size should stop us updating anything
+  badSize: {
+    fails: true,
+    updateList: [
+      { id: "system2@tests.mozilla.org", version: "3.0", path: "system2_3.xpi", size: 2 },
+      { id: "system3@tests.mozilla.org", version: "3.0", path: "system3_3.xpi" }
+    ],
+  },
+
+  // Specifying an incorrect hash should stop us updating anything
+  badHash: {
+    fails: true,
+    updateList: [
+      { id: "system2@tests.mozilla.org", version: "3.0", path: "system2_3.xpi" },
+      { id: "system3@tests.mozilla.org", version: "3.0", path: "system3_3.xpi", hashFunction: "sha1", hashValue: "205a4c49bd513ebd30594e380c19e86bba1f83e2" }
+    ],
+  },
+
+  // Correct sizes and hashes should work
+  checkSizeHash: {
+    updateList: [
+      { id: "system2@tests.mozilla.org", version: "3.0", path: "system2_3.xpi", size: 858 },
+      { id: "system3@tests.mozilla.org", version: "3.0", path: "system3_3.xpi", hashFunction: "sha1", hashValue: "105a4c49bd513ebd30594e380c19e86bba1f83e2" },
+      { id: "system5@tests.mozilla.org", version: "1.0", path: "system5_1.xpi", size: 857, hashFunction: "sha1", hashValue: "664e9218be3c9acbb9029e715c1e5d2fbb4ea2cc" }
+    ],
+    finalState: [true, null, "3.0", "3.0", null, "1.0"]
+  },
+}
+
+add_task(function* setup() {
+  // Initialise the profile
+  startupManager();
+  yield promiseShutdownManager();
+})
+
+function* setup_conditions(setup) {
+  do_print("Setting up conditions.");
+  yield setup.setup();
+
+  // Blow away the cache to force a rescan of the filesystem
+  Services.prefs.clearUserPref(PREF_XPI_STATE);
+  startupManager(false);
+
+  // Make sure the initial state is correct
+  do_print("Checking initial state.");
+  yield check_installed(...setup.initialState);
+}
+
+function* verify_state(finalState) {
+  do_print("Checking final state.");
+
+  // Bug 1204156: Currently switching to the new state requires a restart
+  // yield check_installed(...finalState);
+
+  // Check that the new state is active after a restart
+  yield promiseRestartManager();
+  yield check_installed(...finalState);
+}
+
+function* exec_test(setup, test) {
+  yield setup_conditions(setup);
+
+  try {
+    if ("test" in test) {
+      yield test.test();
+    }
+    else {
+      yield install_system_addons(yield build_xml(test.updateList));
+    }
+
+    if (test.fails) {
+      do_throw("Expected this test to fail");
+    }
+  }
+  catch (e) {
+    if (!test.fails) {
+      do_throw(e);
+    }
+  }
+
+  yield verify_state(test.finalState ? test.finalState : setup.initialState);
+
+  yield promiseShutdownManager();
+}
+
+for (let setup of Object.keys(TEST_CONDITIONS)) {
+  for (let test of Object.keys(TESTS)) {
+    add_task(function*() {
+      do_print("Running test " + setup + " " + test);
+
+      yield exec_test(TEST_CONDITIONS[setup], TESTS[test]);
+    });
+  }
+}
+
+// Some custom tests
+
+// Test that the update check is performed as part of the regular add-on update
+// check
+add_task(function* test_addon_update() {
+  yield setup_conditions(TEST_CONDITIONS.blank);
+
+  yield update_all_addons(yield build_xml([
+    { id: "system2@tests.mozilla.org", version: "2.0", path: "system2_2.xpi" },
+    { id: "system3@tests.mozilla.org", version: "2.0", path: "system3_2.xpi" }
+  ]));
+
+  yield verify_state([true, null, "2.0", "2.0", null, null]);
+
+  yield promiseShutdownManager();
+});
+
+// Disabling app updates should block system add-on updates
+add_task(function* test_app_update_disabled() {
+  yield setup_conditions(TEST_CONDITIONS.blank);
+
+  Services.prefs.setBoolPref(PREF_APP_UPDATE_ENABLED, false);
+  yield update_all_addons(yield build_xml([
+    { id: "system2@tests.mozilla.org", version: "2.0", path: "system2_2.xpi" },
+    { id: "system3@tests.mozilla.org", version: "2.0", path: "system3_2.xpi" }
+  ]));
+  Services.prefs.clearUserPref(PREF_APP_UPDATE_ENABLED);
+
+  yield verify_state(TEST_CONDITIONS.blank.initialState);
+
+  yield promiseShutdownManager();
+});
+
+// Tests that a set that matches the hidden default set works
+add_task(function* test_match_default() {
+  yield setup_conditions(TEST_CONDITIONS.withBothSets);
+
+  yield install_system_addons(yield build_xml([
+    { id: "system1@tests.mozilla.org", version: "1.0", path: "system1_1.xpi" },
+    { id: "system2@tests.mozilla.org", version: "1.0", path: "system2_1.xpi" }
+  ]));
+
+  // Bug 1204159: This should revert to the default set instead of installing
+  // new versions into the updated set.
+  //yield verify_state([false, "1.0", "1.0", null, null, null]);
+  yield verify_state([true, "1.0", "1.0", null, null, null]);
+
+  yield promiseShutdownManager();
+});
+
+// Tests that a set that matches the current set works
+add_task(function* test_match_current() {
+  yield setup_conditions(TEST_CONDITIONS.withBothSets);
+
+  yield install_system_addons(yield build_xml([
+    { id: "system2@tests.mozilla.org", version: "2.0", path: "system2_2.xpi" },
+    { id: "system3@tests.mozilla.org", version: "2.0", path: "system3_2.xpi" }
+  ]));
+
+  // Bug 1204159: This should remain with the current set instead of creating
+  // a new copy
+  //let set = JSON.parse(Services.prefs.getCharPref(PREF_SYSTEM_ADDON_SET));
+  //do_check_eq(set.directory, "prefilled");
+
+  yield verify_state([true, null, "2.0", "2.0", null, null]);
+
+  yield promiseShutdownManager();
+});

toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini

@@ -25,6 +25,7 @@ skip-if = appname != "firefox"
 [test_provider_unsafe_access_startup.js]
 [test_ProductAddonChecker.js]
 [test_shutdown.js]
+[test_system_update.js]
 [test_system_reset.js]
 [test_XPIcancel.js]
 [test_XPIStates.js]