Bug 1783248 - [mochitest] Add notification of changed pref at end of test and error if changed in test-verify. r=ahal

Differential Revision: https://phabricator.services.mozilla.com/D169721

testing/mochitest/ignorePrefs.json

@@ -0,0 +1,14 @@
+[
+    "app.update.lastUpdateTime.search-engine-update-timer",
+    "media.gmp-manager.buildID",
+    "app.update.lastUpdateTime.browser-cleanup-thumbnails",
+    "app.update.lastUpdateTime.services-settings-poll-changes",
+    "app.update.lastUpdateTime.addon-background-update-timer",
+    "media.gmp-manager.lastEmptyCheck",
+    "app.update.lastUpdateTime.background-update-timer",
+    "media.gmp-manager.lastCheck",
+    "toolkit.startup.last_success",
+    "toolkit.telemetry.cachedClientID",
+    "security.webauth.softtoken_counter",
+    "extensions.webextensions.ExtensionStorageIDB.migrated.*"
+  ]

testing/mochitest/mochitest_options.py

@@ -919,6 +919,15 @@ class MochitestArguments(ArgumentContainer):
                 "help": "treat harness level crashes as passing (used for quarantine jobs).",
             },
         ],
+        [
+            ["--compare-preferences"],
+            {
+                "action": "store_true",
+                "dest": "comparePrefs",
+                "default": False,
+                "help": "Compare preferences at the end of each test and report changed ones as failures.",
+            },
+        ],
     ]
 
     defaults = {

testing/mochitest/moz.build

@@ -27,6 +27,7 @@ FINAL_TARGET_FILES.content += [
     "chrome-harness.js",
     "chunkifyTests.js",
     "harness.xhtml",
+    "ignorePrefs.json",
     "manifestLibrary.js",
     "mochitest-e10s-utils.js",
     "redirect.html",
@@ -97,6 +98,7 @@ TEST_HARNESS_FILES.testing.mochitest += [
     "DoHServer/doh_server.js",
     "favicon.ico",
     "harness.xhtml",
+    "ignorePrefs.json",
     "leaks.py",
     "mach_test_package_commands.py",
     "manifest.webapp",

testing/mochitest/runtests.py

@@ -1159,6 +1159,9 @@ class MochitestDesktop(object):
                 options.xOriginTests = True
             if options.xOriginTests:
                 self.urlOpts.append("xOriginTests=true")
+            if options.comparePrefs:
+                self.urlOpts.append("comparePrefs=true")
+            self.urlOpts.append("ignorePrefsFile=ignorePrefs.json")
 
     def normflavor(self, flavor):
         """
@@ -1952,6 +1955,12 @@ toolbar#nav-bar {
             d["runFailures"] = True
         content = json.dumps(d)
 
+        shutil.copy(
+            os.path.join(SCRIPT_DIR, "ignorePrefs.json"),
+            os.path.join(options.profilePath, "ignorePrefs.json"),
+        )
+        d["ignorePrefsFile"] = "ignorePrefs.json"
+
         with open(os.path.join(options.profilePath, "testConfig.js"), "w") as config:
             config.write(content)
 
@@ -3147,6 +3156,7 @@ toolbar#nav-bar {
             options.keep_open = False
             options.runUntilFailure = True
             options.profilePath = None
+            options.comparePrefs = True
             result = self.runTests(options)
             result = result or (-2 if self.countfail > 0 else 0)
             self.message_logger.finish()

testing/mochitest/tests/SimpleTest/TestRunner.js

@@ -128,6 +128,7 @@ TestRunner.interactiveDebugger = false;
 TestRunner.cleanupCrashes = false;
 TestRunner.timeoutAsPass = false;
 TestRunner.conditionedProfile = false;
+TestRunner.comparePrefs = false;
 
 TestRunner._expectingProcessCrash = false;
 TestRunner._structuredFormatter = new StructuredFormatter();
@@ -915,6 +916,31 @@ TestRunner.testUnloaded = function() {
     );
   }
 
+  // Always do this, so we can "reset" preferences between tests
+  SpecialPowers.comparePrefsToBaseline(
+    TestRunner.ignorePrefs,
+    TestRunner.verifyPrefsNextTest
+  );
+};
+
+TestRunner.verifyPrefsNextTest = function(p) {
+  if (TestRunner.comparePrefs) {
+    let prefs = Array.from(SpecialPowers.Cu.waiveXrays(p), x =>
+      SpecialPowers.unwrapIfWrapped(SpecialPowers.Cu.unwaiveXrays(x))
+    );
+    prefs.forEach(pr =>
+      TestRunner.structuredLogger.error(
+        "TEST-UNEXPECTED-FAIL | " +
+          TestRunner.currentTestURL +
+          " | changed preference: " +
+          pr
+      )
+    );
+  }
+  TestRunner.doNextTest();
+};
+
+TestRunner.doNextTest = function() {
   TestRunner._currentTest++;
   if (TestRunner.runSlower) {
     setTimeout(TestRunner.runNextTest, 1000);

testing/mochitest/tests/SimpleTest/setup.js

@@ -68,6 +68,41 @@ function parseQueryString(encodedString, useArrays) {
   return o;
 }
 
+/* helper function, specifically for prefs to ignore */
+function loadFile(url, callback) {
+  let req = new XMLHttpRequest();
+  req.open("GET", url);
+  req.onload = function() {
+    if (req.readyState == 4) {
+      if (req.status == 200) {
+        try {
+          let prefs = JSON.parse(req.responseText);
+          callback(prefs);
+        } catch (e) {
+          dump(
+            "TEST-UNEXPECTED-FAIL: setup.js | error parsing " +
+              url +
+              " (" +
+              e +
+              ")\n"
+          );
+          throw e;
+        }
+      } else {
+        dump(
+          "TEST-UNEXPECTED-FAIL: setup.js | error loading " +
+            url +
+            " (HTTP " +
+            req.status +
+            ")\n"
+        );
+        callback({});
+      }
+    }
+  };
+  req.send();
+}
+
 // Check the query string for arguments
 var params = parseQueryString(location.search.substring(1), true);
 
@@ -193,6 +228,10 @@ if (params.conditionedProfile) {
   TestRunner.conditionedProfile = true;
 }
 
+if (params.comparePrefs) {
+  TestRunner.comparePrefs = true;
+}
+
 // Log things to the console if appropriate.
 TestRunner.logger.addListener("dumpListener", consoleLevel + "", function(msg) {
   dump(msg.info.join(" ") + "\n");
@@ -200,6 +239,7 @@ TestRunner.logger.addListener("dumpListener", consoleLevel + "", function(msg) {
 
 var gTestList = [];
 var RunSet = {};
+
 RunSet.runall = function(e) {
   // Filter tests to include|exclude tests based on data in params.filter.
   // This allows for including or excluding tests from the gTestList
@@ -295,6 +335,17 @@ function hookup() {
   }
 }
 
+function getPrefList() {
+  if (params.ignorePrefsFile) {
+    loadFile(getTestManifestURL(params.ignorePrefsFile), function(prefs) {
+      TestRunner.ignorePrefs = prefs;
+      RunSet.runall();
+    });
+  } else {
+    RunSet.runall();
+  }
+}
+
 function hookupTests(testList) {
   if (testList.length) {
     gTestList = testList;
@@ -309,7 +360,7 @@ function hookupTests(testList) {
   document.getElementById("toggleNonTests").onclick = toggleNonTests;
   // run automatically if autorun specified
   if (params.autorun) {
-    RunSet.runall();
+    getPrefList();
   }
 }
 

testing/specialpowers/content/SpecialPowersChild.sys.mjs

@@ -378,6 +378,12 @@ export class SpecialPowersChild extends JSWindowActorChild {
     return lazy.WrapPrivileged.isWrapper(val);
   }
 
+  unwrapIfWrapped(obj) {
+    return lazy.WrapPrivileged.isWrapper(obj)
+      ? lazy.WrapPrivileged.unwrap(obj)
+      : obj;
+  }
+
   /*
    * Wrap objects on a specified global.
    */
@@ -878,6 +884,34 @@ export class SpecialPowersChild extends JSWindowActorChild {
     }
   }
 
+  /*
+    Collect a snapshot of all preferences in Firefox (i.e. about:prefs).
+    From this, store the results within specialpowers for later reference.
+  */
+  async getBaselinePrefs(callback = null) {
+    await this.sendQuery("getBaselinePrefs");
+    if (callback) {
+      await callback();
+    }
+  }
+
+  /*
+    This uses the stored prefs from getBaselinePrefs, collects a new snapshot
+    of preferences, then compares the new vs the baseline.  If there are differences
+    they are recorded and returned as an array of preferences, in addition
+    all the changed preferences are reset to the value found in the baseline.
+
+    ignorePrefs: array of strings which are preferences.  If they end in *,
+                 we do a partial match
+  */
+  async comparePrefsToBaseline(ignorePrefs, callback = null) {
+    let retVal = await this.sendQuery("comparePrefsToBaseline", ignorePrefs);
+    if (callback) {
+      callback(retVal);
+    }
+    return retVal;
+  }
+
   _promiseEarlyRefresh() {
     return new Promise(r => {
       // for mochitest-browser
@@ -1007,6 +1041,9 @@ export class SpecialPowersChild extends JSWindowActorChild {
   getComplexValue(prefName, iid) {
     return Services.prefs.getComplexValue(prefName, iid);
   }
+  getStringPref(...args) {
+    return Services.prefs.getStringPref(...args);
+  }
 
   getParentBoolPref(prefName, defaultValue) {
     return this._getParentPref(prefName, "BOOL", { defaultValue });
@@ -1017,6 +1054,9 @@ export class SpecialPowersChild extends JSWindowActorChild {
   getParentCharPref(prefName, defaultValue) {
     return this._getParentPref(prefName, "CHAR", { defaultValue });
   }
+  getParentStringPref(prefName, defaultValue) {
+    return this._getParentPref(prefName, "STRING", { defaultValue });
+  }
 
   // Mimic the set*Pref API
   setBoolPref(prefName, value) {
@@ -1031,6 +1071,9 @@ export class SpecialPowersChild extends JSWindowActorChild {
   setComplexValue(prefName, iid, value) {
     return this._setPref(prefName, "COMPLEX", value, iid);
   }
+  setStringPref(prefName, value) {
+    return this._setPref(prefName, "STRING", value);
+  }
 
   // Mimic the clearUserPref API
   clearUserPref(prefName) {
@@ -1065,6 +1108,8 @@ export class SpecialPowersChild extends JSWindowActorChild {
         return Services.prefs.getIntPref(prefName);
       case "CHAR":
         return Services.prefs.getCharPref(prefName);
+      case "STRING":
+        return Services.prefs.getStringPref(prefName);
     }
     return undefined;
   }

testing/specialpowers/content/SpecialPowersParent.sys.mjs

@@ -30,10 +30,10 @@ const PREF_TYPES = {
   [Ci.nsIPrefBranch.PREF_INVALID]: "INVALID",
   [Ci.nsIPrefBranch.PREF_INT]: "INT",
   [Ci.nsIPrefBranch.PREF_BOOL]: "BOOL",
-  [Ci.nsIPrefBranch.PREF_STRING]: "CHAR",
+  [Ci.nsIPrefBranch.PREF_STRING]: "STRING",
   number: "INT",
   boolean: "BOOL",
-  string: "CHAR",
+  string: "STRING",
 };
 
 // We share a single preference environment stack between all
@@ -175,6 +175,7 @@ export class SpecialPowersParent extends JSWindowActorParent {
       },
     };
 
+    this._basePrefs = null;
     this.init();
 
     this._crashDumpDir = null;
@@ -209,6 +210,8 @@ export class SpecialPowersParent extends JSWindowActorParent {
       },
     };
     swm.addListener(this._serviceWorkerListener);
+
+    this.getBaselinePrefs();
   }
 
   uninit() {
@@ -488,7 +491,7 @@ export class SpecialPowersParent extends JSWindowActorParent {
             type = PREF_TYPES[typeof value];
           }
           if (type === "INVALID") {
-            throw new Error("Unexpected preference type");
+            throw new Error("Unexpected preference type for " + name);
           }
 
           pendingActions.push({ action, type, name, value, iid });
@@ -539,8 +542,20 @@ export class SpecialPowersParent extends JSWindowActorParent {
         return Services.prefs.setCharPref(name, value);
       case "COMPLEX":
         return Services.prefs.setComplexValue(name, iid, value);
+      case "STRING":
+        return Services.prefs.setStringPref(name, value);
     }
-    throw new Error(`Unexpected preference type: ${type}`);
+    switch (typeof value) {
+      case "boolean":
+        return Services.prefs.setBoolPref(name, value);
+      case "number":
+        return Services.prefs.setIntPref(name, value);
+      case "string":
+        return Services.prefs.setStringPref(name, value);
+    }
+    throw new Error(
+      `Unexpected preference type: ${type} for ${name} with value ${value} and type ${typeof value}`
+    );
   }
 
   _getPref(name, type, defaultValue, iid) {
@@ -562,8 +577,103 @@ export class SpecialPowersParent extends JSWindowActorParent {
         return Services.prefs.getCharPref(name);
       case "COMPLEX":
         return Services.prefs.getComplexValue(name, iid);
+      case "STRING":
+        if (defaultValue !== undefined) {
+          return Services.prefs.getStringPref(name, defaultValue);
+        }
+        return Services.prefs.getStringPref(name);
     }
-    throw new Error(`Unexpected preference type: ${type}`);
+    throw new Error(
+      `Unexpected preference type: ${type} for preference ${name}`
+    );
+  }
+
+  getBaselinePrefs() {
+    this._basePrefs = this._getAllPreferences();
+  }
+
+  _comparePrefs(base, target, ignorePrefs, partialMatches) {
+    let failures = [];
+    for (const [key, value] of base) {
+      if (ignorePrefs.includes(key)) {
+        continue;
+      }
+      let partialFind = false;
+      partialMatches.forEach(pm => {
+        if (key.startsWith(pm)) {
+          partialFind = true;
+        }
+      });
+      if (partialFind) {
+        continue;
+      }
+
+      if (value === target.get(key)) {
+        continue;
+      }
+      if (!failures.includes(key)) {
+        failures.push(key);
+      }
+    }
+    return failures;
+  }
+
+  comparePrefsToBaseline(ignorePrefs) {
+    let newPrefs = this._getAllPreferences();
+
+    // find all items in ignorePrefs that end in *, add to partialMatch
+    let partialMatch = [];
+    if (ignorePrefs === undefined) {
+      ignorePrefs = [];
+    }
+    ignorePrefs.forEach(pref => {
+      if (pref.endsWith("*")) {
+        partialMatch.push(pref.split("*")[0]);
+      }
+    });
+
+    // find all new prefs different than old
+    let rv1 = this._comparePrefs(
+      newPrefs,
+      this._basePrefs,
+      ignorePrefs,
+      partialMatch
+    );
+
+    // find all old prefs different than new (in case we delete)
+    let rv2 = this._comparePrefs(
+      this._basePrefs,
+      newPrefs,
+      ignorePrefs,
+      partialMatch
+    );
+
+    let failures = [...new Set([...rv1, ...rv2])];
+
+    // reset failures
+    failures.forEach(f => {
+      if (this._basePrefs.get(f)) {
+        this._setPref(
+          f,
+          PREF_TYPES[Services.prefs.getPrefType(f)],
+          this._basePrefs.get(f)
+        );
+      } else {
+        Services.prefs.clearUserPref(f);
+      }
+    });
+
+    return failures;
+  }
+
+  _getAllPreferences() {
+    let names = new Map();
+    for (let prefName of Services.prefs.getChildList("")) {
+      let prefType = PREF_TYPES[Services.prefs.getPrefType(prefName)];
+      let prefValue = this._getPref(prefName, prefType);
+      names.set(prefName, prefValue);
+    }
+    return names;
   }
 
   _toggleMuteAudio(aMuted) {
@@ -825,6 +935,12 @@ export class SpecialPowersParent extends JSWindowActorParent {
           this.browsingContext.top.sessionHistory.evictAllContentViewers();
           return undefined;
 
+        case "getBaselinePrefs":
+          return this.getBaselinePrefs();
+
+        case "comparePrefsToBaseline":
+          return this.comparePrefsToBaseline(aMessage.data);
+
         case "PushPrefEnv":
           return this.pushPrefEnv(aMessage.data);