Merge mozilla-central to mozilla-inbound

browser/app/nsBrowserApp.cpp

@@ -104,7 +104,7 @@ using namespace mozilla;
 #endif
 #define kDesktopFolder "browser"
 
-static void Output(const char *fmt, ... )
+static MOZ_FORMAT_PRINTF(1, 2) void Output(const char *fmt, ... )
 {
   va_list ap;
   va_start(ap, fmt);

browser/base/content/browser.js

@@ -5051,27 +5051,26 @@ var CombinedStopReload = {
 };
 
 var TabsProgressListener = {
-  // Keep track of which browsers we've started load timers for, since
-  // we won't see STATE_START events for pre-rendered tabs.
-  _startedLoadTimer: new WeakSet(),
-
   onStateChange(aBrowser, aWebProgress, aRequest, aStateFlags, aStatus) {
     // Collect telemetry data about tab load times.
     if (aWebProgress.isTopLevel && (!aRequest.originalURI || aRequest.originalURI.spec.scheme != "about")) {
+      let stopwatchRunning = TelemetryStopwatch.running("FX_PAGE_LOAD_MS", aBrowser);
+
       if (aStateFlags & Ci.nsIWebProgressListener.STATE_IS_WINDOW) {
         if (aStateFlags & Ci.nsIWebProgressListener.STATE_START) {
-          this._startedLoadTimer.add(aBrowser);
+          if (stopwatchRunning) {
+            // Oops, we're seeing another start without having noticed the previous stop.
+            TelemetryStopwatch.cancel("FX_PAGE_LOAD_MS", aBrowser);
+          }
           TelemetryStopwatch.start("FX_PAGE_LOAD_MS", aBrowser);
           Services.telemetry.getHistogramById("FX_TOTAL_TOP_VISITS").add(true);
         } else if (aStateFlags & Ci.nsIWebProgressListener.STATE_STOP &&
-                   this._startedLoadTimer.has(aBrowser)) {
-          this._startedLoadTimer.delete(aBrowser);
+                   stopwatchRunning /* we won't see STATE_START events for pre-rendered tabs */) {
           TelemetryStopwatch.finish("FX_PAGE_LOAD_MS", aBrowser);
         }
       } else if (aStateFlags & Ci.nsIWebProgressListener.STATE_STOP &&
                  aStatus == Cr.NS_BINDING_ABORTED &&
-                 this._startedLoadTimer.has(aBrowser)) {
-        this._startedLoadTimer.delete(aBrowser);
+                 stopwatchRunning /* we won't see STATE_START events for pre-rendered tabs */) {
         TelemetryStopwatch.cancel("FX_PAGE_LOAD_MS", aBrowser);
       }
     }

browser/base/content/test/static/browser_all_files_referenced.js

@@ -80,6 +80,9 @@ var whitelist = new Set([
   {file: "resource://app/modules/NewTabSearchProvider.jsm"},
   {file: "resource://app/modules/NewTabWebChannel.jsm"},
 
+  // Activity Stream currently needs this file in all channels except Nightly
+  {file: "resource://app/modules/PreviewProvider.jsm", skipNightly: true},
+
   // layout/mathml/nsMathMLChar.cpp
   {file: "resource://gre/res/fonts/mathfontSTIXGeneral.properties"},
   {file: "resource://gre/res/fonts/mathfontUnicode.properties"},
@@ -243,6 +246,7 @@ var whitelist = new Set([
 
 ].filter(item =>
   ("isFromDevTools" in item) == isDevtools &&
+  (!item.skipNightly || !AppConstants.NIGHTLY_BUILD) &&
   (!item.platforms || item.platforms.includes(AppConstants.platform))
 ).map(item => item.file));
 

browser/components/customizableui/CustomizableUI.jsm

@@ -234,6 +234,13 @@ var CustomizableUIInternal = {
     }, true);
     PanelWideWidgetTracker.init();
 
+    if (Services.prefs.getBoolPref("browser.photon.structure.enabled")) {
+      this.registerArea(CustomizableUI.AREA_FIXED_OVERFLOW_PANEL, {
+        type: CustomizableUI.TYPE_MENU_PANEL,
+        defaultPlacements: [],
+      }, true);
+    }
+
     let navbarPlacements = [
       "urlbar-container",
       "search-container",
@@ -897,9 +904,8 @@ var CustomizableUIInternal = {
     return [null, null];
   },
 
-  registerMenuPanel(aPanelContents) {
-    if (gBuildAreas.has(CustomizableUI.AREA_PANEL) &&
-        gBuildAreas.get(CustomizableUI.AREA_PANEL).has(aPanelContents)) {
+  registerMenuPanel(aPanelContents, aArea) {
+    if (gBuildAreas.has(aArea) && gBuildAreas.get(aArea).has(aPanelContents)) {
       return;
     }
 
@@ -910,9 +916,9 @@ var CustomizableUIInternal = {
 
     this.addPanelCloseListeners(this._getPanelForNode(aPanelContents));
 
-    let placements = gPlacements.get(CustomizableUI.AREA_PANEL);
-    this.buildArea(CustomizableUI.AREA_PANEL, placements, aPanelContents);
-    this.notifyListeners("onAreaNodeRegistered", CustomizableUI.AREA_PANEL, aPanelContents);
+    let placements = gPlacements.get(aArea);
+    this.buildArea(aArea, placements, aPanelContents);
+    this.notifyListeners("onAreaNodeRegistered", aArea, aPanelContents);
 
     for (let child of aPanelContents.children) {
       if (child.localName != "toolbarbutton") {
@@ -925,7 +931,7 @@ var CustomizableUIInternal = {
       child.setAttribute("wrap", "true");
     }
 
-    this.registerBuildArea(CustomizableUI.AREA_PANEL, aPanelContents);
+    this.registerBuildArea(aArea, aPanelContents);
   },
 
   onWidgetAdded(aWidgetId, aArea, aPosition) {
@@ -1489,8 +1495,9 @@ var CustomizableUIInternal = {
     } else if (aWidget.type == "view") {
       let ownerWindow = aNode.ownerGlobal;
       let area = this.getPlacementOfWidget(aNode.id).area;
+      let areaType = CustomizableUI.getAreaType(area);
       let anchor = aNode;
-      if (area != CustomizableUI.AREA_PANEL) {
+      if (areaType != CustomizableUI.TYPE_MENU_PANEL) {
         let wrapper = this.wrapWidget(aWidget.id).forWindow(ownerWindow);
 
         if (wrapper && !wrapper.overflowed && wrapper.anchor) {
@@ -2151,7 +2158,8 @@ var CustomizableUIInternal = {
       let addToDefaultPlacements = false;
       let area = gAreas.get(widget.defaultArea);
       if (!CustomizableUI.isBuiltinToolbar(widget.defaultArea) &&
-          widget.defaultArea != CustomizableUI.AREA_PANEL) {
+          widget.defaultArea != CustomizableUI.AREA_PANEL &&
+          widget.defaultArea != CustomizableUI.AREA_FIXED_OVERFLOW_PANEL) {
         addToDefaultPlacements = true;
       }
 
@@ -2824,6 +2832,11 @@ this.CustomizableUI = {
    * @deprecated
    */
   AREA_ADDONBAR: "addon-bar",
+  /**
+   * Constant reference to the ID of the non-dymanic (fixed) list in the overflow panel.
+   */
+  AREA_FIXED_OVERFLOW_PANEL: "widget-overflow-fixed-list",
+
   /**
    * Constant indicating the area is a menu panel.
    */
@@ -3045,10 +3058,11 @@ this.CustomizableUI = {
   /**
    * Register the menu panel node. This method should not be called by anyone
    * apart from the built-in PanelUI.
-   * @param aPanel the panel DOM node being registered.
+   * @param aPanelContents the panel contents DOM node being registered.
+   * @param aArea the area for which to register this node.
    */
-  registerMenuPanel(aPanel) {
-    CustomizableUIInternal.registerMenuPanel(aPanel);
+  registerMenuPanel(aPanelContents, aArea) {
+    CustomizableUIInternal.registerMenuPanel(aPanelContents, aArea);
   },
   /**
    * Unregister a customizable area. The inverse of registerArea.
@@ -4150,8 +4164,11 @@ OverflowableToolbar.prototype = {
     this._panel.removeEventListener("dragover", this);
     this._panel.removeEventListener("dragend", this);
     let doc = aEvent.target.ownerDocument;
-    let contextMenu = doc.getElementById(this._panel.getAttribute("context"));
-    gELS.removeSystemEventListener(contextMenu, "command", this, true);
+    let contextMenuId = this._panel.getAttribute("context");
+    if (contextMenuId) {
+      let contextMenu = doc.getElementById(contextMenuId);
+      gELS.removeSystemEventListener(contextMenu, "command", this, true);
+    }
   },
 
   onOverflow(aEvent) {

browser/components/customizableui/CustomizableWidgets.jsm

@@ -1006,7 +1006,6 @@ const CustomizableWidgets = [
       }
     },
     onCreated(aNode) {
-      const kPanelId = "PanelUI-popup";
       let document = aNode.ownerDocument;
 
       let updateButton = () => {
@@ -1016,27 +1015,32 @@ const CustomizableWidgets = [
           aNode.removeAttribute("disabled");
       };
 
-      if (this.currentArea == CustomizableUI.AREA_PANEL) {
-        let panel = document.getElementById(kPanelId);
-        panel.addEventListener("popupshowing", updateButton);
+      let getPanel = () => {
+        let {PanelUI} = document.ownerGlobal;
+        if (PanelUI.overflowContents) {
+          return document.getElementById("widget-overflow");
+        }
+        return PanelUI.panel;
+      }
+
+      if (CustomizableUI.getAreaType(this.currentArea) == CustomizableUI.TYPE_MENU_PANEL) {
+        getPanel().addEventListener("popupshowing", updateButton);
       }
 
       let listener = {
         onWidgetAdded: (aWidgetId, aArea) => {
           if (aWidgetId != this.id)
             return;
-          if (aArea == CustomizableUI.AREA_PANEL) {
-            let panel = document.getElementById(kPanelId);
-            panel.addEventListener("popupshowing", updateButton);
+          if (CustomizableUI.getAreaType(aArea) == CustomizableUI.TYPE_MENU_PANEL) {
+            getPanel().addEventListener("popupshowing", updateButton);
           }
         },
         onWidgetRemoved: (aWidgetId, aPrevArea) => {
           if (aWidgetId != this.id)
             return;
           aNode.removeAttribute("disabled");
-          if (aPrevArea == CustomizableUI.AREA_PANEL) {
-            let panel = document.getElementById(kPanelId);
-            panel.removeEventListener("popupshowing", updateButton);
+          if (CustomizableUI.getAreaType(aPrevArea) == CustomizableUI.TYPE_MENU_PANEL) {
+            getPanel().removeEventListener("popupshowing", updateButton);
           }
         },
         onWidgetInstanceRemoved: (aWidgetId, aDoc) => {
@@ -1044,8 +1048,7 @@ const CustomizableWidgets = [
             return;
 
           CustomizableUI.removeListener(listener);
-          let panel = aDoc.getElementById(kPanelId);
-          panel.removeEventListener("popupshowing", updateButton);
+          getPanel().removeEventListener("popupshowing", updateButton);
         }
       };
       CustomizableUI.addListener(listener);

browser/components/customizableui/DragPositionManager.jsm

@@ -379,7 +379,7 @@ AreaPositionManager.prototype = {
 
 var DragPositionManager = {
   start(aWindow) {
-    let areas = CustomizableUI.areas.filter((area) => CustomizableUI.getAreaType(area) != "toolbar");
+    let areas = [CustomizableUI.AREA_PANEL];
     areas = areas.map((area) => CustomizableUI.getCustomizeTargetForArea(area, aWindow));
     areas.push(aWindow.document.getElementById(kPaletteId));
     for (let areaNode of areas) {
@@ -393,7 +393,7 @@ var DragPositionManager = {
   },
 
   add(aWindow, aArea, aContainer) {
-    if (CustomizableUI.getAreaType(aArea) != "toolbar") {
+    if (aArea != CustomizableUI.AREA_PANEL) {
       return;
     }
 
@@ -401,7 +401,7 @@ var DragPositionManager = {
   },
 
   remove(aWindow, aArea, aContainer) {
-    if (CustomizableUI.getAreaType(aArea) != "toolbar") {
+    if (aArea != CustomizableUI.AREA_PANEL) {
       return;
     }
 

browser/components/customizableui/content/panelUI.inc.xul

@@ -383,6 +383,8 @@
     <panelview id="widget-overflow-mainView"
                context="toolbar-context-menu">
       <vbox id="widget-overflow-scroller">
+        <vbox id="widget-overflow-fixed-list" class="widget-overflow-list" hidden="true"/>
+        <toolbarseparator id="widget-overflow-fixed-separator" hidden="true"/>
         <vbox id="widget-overflow-list" class="widget-overflow-list"
               overflowfortoolbar="nav-bar"/>
       </vbox>

browser/components/customizableui/content/panelUI.js

@@ -37,13 +37,18 @@ const PanelUI = {
       panel: gPhotonStructure ? "appMenu-popup" : "PanelUI-popup",
       notificationPanel: "PanelUI-notification-popup",
       scroller: "PanelUI-contents-scroller",
-      footer: "PanelUI-footer"
+      footer: "PanelUI-footer",
+
+      overflowFixedList: gPhotonStructure ? "widget-overflow-fixed-list" : "",
     };
   },
 
   _initialized: false,
   init() {
     for (let [k, v] of Object.entries(this.kElements)) {
+      if (!v) {
+        continue;
+      }
       // Need to do fresh let-bindings per iteration
       let getKey = k;
       let id = v;
@@ -70,6 +75,12 @@ const PanelUI = {
       this.notificationPanel.addEventListener(event, this);
     }
 
+    if (gPhotonStructure) {
+      this.overflowFixedList.hidden = false;
+      this.overflowFixedList.nextSibling.hidden = false;
+      CustomizableUI.registerMenuPanel(this.overflowFixedList, CustomizableUI.AREA_FIXED_OVERFLOW_PANEL);
+    }
+
     this._initialized = true;
   },
 
@@ -385,11 +396,11 @@ const PanelUI = {
       }
 
       if (aCustomizing) {
-        CustomizableUI.registerMenuPanel(this.contents);
+        CustomizableUI.registerMenuPanel(this.contents, CustomizableUI.AREA_PANEL);
       } else {
         this.beginBatchUpdate();
         try {
-          CustomizableUI.registerMenuPanel(this.contents);
+          CustomizableUI.registerMenuPanel(this.contents, CustomizableUI.AREA_PANEL);
         } finally {
           this.endBatchUpdate();
         }

browser/components/resistfingerprinting/test/browser/browser.ini

@@ -4,6 +4,7 @@ support-files =
   file_dummy.html
   head.js
 
+[browser_roundedWindow_dialogWindow.js]
 [browser_roundedWindow_newWindow.js]
 [browser_roundedWindow_open_max.js]
 [browser_roundedWindow_open_mid.js]

browser/components/resistfingerprinting/test/browser/browser_roundedWindow_dialogWindow.js

@@ -0,0 +1,40 @@
+/**
+ * Bug 1352305 - A test case for dialog windows that it should not be rounded
+ *   even after fingerprinting resistance is enabled.
+ */
+
+async function test_dialog_window() {
+  let diagWin;
+
+  await new Promise(resolve => {
+    // Open a dialog window which is not rounded size.
+    diagWin = window.openDialog("about:blank", null,
+                                "innerWidth=250,innerHeight=350");
+
+    diagWin.addEventListener("load", function() {
+      resolve();
+    }, {once: true});
+  });
+
+  is(diagWin.innerWidth, 250, "The dialog window doesn't have a rounded size.");
+  is(diagWin.innerHeight, 350, "The dialog window doesn't have a rounded size.");
+
+  await BrowserTestUtils.closeWindow(diagWin);
+}
+
+add_task(async function setup() {
+  await SpecialPowers.pushPrefEnv({"set":
+    [["privacy.resistFingerprinting", true]]
+  });
+});
+
+add_task(test_dialog_window);
+
+add_task(async function test_dialog_window_without_resistFingerprinting() {
+  // Test dialog windows with 'privacy.resistFingerprinting' is false.
+  await SpecialPowers.pushPrefEnv({"set":
+    [["privacy.resistFingerprinting", false]]
+  });
+
+  await test_dialog_window();
+});

browser/components/sessionstore/test/browser.ini

@@ -183,7 +183,12 @@ skip-if = true # Needs to be rewritten as Marionette test, bug 995916
 [browser_600545.js]
 [browser_601955.js]
 [browser_607016.js]
-[browser_615394-SSWindowState_events.js]
+[browser_615394-SSWindowState_events_duplicateTab.js]
+[browser_615394-SSWindowState_events_setBrowserState.js]
+[browser_615394-SSWindowState_events_setTabState.js]
+[browser_615394-SSWindowState_events_setWindowState.js]
+[browser_615394-SSWindowState_events_undoCloseTab.js]
+[browser_615394-SSWindowState_events_undoCloseWindow.js]
 [browser_618151.js]
 [browser_623779.js]
 [browser_624727.js]

browser/components/sessionstore/test/browser_615394-SSWindowState_events.js

@@ -1,363 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-const stateBackup = JSON.parse(ss.getBrowserState());
-const testState = {
-  windows: [{
-    tabs: [
-      { entries: [{ url: "about:blank", triggeringPrincipal_base64 }] },
-      { entries: [{ url: "about:rights", triggeringPrincipal_base64 }] }
-    ]
-  }]
-};
-const lameMultiWindowState = { windows: [
-    {
-      tabs: [
-        { entries: [{ url: "http://example.org#1", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
-        { entries: [{ url: "http://example.org#2", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
-        { entries: [{ url: "http://example.org#3", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
-        { entries: [{ url: "http://example.org#4", triggeringPrincipal_base64 }], extData: { "uniq": r() } }
-      ],
-      selected: 1
-    },
-    {
-      tabs: [
-        { entries: [{ url: "http://example.com#1", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
-        { entries: [{ url: "http://example.com#2", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
-        { entries: [{ url: "http://example.com#3", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
-        { entries: [{ url: "http://example.com#4", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
-      ],
-      selected: 3
-    }
-  ] };
-
-
-function getOuterWindowID(aWindow) {
-  return aWindow.QueryInterface(Ci.nsIInterfaceRequestor).
-         getInterface(Ci.nsIDOMWindowUtils).outerWindowID;
-}
-
-function test() {
-  /** Test for Bug 615394 - Session Restore should notify when it is beginning and ending a restore **/
-  waitForExplicitFinish();
-  // Preemptively extend the timeout to prevent [orange]
-  requestLongerTimeout(4);
-  runNextTest();
-}
-
-
-var tests = [
-  test_setTabState,
-  test_duplicateTab,
-  test_undoCloseTab,
-  test_setWindowState,
-  test_setBrowserState,
-  test_undoCloseWindow
-];
-function runNextTest() {
-  // set an empty state & run the next test, or finish
-  if (tests.length) {
-    // Enumerate windows and close everything but our primary window. We can't
-    // use waitForFocus() because apparently it's buggy. See bug 599253.
-    var windowsEnum = Services.wm.getEnumerator("navigator:browser");
-    let closeWinPromises = [];
-    while (windowsEnum.hasMoreElements()) {
-      var currentWindow = windowsEnum.getNext();
-      if (currentWindow != window) {
-        closeWinPromises.push(BrowserTestUtils.closeWindow(currentWindow));
-      }
-    }
-
-    Promise.all(closeWinPromises).then(() => {
-      let currentTest = tests.shift();
-      info("prepping for " + currentTest.name);
-      waitForBrowserState(testState, currentTest);
-    });
-  } else {
-    waitForBrowserState(stateBackup, finish);
-  }
-}
-
-/**  ACTUAL TESTS  **/
-
-function test_setTabState() {
-  let tab = gBrowser.tabs[1];
-  let newTabState = JSON.stringify({ entries: [{ url: "http://example.org", triggeringPrincipal_base64 }], extData: { foo: "bar" } });
-  let busyEventCount = 0;
-  let readyEventCount = 0;
-
-  function onSSWindowStateBusy(aEvent) {
-    busyEventCount++;
-  }
-
-  function onSSWindowStateReady(aEvent) {
-    readyEventCount++;
-    is(ss.getTabValue(tab, "foo"), "bar");
-    ss.setTabValue(tab, "baz", "qux");
-  }
-
-  function onSSTabRestoring(aEvent) {
-    if (aEvent.target == tab) {
-      is(busyEventCount, 1);
-      is(readyEventCount, 1);
-      is(ss.getTabValue(tab, "baz"), "qux");
-      is(tab.linkedBrowser.currentURI.spec, "http://example.org/");
-
-      window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-      window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
-      gBrowser.tabContainer.removeEventListener("SSTabRestoring", onSSTabRestoring);
-
-      runNextTest();
-    }
-  }
-
-  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
-  gBrowser.tabContainer.addEventListener("SSTabRestoring", onSSTabRestoring);
-  // Browser must be inserted in order to restore.
-  gBrowser._insertBrowser(tab);
-  ss.setTabState(tab, newTabState);
-}
-
-
-function test_duplicateTab() {
-  let tab = gBrowser.tabs[1];
-  let busyEventCount = 0;
-  let readyEventCount = 0;
-  let newTab;
-
-  // We'll look to make sure this value is on the duplicated tab
-  ss.setTabValue(tab, "foo", "bar");
-
-  function onSSWindowStateBusy(aEvent) {
-    busyEventCount++;
-  }
-
-  function onSSWindowStateReady(aEvent) {
-    newTab = gBrowser.tabs[2];
-    readyEventCount++;
-    is(ss.getTabValue(newTab, "foo"), "bar");
-    ss.setTabValue(newTab, "baz", "qux");
-  }
-
-  function onSSTabRestoring(aEvent) {
-    if (aEvent.target == newTab) {
-      is(busyEventCount, 1);
-      is(readyEventCount, 1);
-      is(ss.getTabValue(newTab, "baz"), "qux");
-      is(newTab.linkedBrowser.currentURI.spec, "about:rights");
-
-      window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-      window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
-      gBrowser.tabContainer.removeEventListener("SSTabRestoring", onSSTabRestoring);
-
-      runNextTest();
-    }
-  }
-
-  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
-  gBrowser.tabContainer.addEventListener("SSTabRestoring", onSSTabRestoring);
-
-  gBrowser._insertBrowser(tab);
-  newTab = ss.duplicateTab(window, tab);
-}
-
-
-function test_undoCloseTab() {
-  let tab = gBrowser.tabs[1],
-      busyEventCount = 0,
-      readyEventCount = 0,
-      reopenedTab;
-
-  ss.setTabValue(tab, "foo", "bar");
-
-  function onSSWindowStateBusy(aEvent) {
-    busyEventCount++;
-  }
-
-  function onSSWindowStateReady(aEvent) {
-    reopenedTab = gBrowser.tabs[1];
-    readyEventCount++;
-    is(ss.getTabValue(reopenedTab, "foo"), "bar");
-    ss.setTabValue(reopenedTab, "baz", "qux");
-  }
-
-  function onSSTabRestored(aEvent) {
-    is(busyEventCount, 1);
-    is(readyEventCount, 1);
-    is(ss.getTabValue(reopenedTab, "baz"), "qux");
-    is(reopenedTab.linkedBrowser.currentURI.spec, "about:rights");
-
-    window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-    window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
-    gBrowser.tabContainer.removeEventListener("SSTabRestored", onSSTabRestored);
-
-    runNextTest();
-  }
-
-  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
-  gBrowser.tabContainer.addEventListener("SSTabRestored", onSSTabRestored);
-
-  gBrowser.removeTab(tab);
-  reopenedTab = ss.undoCloseTab(window, 0);
-}
-
-
-function test_setWindowState() {
-  let newState = {
-    windows: [{
-      tabs: [
-        { entries: [{ url: "about:mozilla", triggeringPrincipal_base64 }], extData: { "foo": "bar" } },
-        { entries: [{ url: "http://example.org", triggeringPrincipal_base64 }], extData: { "baz": "qux" } }
-      ]
-    }]
-  };
-
-  let busyEventCount = 0,
-      readyEventCount = 0,
-      tabRestoredCount = 0;
-
-  function onSSWindowStateBusy(aEvent) {
-    busyEventCount++;
-  }
-
-  function onSSWindowStateReady(aEvent) {
-    readyEventCount++;
-    is(ss.getTabValue(gBrowser.tabs[0], "foo"), "bar");
-    is(ss.getTabValue(gBrowser.tabs[1], "baz"), "qux");
-  }
-
-  function onSSTabRestored(aEvent) {
-    if (++tabRestoredCount < 2)
-      return;
-
-    is(busyEventCount, 1);
-    is(readyEventCount, 1);
-    is(gBrowser.tabs[0].linkedBrowser.currentURI.spec, "about:mozilla");
-    is(gBrowser.tabs[1].linkedBrowser.currentURI.spec, "http://example.org/");
-
-    window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-    window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
-    gBrowser.tabContainer.removeEventListener("SSTabRestored", onSSTabRestored);
-
-    runNextTest();
-  }
-
-  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
-  gBrowser.tabContainer.addEventListener("SSTabRestored", onSSTabRestored);
-
-  ss.setWindowState(window, JSON.stringify(newState), true);
-}
-
-
-function test_setBrowserState() {
-  // We'll track events per window so we are sure that they are each happening once
-  // pre window.
-  let windowEvents = {};
-  windowEvents[getOuterWindowID(window)] = { busyEventCount: 0, readyEventCount: 0 };
-
-  // waitForBrowserState does it's own observing for windows, but doesn't attach
-  // the listeners we want here, so do it ourselves.
-  let newWindow;
-  function windowObserver(aSubject, aTopic, aData) {
-    if (aTopic == "domwindowopened") {
-      newWindow = aSubject.QueryInterface(Ci.nsIDOMWindow);
-      newWindow.addEventListener("load", function() {
-        Services.ww.unregisterNotification(windowObserver);
-
-        windowEvents[getOuterWindowID(newWindow)] = { busyEventCount: 0, readyEventCount: 0 };
-
-        newWindow.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-        newWindow.addEventListener("SSWindowStateReady", onSSWindowStateReady);
-      }, {once: true});
-    }
-  }
-
-  function onSSWindowStateBusy(aEvent) {
-    windowEvents[getOuterWindowID(aEvent.originalTarget)].busyEventCount++;
-  }
-
-  function onSSWindowStateReady(aEvent) {
-    windowEvents[getOuterWindowID(aEvent.originalTarget)].readyEventCount++;
-  }
-
-  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
-  Services.ww.registerNotification(windowObserver);
-
-  waitForBrowserState(lameMultiWindowState, function() {
-    let checkedWindows = 0;
-    for (let id of Object.keys(windowEvents)) {
-      let winEvents = windowEvents[id];
-      is(winEvents.busyEventCount, 1,
-         "[test_setBrowserState] window" + id + " busy event count correct");
-      is(winEvents.readyEventCount, 1,
-         "[test_setBrowserState] window" + id + " ready event count correct");
-      checkedWindows++;
-    }
-    is(checkedWindows, 2,
-       "[test_setBrowserState] checked 2 windows");
-    window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-    window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
-    newWindow.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-    newWindow.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
-    runNextTest();
-  });
-}
-
-
-function test_undoCloseWindow() {
-  let newWindow, reopenedWindow;
-
-  function firstWindowObserver(aSubject, aTopic, aData) {
-    if (aTopic == "domwindowopened") {
-      newWindow = aSubject.QueryInterface(Ci.nsIDOMWindow);
-      Services.ww.unregisterNotification(firstWindowObserver);
-    }
-  }
-  Services.ww.registerNotification(firstWindowObserver);
-
-  waitForBrowserState(lameMultiWindowState, function() {
-    // Close the window which isn't window
-    BrowserTestUtils.closeWindow(newWindow).then(() => {
-      // Now give it time to close
-      reopenedWindow = ss.undoCloseWindow(0);
-      reopenedWindow.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-      reopenedWindow.addEventListener("SSWindowStateReady", onSSWindowStateReady);
-
-      reopenedWindow.addEventListener("load", function() {
-        reopenedWindow.gBrowser.tabContainer.addEventListener("SSTabRestored", onSSTabRestored);
-      }, {once: true});
-    });
-  });
-
-  let busyEventCount = 0,
-      readyEventCount = 0,
-      tabRestoredCount = 0;
-  // These will listen to the reopened closed window...
-  function onSSWindowStateBusy(aEvent) {
-    busyEventCount++;
-  }
-
-  function onSSWindowStateReady(aEvent) {
-    readyEventCount++;
-  }
-
-  function onSSTabRestored(aEvent) {
-    if (++tabRestoredCount < 4)
-      return;
-
-    is(busyEventCount, 1);
-    is(readyEventCount, 1);
-
-    reopenedWindow.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
-    reopenedWindow.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
-    reopenedWindow.gBrowser.tabContainer.removeEventListener("SSTabRestored", onSSTabRestored);
-
-    BrowserTestUtils.closeWindow(reopenedWindow).then(runNextTest);
-  }
-}

browser/components/sessionstore/test/browser_615394-SSWindowState_events_duplicateTab.js

@@ -0,0 +1,65 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const testState = {
+  windows: [{
+    tabs: [
+      { entries: [{ url: "about:blank", triggeringPrincipal_base64 }] },
+      { entries: [{ url: "about:rights", triggeringPrincipal_base64 }] }
+    ]
+  }]
+};
+
+function test() {
+  /** Test for Bug 615394 - Session Restore should notify when it is beginning and ending a restore **/
+  waitForExplicitFinish();
+
+  waitForBrowserState(testState, test_duplicateTab);
+}
+
+function test_duplicateTab() {
+  let tab = gBrowser.tabs[1];
+  let busyEventCount = 0;
+  let readyEventCount = 0;
+  let newTab;
+
+  // We'll look to make sure this value is on the duplicated tab
+  ss.setTabValue(tab, "foo", "bar");
+
+  function onSSWindowStateBusy(aEvent) {
+    busyEventCount++;
+  }
+
+  function onSSWindowStateReady(aEvent) {
+    newTab = gBrowser.tabs[2];
+    readyEventCount++;
+    is(ss.getTabValue(newTab, "foo"), "bar");
+    ss.setTabValue(newTab, "baz", "qux");
+  }
+
+  function onSSTabRestoring(aEvent) {
+    if (aEvent.target == newTab) {
+      is(busyEventCount, 1);
+      is(readyEventCount, 1);
+      is(ss.getTabValue(newTab, "baz"), "qux");
+      is(newTab.linkedBrowser.currentURI.spec, "about:rights");
+
+      window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+      window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
+      gBrowser.tabContainer.removeEventListener("SSTabRestoring", onSSTabRestoring);
+
+      gBrowser.removeTab(tab);
+      gBrowser.removeTab(newTab);
+      finish();
+    }
+  }
+
+  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
+  gBrowser.tabContainer.addEventListener("SSTabRestoring", onSSTabRestoring);
+
+  gBrowser._insertBrowser(tab);
+  newTab = ss.duplicateTab(window, tab);
+}
+

browser/components/sessionstore/test/browser_615394-SSWindowState_events_setBrowserState.js

@@ -0,0 +1,92 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const lameMultiWindowState = { windows: [
+    {
+      tabs: [
+        { entries: [{ url: "http://example.org#1", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.org#2", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.org#3", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.org#4", triggeringPrincipal_base64 }], extData: { "uniq": r() } }
+      ],
+      selected: 1
+    },
+    {
+      tabs: [
+        { entries: [{ url: "http://example.com#1", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.com#2", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.com#3", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.com#4", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+      ],
+      selected: 3
+    }
+  ] };
+
+
+function getOuterWindowID(aWindow) {
+  return aWindow.QueryInterface(Ci.nsIInterfaceRequestor).
+         getInterface(Ci.nsIDOMWindowUtils).outerWindowID;
+}
+
+function test() {
+  /** Test for Bug 615394 - Session Restore should notify when it is beginning and ending a restore **/
+  waitForExplicitFinish();
+
+  // We'll track events per window so we are sure that they are each happening once
+  // pre window.
+  let windowEvents = {};
+  windowEvents[getOuterWindowID(window)] = { busyEventCount: 0, readyEventCount: 0 };
+
+  // waitForBrowserState does it's own observing for windows, but doesn't attach
+  // the listeners we want here, so do it ourselves.
+  let newWindow;
+  function windowObserver(aSubject, aTopic, aData) {
+    if (aTopic == "domwindowopened") {
+      Services.ww.unregisterNotification(windowObserver);
+
+      newWindow = aSubject.QueryInterface(Ci.nsIDOMWindow);
+      newWindow.addEventListener("load", function() {
+        windowEvents[getOuterWindowID(newWindow)] = { busyEventCount: 0, readyEventCount: 0 };
+
+        newWindow.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+        newWindow.addEventListener("SSWindowStateReady", onSSWindowStateReady);
+      }, {once: true});
+    }
+  }
+
+  function onSSWindowStateBusy(aEvent) {
+    windowEvents[getOuterWindowID(aEvent.originalTarget)].busyEventCount++;
+  }
+
+  function onSSWindowStateReady(aEvent) {
+    windowEvents[getOuterWindowID(aEvent.originalTarget)].readyEventCount++;
+  }
+
+  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
+  Services.ww.registerNotification(windowObserver);
+
+  waitForBrowserState(lameMultiWindowState, function() {
+    let checkedWindows = 0;
+    for (let id of Object.keys(windowEvents)) {
+      let winEvents = windowEvents[id];
+      is(winEvents.busyEventCount, 1, "window" + id + " busy event count correct");
+      is(winEvents.readyEventCount, 1, "window" + id + " ready event count correct");
+      checkedWindows++;
+    }
+    is(checkedWindows, 2, "checked 2 windows");
+    window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+    window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
+    newWindow.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+    newWindow.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
+
+    newWindow.close();
+    while (gBrowser.tabs.length > 1) {
+      gBrowser.removeTab(gBrowser.tabs[1]);
+    }
+
+    finish();
+  });
+}
+

browser/components/sessionstore/test/browser_615394-SSWindowState_events_setTabState.js

@@ -0,0 +1,57 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const testState = {
+  windows: [{
+    tabs: [
+      { entries: [{ url: "about:blank", triggeringPrincipal_base64 }] },
+      { entries: [{ url: "about:rights", triggeringPrincipal_base64 }] }
+    ]
+  }]
+};
+
+function test() {
+  /** Test for Bug 615394 - Session Restore should notify when it is beginning and ending a restore **/
+  waitForExplicitFinish();
+
+  waitForBrowserState(testState, test_setTabState);
+}
+
+function test_setTabState() {
+  let tab = gBrowser.tabs[1];
+  let newTabState = JSON.stringify({ entries: [{ url: "http://example.org", triggeringPrincipal_base64 }], extData: { foo: "bar" } });
+  let busyEventCount = 0;
+  let readyEventCount = 0;
+
+  function onSSWindowStateBusy(aEvent) {
+    busyEventCount++;
+  }
+
+  function onSSWindowStateReady(aEvent) {
+    readyEventCount++;
+    is(ss.getTabValue(tab, "foo"), "bar");
+    ss.setTabValue(tab, "baz", "qux");
+  }
+
+  function onSSTabRestoring(aEvent) {
+    is(busyEventCount, 1);
+    is(readyEventCount, 1);
+    is(ss.getTabValue(tab, "baz"), "qux");
+    is(tab.linkedBrowser.currentURI.spec, "http://example.org/");
+
+    window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+    window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
+
+    gBrowser.removeTab(tab)
+    finish();
+  }
+
+  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
+  tab.addEventListener("SSTabRestoring", onSSTabRestoring, { once: true });
+  // Browser must be inserted in order to restore.
+  gBrowser._insertBrowser(tab);
+  ss.setTabState(tab, newTabState);
+}
+

browser/components/sessionstore/test/browser_615394-SSWindowState_events_setWindowState.js

@@ -0,0 +1,55 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+function test() {
+  /** Test for Bug 615394 - Session Restore should notify when it is beginning and ending a restore **/
+  waitForExplicitFinish();
+
+  let newState = {
+    windows: [{
+      tabs: [
+        { entries: [{ url: "about:mozilla", triggeringPrincipal_base64 }], extData: { "foo": "bar" } },
+        { entries: [{ url: "http://example.org", triggeringPrincipal_base64 }], extData: { "baz": "qux" } }
+      ]
+    }]
+  };
+
+  let busyEventCount = 0,
+      readyEventCount = 0,
+      tabRestoredCount = 0;
+
+  function onSSWindowStateBusy(aEvent) {
+    busyEventCount++;
+  }
+
+  function onSSWindowStateReady(aEvent) {
+    readyEventCount++;
+    is(ss.getTabValue(gBrowser.tabs[0], "foo"), "bar");
+    is(ss.getTabValue(gBrowser.tabs[1], "baz"), "qux");
+  }
+
+  function onSSTabRestored(aEvent) {
+    if (++tabRestoredCount < 2)
+      return;
+
+    is(busyEventCount, 1);
+    is(readyEventCount, 1);
+    is(gBrowser.tabs[0].linkedBrowser.currentURI.spec, "about:mozilla");
+    is(gBrowser.tabs[1].linkedBrowser.currentURI.spec, "http://example.org/");
+
+    window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+    window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
+    gBrowser.tabContainer.removeEventListener("SSTabRestored", onSSTabRestored);
+
+    gBrowser.removeTab(gBrowser.tabs[1]);
+    finish();
+  }
+
+  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
+  gBrowser.tabContainer.addEventListener("SSTabRestored", onSSTabRestored);
+
+  ss.setWindowState(window, JSON.stringify(newState), true);
+}
+

browser/components/sessionstore/test/browser_615394-SSWindowState_events_undoCloseTab.js

@@ -0,0 +1,60 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const testState = {
+  windows: [{
+    tabs: [
+      { entries: [{ url: "about:blank", triggeringPrincipal_base64 }] },
+      { entries: [{ url: "about:rights", triggeringPrincipal_base64 }] }
+    ]
+  }]
+};
+
+function test() {
+  /** Test for Bug 615394 - Session Restore should notify when it is beginning and ending a restore **/
+  waitForExplicitFinish();
+
+  waitForBrowserState(testState, test_undoCloseTab);
+}
+
+function test_undoCloseTab() {
+  let tab = gBrowser.tabs[1],
+      busyEventCount = 0,
+      readyEventCount = 0,
+      reopenedTab;
+
+  ss.setTabValue(tab, "foo", "bar");
+
+  function onSSWindowStateBusy(aEvent) {
+    busyEventCount++;
+  }
+
+  function onSSWindowStateReady(aEvent) {
+    reopenedTab = gBrowser.tabs[1];
+    readyEventCount++;
+    is(ss.getTabValue(reopenedTab, "foo"), "bar");
+    ss.setTabValue(reopenedTab, "baz", "qux");
+  }
+
+  function onSSTabRestored(aEvent) {
+    is(busyEventCount, 1);
+    is(readyEventCount, 1);
+    is(ss.getTabValue(reopenedTab, "baz"), "qux");
+    is(reopenedTab.linkedBrowser.currentURI.spec, "about:rights");
+
+    window.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+    window.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
+
+    gBrowser.removeTab(gBrowser.tabs[1]);
+    finish();
+  }
+
+  window.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+  window.addEventListener("SSWindowStateReady", onSSWindowStateReady);
+  gBrowser.tabContainer.addEventListener("SSTabRestored", onSSTabRestored, { once: true });
+
+  gBrowser.removeTab(tab);
+  reopenedTab = ss.undoCloseTab(window, 0);
+}
+

browser/components/sessionstore/test/browser_615394-SSWindowState_events_undoCloseWindow.js

@@ -0,0 +1,85 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const lameMultiWindowState = { windows: [
+    {
+      tabs: [
+        { entries: [{ url: "http://example.org#1", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.org#2", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.org#3", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.org#4", triggeringPrincipal_base64 }], extData: { "uniq": r() } }
+      ],
+      selected: 1
+    },
+    {
+      tabs: [
+        { entries: [{ url: "http://example.com#1", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.com#2", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.com#3", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+        { entries: [{ url: "http://example.com#4", triggeringPrincipal_base64 }], extData: { "uniq": r() } },
+      ],
+      selected: 3
+    }
+  ] };
+
+function test() {
+  /** Test for Bug 615394 - Session Restore should notify when it is beginning and ending a restore **/
+  waitForExplicitFinish();
+
+  let newWindow, reopenedWindow;
+
+  function firstWindowObserver(aSubject, aTopic, aData) {
+    if (aTopic == "domwindowopened") {
+      newWindow = aSubject.QueryInterface(Ci.nsIDOMWindow);
+      Services.ww.unregisterNotification(firstWindowObserver);
+    }
+  }
+  Services.ww.registerNotification(firstWindowObserver);
+
+  waitForBrowserState(lameMultiWindowState, function() {
+    // Close the window which isn't window
+    BrowserTestUtils.closeWindow(newWindow).then(() => {
+      // Now give it time to close
+      reopenedWindow = ss.undoCloseWindow(0);
+      reopenedWindow.addEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+      reopenedWindow.addEventListener("SSWindowStateReady", onSSWindowStateReady);
+
+      reopenedWindow.addEventListener("load", function() {
+        reopenedWindow.gBrowser.tabContainer.addEventListener("SSTabRestored", onSSTabRestored);
+      }, {once: true});
+    });
+  });
+
+  let busyEventCount = 0,
+      readyEventCount = 0,
+      tabRestoredCount = 0;
+  // These will listen to the reopened closed window...
+  function onSSWindowStateBusy(aEvent) {
+    busyEventCount++;
+  }
+
+  function onSSWindowStateReady(aEvent) {
+    readyEventCount++;
+  }
+
+  function onSSTabRestored(aEvent) {
+    if (++tabRestoredCount < 4)
+      return;
+
+    is(busyEventCount, 1);
+    is(readyEventCount, 1);
+
+    reopenedWindow.removeEventListener("SSWindowStateBusy", onSSWindowStateBusy);
+    reopenedWindow.removeEventListener("SSWindowStateReady", onSSWindowStateReady);
+    reopenedWindow.gBrowser.tabContainer.removeEventListener("SSTabRestored", onSSTabRestored);
+
+    reopenedWindow.close();
+    while (gBrowser.tabs.length > 1) {
+      gBrowser.removeTab(gBrowser.tabs[1]);
+    }
+
+    finish();
+  }
+}
+

browser/extensions/shield-recipe-client/bootstrap.js

@@ -4,81 +4,27 @@
 "use strict";
 
 const {utils: Cu} = Components;
-Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/Preferences.jsm");
-Cu.import("resource://gre/modules/Log.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "LogManager",
   "resource://shield-recipe-client/lib/LogManager.jsm");
-XPCOMUtils.defineLazyModuleGetter(this, "RecipeRunner",
-  "resource://shield-recipe-client/lib/RecipeRunner.jsm");
-XPCOMUtils.defineLazyModuleGetter(this, "CleanupManager",
-  "resource://shield-recipe-client/lib/CleanupManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "ShieldRecipeClient",
+  "resource://shield-recipe-client/lib/ShieldRecipeClient.jsm");
 
-const REASONS = {
-  APP_STARTUP: 1,      // The application is starting up.
-  APP_SHUTDOWN: 2,     // The application is shutting down.
-  ADDON_ENABLE: 3,     // The add-on is being enabled.
-  ADDON_DISABLE: 4,    // The add-on is being disabled. (Also sent during uninstallation)
-  ADDON_INSTALL: 5,    // The add-on is being installed.
-  ADDON_UNINSTALL: 6,  // The add-on is being uninstalled.
-  ADDON_UPGRADE: 7,    // The add-on is being upgraded.
-  ADDON_DOWNGRADE: 8,  // The add-on is being downgraded.
-};
+this.install = function() {};
 
-const PREF_BRANCH = "extensions.shield-recipe-client.";
-const DEFAULT_PREFS = {
-  api_url: "https://normandy.cdn.mozilla.net/api/v1",
-  dev_mode: false,
-  enabled: true,
-  startup_delay_seconds: 300,
-  "logging.level": Log.Level.Warn,
-  user_id: "",
-};
-const PREF_DEV_MODE = "extensions.shield-recipe-client.dev_mode";
-const PREF_SELF_SUPPORT_ENABLED = "browser.selfsupport.enabled";
-const PREF_LOGGING_LEVEL = PREF_BRANCH + "logging.level";
-
-let shouldRun = true;
-let log = null;
-
-this.install = function() {
-  // Self Repair only checks its pref on start, so if we disable it, wait until
-  // next startup to run, unless the dev_mode preference is set.
-  if (Preferences.get(PREF_SELF_SUPPORT_ENABLED, true)) {
-    Preferences.set(PREF_SELF_SUPPORT_ENABLED, false);
-    if (!Preferences.get(PREF_DEV_MODE, false)) {
-      shouldRun = false;
-    }
-  }
-};
-
-this.startup = function() {
-  setDefaultPrefs();
-
-  // Setup logging and listen for changes to logging prefs
-  LogManager.configure(Services.prefs.getIntPref(PREF_LOGGING_LEVEL));
-  log = LogManager.getLogger("bootstrap");
-  Preferences.observe(PREF_LOGGING_LEVEL, LogManager.configure);
-  CleanupManager.addCleanupHandler(
-    () => Preferences.ignore(PREF_LOGGING_LEVEL, LogManager.configure));
-
-  if (!shouldRun) {
-    return;
-  }
-
-  RecipeRunner.init();
+this.startup = async function() {
+  await ShieldRecipeClient.startup();
 };
 
 this.shutdown = function(data, reason) {
-  CleanupManager.cleanup();
-
-  if (reason === REASONS.ADDON_DISABLE || reason === REASONS.ADDON_UNINSTALL) {
-    Services.prefs.setBoolPref(PREF_SELF_SUPPORT_ENABLED, true);
-  }
+  ShieldRecipeClient.shutdown(reason);
 
+  // Unload add-on modules. We don't do this in ShieldRecipeClient so that
+  // modules are not unloaded accidentally during tests.
+  const log = LogManager.getLogger("bootstrap");
   const modules = [
+    "lib/ActionSandboxManager.jsm",
     "lib/CleanupManager.jsm",
     "lib/ClientEnvironment.jsm",
     "lib/FilterExpressions.jsm",
@@ -87,29 +33,18 @@ this.shutdown = function(data, reason) {
     "lib/LogManager.jsm",
     "lib/NormandyApi.jsm",
     "lib/NormandyDriver.jsm",
+    "lib/PreferenceExperiments.jsm",
     "lib/RecipeRunner.jsm",
     "lib/Sampling.jsm",
     "lib/SandboxManager.jsm",
+    "lib/ShieldRecipeClient.jsm",
     "lib/Storage.jsm",
+    "lib/Utils.jsm",
   ];
   for (const module of modules) {
     log.debug(`Unloading ${module}`);
     Cu.unload(`resource://shield-recipe-client/${module}`);
   }
-
-  // Don't forget the logger!
-  log = null;
 };
 
-this.uninstall = function() {
-};
-
-function setDefaultPrefs() {
-  for (const [key, val] of Object.entries(DEFAULT_PREFS)) {
-    const fullKey = PREF_BRANCH + key;
-    // If someone beat us to setting a default, don't overwrite it.
-    if (!Preferences.isSet(fullKey)) {
-      Preferences.set(fullKey, val);
-    }
-  }
-}
+this.uninstall = function() {};

browser/extensions/shield-recipe-client/install.rdf.in

@@ -8,7 +8,7 @@
     <em:type>2</em:type>
     <em:bootstrap>true</em:bootstrap>
     <em:unpack>false</em:unpack>
-    <em:version>1.0.0</em:version>
+    <em:version>51</em:version>
     <em:name>Shield Recipe Client</em:name>
     <em:description>Client to download and run recipes for SHIELD, Heartbeat, etc.</em:description>
     <em:multiprocessCompatible>true</em:multiprocessCompatible>

browser/extensions/shield-recipe-client/lib/ActionSandboxManager.jsm

@@ -0,0 +1,84 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {utils: Cu} = Components;
+Cu.import("resource://shield-recipe-client/lib/NormandyDriver.jsm");
+Cu.import("resource://shield-recipe-client/lib/SandboxManager.jsm");
+Cu.import("resource://shield-recipe-client/lib/LogManager.jsm");
+
+this.EXPORTED_SYMBOLS = ["ActionSandboxManager"];
+
+const log = LogManager.getLogger("recipe-sandbox-manager");
+
+/**
+ * An extension to SandboxManager that prepares a sandbox for executing
+ * Normandy actions.
+ *
+ * Actions register a set of named callbacks, which this class makes available
+ * for execution. This allows a single action script to define multiple,
+ * independent steps that execute in isolated sandboxes.
+ *
+ * Callbacks are assumed to be async and must return Promises.
+ */
+this.ActionSandboxManager = class extends SandboxManager {
+  constructor(actionScript) {
+    super();
+
+    // Prepare the sandbox environment
+    const driver = new NormandyDriver(this);
+    this.cloneIntoGlobal("sandboxedDriver", driver, {cloneFunctions: true});
+    this.evalInSandbox(`
+      // Shim old API for registering actions
+      function registerAction(name, Action) {
+        registerAsyncCallback("action", (driver, recipe) => {
+          return new Action(driver, recipe).execute();
+        });
+      };
+
+      this.asyncCallbacks = new Map();
+      function registerAsyncCallback(name, callback) {
+        asyncCallbacks.set(name, callback);
+      }
+
+      this.window = this;
+      this.setTimeout = sandboxedDriver.setTimeout;
+      this.clearTimeout = sandboxedDriver.clearTimeout;
+    `);
+    this.evalInSandbox(actionScript);
+  }
+
+  /**
+   * Execute a callback in the sandbox with the given name. If the script does
+   * not register a callback with the given name, we log a message and return.
+   * @param {String} callbackName
+   * @param {...*} [args]
+   *   Remaining arguments are cloned into the sandbox and passed as arguments
+   *   to the callback.
+   * @resolves
+   *   The return value of the callback, cloned into the current compartment, or
+   *   undefined if a matching callback was not found.
+   * @rejects
+   *   If the sandbox rejects, an error object with the message from the sandbox
+   *   error. Due to sandbox limitations, the stack trace is not preserved.
+   */
+  async runAsyncCallback(callbackName, ...args) {
+    const callbackWasRegistered = this.evalInSandbox(`asyncCallbacks.has("${callbackName}")`);
+    if (!callbackWasRegistered) {
+      log.debug(`Script did not register a callback with the name "${callbackName}"`);
+      return undefined;
+    }
+
+    this.cloneIntoGlobal("callbackArgs", args);
+    try {
+      const result = await this.evalInSandbox(`
+        asyncCallbacks.get("${callbackName}")(sandboxedDriver, ...callbackArgs);
+      `);
+      return Cu.cloneInto(result, {});
+    } catch (err) {
+      throw new Error(Cu.cloneInto(err.message, {}));
+    }
+  }
+};

browser/extensions/shield-recipe-client/lib/ClientEnvironment.jsm

@@ -7,13 +7,18 @@
 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "ShellService", "resource:///modules/ShellService.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "AddonManager", "resource://gre/modules/AddonManager.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "TelemetryArchive", "resource://gre/modules/TelemetryArchive.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "NormandyApi", "resource://shield-recipe-client/lib/NormandyApi.jsm");
+XPCOMUtils.defineLazyModuleGetter(
+    this,
+    "PreferenceExperiments",
+    "resource://shield-recipe-client/lib/PreferenceExperiments.jsm",
+);
+XPCOMUtils.defineLazyModuleGetter(this, "Utils", "resource://shield-recipe-client/lib/Utils.jsm");
 
 const {generateUUID} = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator);
 
@@ -31,12 +36,12 @@ this.ClientEnvironment = {
    * The server request is made lazily and is cached for the entire browser
    * session.
    */
-  getClientClassification: Task.async(function *() {
+  async getClientClassification() {
     if (!_classifyRequest) {
       _classifyRequest = NormandyApi.classifyClient();
     }
-    return yield _classifyRequest;
-  }),
+    return await _classifyRequest;
+  },
 
   clearClassifyCache() {
     _classifyRequest = null;
@@ -45,13 +50,13 @@ this.ClientEnvironment = {
   /**
    * Test wrapper that mocks the server request for classifying the client.
    * @param  {Object}   data          Fake server data to use
-   * @param  {Function} testGenerator Test generator to execute while mock data is in effect.
+   * @param  {Function} testFunction  Test function to execute while mock data is in effect.
    */
-  withMockClassify(data, testGenerator) {
-    return function* inner() {
+  withMockClassify(data, testFunction) {
+    return async function inner() {
       const oldRequest = _classifyRequest;
       _classifyRequest = Promise.resolve(data);
-      yield testGenerator();
+      await testFunction();
       _classifyRequest = oldRequest;
     };
   },
@@ -94,8 +99,8 @@ this.ClientEnvironment = {
       return Preferences.get("distribution.id", "default");
     });
 
-    XPCOMUtils.defineLazyGetter(environment, "telemetry", Task.async(function *() {
-      const pings = yield TelemetryArchive.promiseArchivedPingList();
+    XPCOMUtils.defineLazyGetter(environment, "telemetry", async function() {
+      const pings = await TelemetryArchive.promiseArchivedPingList();
 
       // get most recent ping per type
       const mostRecentPings = {};
@@ -112,10 +117,10 @@ this.ClientEnvironment = {
       const telemetry = {};
       for (const key in mostRecentPings) {
         const ping = mostRecentPings[key];
-        telemetry[ping.type] = yield TelemetryArchive.promiseArchivedPingById(ping.id);
+        telemetry[ping.type] = await TelemetryArchive.promiseArchivedPingById(ping.id);
       }
       return telemetry;
-    }));
+    });
 
     XPCOMUtils.defineLazyGetter(environment, "version", () => {
       return Services.appinfo.version;
@@ -129,13 +134,13 @@ this.ClientEnvironment = {
       return ShellService.isDefaultBrowser();
     });
 
-    XPCOMUtils.defineLazyGetter(environment, "searchEngine", Task.async(function* () {
-      const searchInitialized = yield new Promise(resolve => Services.search.init(resolve));
+    XPCOMUtils.defineLazyGetter(environment, "searchEngine", async function() {
+      const searchInitialized = await new Promise(resolve => Services.search.init(resolve));
       if (Components.isSuccessCode(searchInitialized)) {
         return Services.search.defaultEngine.identifier;
       }
       return null;
-    }));
+    });
 
     XPCOMUtils.defineLazyGetter(environment, "syncSetup", () => {
       return Preferences.isSet("services.sync.username");
@@ -150,31 +155,48 @@ this.ClientEnvironment = {
     });
 
     XPCOMUtils.defineLazyGetter(environment, "syncTotalDevices", () => {
-      return Preferences.get("services.sync.numClients", 0);
+      return environment.syncDesktopDevices + environment.syncMobileDevices;
     });
 
-    XPCOMUtils.defineLazyGetter(environment, "plugins", Task.async(function* () {
-      const plugins = yield AddonManager.getAddonsByTypes(["plugin"]);
-      return plugins.reduce((pluginMap, plugin) => {
-        pluginMap[plugin.name] = {
-          name: plugin.name,
-          description: plugin.description,
-          version: plugin.version,
-        };
-        return pluginMap;
-      }, {});
-    }));
+    XPCOMUtils.defineLazyGetter(environment, "plugins", async function() {
+      let plugins = await AddonManager.getAddonsByTypes(["plugin"]);
+      plugins = plugins.map(plugin => ({
+        name: plugin.name,
+        description: plugin.description,
+        version: plugin.version,
+      }));
+      return Utils.keyBy(plugins, "name");
+    });
 
     XPCOMUtils.defineLazyGetter(environment, "locale", () => {
+      if (Services.locale.getAppLocaleAsLangTag) {
+        return Services.locale.getAppLocaleAsLangTag();
+      }
+
       return Cc["@mozilla.org/chrome/chrome-registry;1"]
         .getService(Ci.nsIXULChromeRegistry)
-        .getSelectedLocale("browser");
+        .getSelectedLocale("global");
     });
 
     XPCOMUtils.defineLazyGetter(environment, "doNotTrack", () => {
       return Preferences.get("privacy.donottrackheader.enabled", false);
     });
 
+    XPCOMUtils.defineLazyGetter(environment, "experiments", async () => {
+      const names = {all: [], active: [], expired: []};
+
+      for (const experiment of await PreferenceExperiments.getAll()) {
+        names.all.push(experiment.name);
+        if (experiment.expired) {
+          names.expired.push(experiment.name);
+        } else {
+          names.active.push(experiment.name);
+        }
+      }
+
+      return names;
+    });
+
     return environment;
   },
 };

browser/extensions/shield-recipe-client/lib/FilterExpressions.jsm

@@ -7,6 +7,7 @@
 const {utils: Cu} = Components;
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 Cu.import("resource://shield-recipe-client/lib/Sampling.jsm");
+Cu.import("resource://shield-recipe-client/lib/PreferenceFilters.jsm");
 
 this.EXPORTED_SYMBOLS = ["FilterExpressions"];
 
@@ -27,6 +28,9 @@ XPCOMUtils.defineLazyGetter(this, "jexl", () => {
     date: dateString => new Date(dateString),
     stableSample: Sampling.stableSample,
     bucketSample: Sampling.bucketSample,
+    preferenceValue: PreferenceFilters.preferenceValue,
+    preferenceIsUserSet: PreferenceFilters.preferenceIsUserSet,
+    preferenceExists: PreferenceFilters.preferenceExists,
   });
   return jexl;
 });

browser/extensions/shield-recipe-client/lib/NormandyApi.jsm

@@ -6,9 +6,9 @@
 
 const {utils: Cu, classes: Cc, interfaces: Ci} = Components;
 Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/CanonicalJSON.jsm");
 Cu.import("resource://shield-recipe-client/lib/LogManager.jsm");
+Cu.import("resource://shield-recipe-client/lib/Utils.jsm");
 Cu.importGlobalProperties(["fetch", "URL"]); /* globals fetch, URL */
 
 this.EXPORTED_SYMBOLS = ["NormandyApi"];
@@ -61,23 +61,23 @@ this.NormandyApi = {
     throw new Error("Can't use relative urls");
   },
 
-  getApiUrl: Task.async(function * (name) {
+  async getApiUrl(name) {
     const apiBase = prefs.getCharPref("api_url");
     if (!indexPromise) {
       indexPromise = this.get(apiBase).then(res => res.json());
     }
-    const index = yield indexPromise;
+    const index = await indexPromise;
     if (!(name in index)) {
       throw new Error(`API endpoint with name "${name}" not found.`);
     }
     const url = index[name];
     return this.absolutify(url);
-  }),
+  },
 
-  fetchRecipes: Task.async(function* (filters = {enabled: true}) {
-    const signedRecipesUrl = yield this.getApiUrl("recipe-signed");
-    const recipesResponse = yield this.get(signedRecipesUrl, filters);
-    const rawText = yield recipesResponse.text();
+  async fetchRecipes(filters = {enabled: true}) {
+    const signedRecipesUrl = await this.getApiUrl("recipe-signed");
+    const recipesResponse = await this.get(signedRecipesUrl, filters);
+    const rawText = await recipesResponse.text();
     const recipesWithSigs = JSON.parse(rawText);
 
     const verifiedRecipes = [];
@@ -89,8 +89,8 @@ this.NormandyApi = {
         throw new Error("Canonical recipe serialization does not match!");
       }
 
-      const certChainResponse = yield fetch(this.absolutify(x5u));
-      const certChain = yield certChainResponse.text();
+      const certChainResponse = await fetch(this.absolutify(x5u));
+      const certChain = await certChainResponse.text();
       const builtSignature = `p384ecdsa=${signature}`;
 
       const verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
@@ -114,26 +114,36 @@ this.NormandyApi = {
     );
 
     return verifiedRecipes;
-  }),
+  },
 
   /**
    * Fetch metadata about this client determined by the server.
    * @return {object} Metadata specified by the server
    */
-  classifyClient: Task.async(function* () {
-    const classifyClientUrl = yield this.getApiUrl("classify-client");
-    const response = yield this.get(classifyClientUrl);
-    const clientData = yield response.json();
+  async classifyClient() {
+    const classifyClientUrl = await this.getApiUrl("classify-client");
+    const response = await this.get(classifyClientUrl);
+    const clientData = await response.json();
     clientData.request_time = new Date(clientData.request_time);
     return clientData;
-  }),
+  },
 
-  fetchAction: Task.async(function* (name) {
-    let actionApiUrl = yield this.getApiUrl("action-list");
-    if (!actionApiUrl.endsWith("/")) {
-      actionApiUrl += "/";
+  /**
+   * Fetch an array of available actions from the server.
+   * @resolves {Array}
+   */
+  async fetchActions() {
+    const actionApiUrl = await this.getApiUrl("action-list");
+    const res = await this.get(actionApiUrl);
+    return await res.json();
+  },
+
+  async fetchImplementation(action) {
+    const response = await fetch(action.implementation_url);
+    if (response.ok) {
+      return await response.text();
     }
-    const res = yield this.get(actionApiUrl + name);
-    return yield res.json();
-  }),
+
+    throw new Error(`Failed to fetch action implementation for ${action.name}: ${response.status}`);
+  },
 };

browser/extensions/shield-recipe-client/lib/NormandyDriver.jsm

@@ -3,7 +3,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
-/* globals Components */
 
 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 
@@ -17,6 +16,8 @@ Cu.import("resource://shield-recipe-client/lib/Storage.jsm");
 Cu.import("resource://shield-recipe-client/lib/Heartbeat.jsm");
 Cu.import("resource://shield-recipe-client/lib/FilterExpressions.jsm");
 Cu.import("resource://shield-recipe-client/lib/ClientEnvironment.jsm");
+Cu.import("resource://shield-recipe-client/lib/PreferenceExperiments.jsm");
+Cu.import("resource://shield-recipe-client/lib/Sampling.jsm");
 
 const {generateUUID} = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator);
 
@@ -35,9 +36,13 @@ this.NormandyDriver = function(sandboxManager) {
     testing: false,
 
     get locale() {
+      if (Services.locale.getAppLocaleAsLangTag) {
+        return Services.locale.getAppLocaleAsLangTag();
+      }
+
       return Cc["@mozilla.org/chrome/chrome-registry;1"]
         .getService(Ci.nsIXULChromeRegistry)
-        .getSelectedLocale("browser");
+        .getSelectedLocale("global");
     },
 
     get userId() {
@@ -78,11 +83,12 @@ this.NormandyDriver = function(sandboxManager) {
         syncSetup: Preferences.isSet("services.sync.username"),
         syncDesktopDevices: Preferences.get("services.sync.clients.devices.desktop", 0),
         syncMobileDevices: Preferences.get("services.sync.clients.devices.mobile", 0),
-        syncTotalDevices: Preferences.get("services.sync.numClients", 0),
+        syncTotalDevices: null,
         plugins: {},
         doNotTrack: Preferences.get("privacy.donottrackheader.enabled", false),
         distribution: Preferences.get("distribution.id", "default"),
       };
+      appinfo.syncTotalDevices = appinfo.syncDesktopDevices + appinfo.syncMobileDevices;
 
       const searchEnginePromise = new Promise(resolve => {
         Services.search.init(rv => {
@@ -144,5 +150,18 @@ this.NormandyDriver = function(sandboxManager) {
       clearTimeout(token);
       sandboxManager.removeHold(`setTimeout-${token}`);
     },
+
+    // Sampling
+    ratioSample: sandboxManager.wrapAsync(Sampling.ratioSample),
+
+    // Preference Experiment API
+    preferenceExperiments: {
+      start: sandboxManager.wrapAsync(PreferenceExperiments.start, {cloneArguments: true}),
+      markLastSeen: sandboxManager.wrapAsync(PreferenceExperiments.markLastSeen),
+      stop: sandboxManager.wrapAsync(PreferenceExperiments.stop),
+      get: sandboxManager.wrapAsync(PreferenceExperiments.get, {cloneInto: true}),
+      getAllActive: sandboxManager.wrapAsync(PreferenceExperiments.getAllActive, {cloneInto: true}),
+      has: sandboxManager.wrapAsync(PreferenceExperiments.has),
+    },
   };
 };

browser/extensions/shield-recipe-client/lib/PreferenceExperiments.jsm

@@ -0,0 +1,424 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Preference Experiments temporarily change a preference to one of several test
+ * values for the duration of the experiment. Telemetry packets are annotated to
+ * show what experiments are active, and we use this data to measure the
+ * effectiveness of the preference change.
+ *
+ * Info on active and past experiments is stored in a JSON file in the profile
+ * folder.
+ *
+ * Active preference experiments are stopped if they aren't active on the recipe
+ * server. They also expire if Firefox isn't able to contact the recipe server
+ * after a period of time, as well as if the user modifies the preference during
+ * an active experiment.
+ */
+
+/**
+ * Experiments store info about an active or expired preference experiment.
+ * They are single-depth objects to simplify cloning.
+ * @typedef {Object} Experiment
+ * @property {string} name
+ *   Unique name of the experiment
+ * @property {string} branch
+ *   Experiment branch that the user was matched to
+ * @property {boolean} expired
+ *   If false, the experiment is active.
+ * @property {string} lastSeen
+ *   ISO-formatted date string of when the experiment was last seen from the
+ *   recipe server.
+ * @property {string} preferenceName
+ *   Name of the preference affected by this experiment.
+ * @property {string|integer|boolean} preferenceValue
+ *   Value to change the preference to during the experiment.
+ * @property {string} preferenceType
+ *   Type of the preference value being set.
+ * @property {string|integer|boolean|undefined} previousPreferenceValue
+ *   Value of the preference prior to the experiment, or undefined if it was
+ *   unset.
+ * @property {PreferenceBranchType} preferenceBranchType
+ *   Controls how we modify the preference to affect the client.
+ * @rejects {Error}
+ *   If the given preferenceType does not match the existing stored preference.
+ *
+ *   If "default", when the experiment is active, the default value for the
+ *   preference is modified on startup of the add-on. If "user", the user value
+ *   for the preference is modified when the experiment starts, and is reset to
+ *   its original value when the experiment ends.
+ */
+
+"use strict";
+
+const {utils: Cu} = Components;
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "Services", "resource://gre/modules/Services.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "CleanupManager", "resource://shield-recipe-client/lib/CleanupManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "JSONFile", "resource://gre/modules/JSONFile.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "LogManager", "resource://shield-recipe-client/lib/LogManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Preferences", "resource://gre/modules/Preferences.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "TelemetryEnvironment", "resource://gre/modules/TelemetryEnvironment.jsm");
+
+this.EXPORTED_SYMBOLS = ["PreferenceExperiments"];
+
+const EXPERIMENT_FILE = "shield-preference-experiments.json";
+
+const PREFERENCE_TYPE_MAP = {
+  boolean: Services.prefs.PREF_BOOL,
+  string: Services.prefs.PREF_STRING,
+  integer: Services.prefs.PREF_INT,
+};
+
+const DefaultPreferences = new Preferences({defaultBranch: true});
+
+/**
+ * Enum storing Preference modules for each type of preference branch.
+ * @enum {Object}
+ */
+const PreferenceBranchType = {
+  user: Preferences,
+  default: DefaultPreferences,
+};
+
+/**
+ * Asynchronously load the JSON file that stores experiment status in the profile.
+ */
+let storePromise;
+function ensureStorage() {
+  if (storePromise === undefined) {
+    const path = OS.Path.join(OS.Constants.Path.profileDir, EXPERIMENT_FILE);
+    const storage = new JSONFile({path});
+    storePromise = storage.load().then(() => storage);
+  }
+  return storePromise;
+}
+
+const log = LogManager.getLogger("preference-experiments");
+
+// List of active preference observers. Cleaned up on shutdown.
+let experimentObservers = new Map();
+CleanupManager.addCleanupHandler(() => PreferenceExperiments.stopAllObservers());
+
+this.PreferenceExperiments = {
+  /**
+   * Set the default preference value for active experiments that use the
+   * default preference branch.
+   */
+  async init() {
+    for (const experiment of await this.getAllActive()) {
+      // Set experiment default preferences, since they don't persist between restarts
+      if (experiment.preferenceBranchType === "default") {
+        DefaultPreferences.set(experiment.preferenceName, experiment.preferenceValue);
+      }
+
+      // Check that the current value of the preference is still what we set it to
+      if (Preferences.get(experiment.preferenceName, undefined) !== experiment.preferenceValue) {
+        // if not, stop the experiment, and skip the remaining steps
+        log.info(`Stopping experiment "${experiment.name}" because its value changed`);
+        await this.stop(experiment.name, false);
+        continue;
+      }
+
+      // Notify Telemetry of experiments we're running, since they don't persist between restarts
+      TelemetryEnvironment.setExperimentActive(experiment.name, experiment.branch);
+
+      // Watch for changes to the experiment's preference
+      this.startObserver(experiment.name, experiment.preferenceName, experiment.preferenceValue);
+    }
+  },
+
+  /**
+   * Test wrapper that temporarily replaces the stored experiment data with fake
+   * data for testing.
+   */
+  withMockExperiments(testFunction) {
+    return async function inner(...args) {
+      const oldPromise = storePromise;
+      const mockExperiments = {};
+      storePromise = Promise.resolve({
+        data: mockExperiments,
+        saveSoon() { },
+      });
+      const oldObservers = experimentObservers;
+      experimentObservers = new Map();
+      try {
+        await testFunction(...args, mockExperiments);
+      } finally {
+        storePromise = oldPromise;
+        PreferenceExperiments.stopAllObservers();
+        experimentObservers = oldObservers;
+      }
+    };
+  },
+
+  /**
+   * Clear all stored data about active and past experiments.
+   */
+  async clearAllExperimentStorage() {
+    const store = await ensureStorage();
+    store.data = {};
+    store.saveSoon();
+  },
+
+  /**
+   * Start a new preference experiment.
+   * @param {Object} experiment
+   * @param {string} experiment.name
+   * @param {string} experiment.branch
+   * @param {string} experiment.preferenceName
+   * @param {string|integer|boolean} experiment.preferenceValue
+   * @param {PreferenceBranchType} experiment.preferenceBranchType
+   * @rejects {Error}
+   *   If an experiment with the given name already exists, or if an experiment
+   *   for the given preference is active.
+   */
+  async start({name, branch, preferenceName, preferenceValue, preferenceBranchType, preferenceType}) {
+    log.debug(`PreferenceExperiments.start(${name}, ${branch})`);
+
+    const store = await ensureStorage();
+    if (name in store.data) {
+      throw new Error(`A preference experiment named "${name}" already exists.`);
+    }
+
+    const activeExperiments = Object.values(store.data).filter(e => !e.expired);
+    const hasConflictingExperiment = activeExperiments.some(
+      e => e.preferenceName === preferenceName
+    );
+    if (hasConflictingExperiment) {
+      throw new Error(
+        `Another preference experiment for the pref "${preferenceName}" is currently active.`
+      );
+    }
+
+    const preferences = PreferenceBranchType[preferenceBranchType];
+    if (!preferences) {
+      throw new Error(`Invalid value for preferenceBranchType: ${preferenceBranchType}`);
+    }
+
+    /** @type {Experiment} */
+    const experiment = {
+      name,
+      branch,
+      expired: false,
+      lastSeen: new Date().toJSON(),
+      preferenceName,
+      preferenceValue,
+      preferenceType,
+      previousPreferenceValue: preferences.get(preferenceName, undefined),
+      preferenceBranchType,
+    };
+
+    const prevPrefType = Services.prefs.getPrefType(preferenceName);
+    const givenPrefType = PREFERENCE_TYPE_MAP[preferenceType];
+
+    if (!preferenceType || !givenPrefType) {
+      throw new Error(`Invalid preferenceType provided (given "${preferenceType}")`);
+    }
+
+    if (prevPrefType !== Services.prefs.PREF_INVALID && prevPrefType !== givenPrefType) {
+      throw new Error(
+        `Previous preference value is of type "${prevPrefType}", but was given "${givenPrefType}" (${preferenceType})`
+      );
+    }
+
+    preferences.set(preferenceName, preferenceValue);
+    PreferenceExperiments.startObserver(name, preferenceName, preferenceValue);
+    store.data[name] = experiment;
+    store.saveSoon();
+
+    TelemetryEnvironment.setExperimentActive(name, branch);
+  },
+
+  /**
+   * Register a preference observer that stops an experiment when the user
+   * modifies the preference.
+   * @param {string} experimentName
+   * @param {string} preferenceName
+   * @param {string|integer|boolean} preferenceValue
+   * @throws {Error}
+   *   If an observer for the named experiment is already active.
+   */
+  startObserver(experimentName, preferenceName, preferenceValue) {
+    log.debug(`PreferenceExperiments.startObserver(${experimentName})`);
+
+    if (experimentObservers.has(experimentName)) {
+      throw new Error(
+        `An observer for the preference experiment ${experimentName} is already active.`
+      );
+    }
+
+    const observerInfo = {
+      preferenceName,
+      observer(newValue) {
+        if (newValue !== preferenceValue) {
+          PreferenceExperiments.stop(experimentName, false);
+        }
+      },
+    };
+    experimentObservers.set(experimentName, observerInfo);
+    Preferences.observe(preferenceName, observerInfo.observer);
+  },
+
+  /**
+   * Check if a preference observer is active for an experiment.
+   * @param {string} experimentName
+   * @return {Boolean}
+   */
+  hasObserver(experimentName) {
+    log.debug(`PreferenceExperiments.hasObserver(${experimentName})`);
+    return experimentObservers.has(experimentName);
+  },
+
+  /**
+   * Disable a preference observer for the named experiment.
+   * @param {string} experimentName
+   * @throws {Error}
+   *   If there is no active observer for the named experiment.
+   */
+  stopObserver(experimentName) {
+    log.debug(`PreferenceExperiments.stopObserver(${experimentName})`);
+
+    if (!experimentObservers.has(experimentName)) {
+      throw new Error(`No observer for the preference experiment ${experimentName} found.`);
+    }
+
+    const {preferenceName, observer} = experimentObservers.get(experimentName);
+    Preferences.ignore(preferenceName, observer);
+    experimentObservers.delete(experimentName);
+  },
+
+  /**
+   * Disable all currently-active preference observers for experiments.
+   */
+  stopAllObservers() {
+    log.debug("PreferenceExperiments.stopAllObservers()");
+    for (const {preferenceName, observer} of experimentObservers.values()) {
+      Preferences.ignore(preferenceName, observer);
+    }
+    experimentObservers.clear();
+  },
+
+  /**
+   * Update the timestamp storing when Normandy last sent a recipe for the named
+   * experiment.
+   * @param {string} experimentName
+   * @rejects {Error}
+   *   If there is no stored experiment with the given name.
+   */
+  async markLastSeen(experimentName) {
+    log.debug(`PreferenceExperiments.markLastSeen(${experimentName})`);
+
+    const store = await ensureStorage();
+    if (!(experimentName in store.data)) {
+      throw new Error(`Could not find a preference experiment named "${experimentName}"`);
+    }
+
+    store.data[experimentName].lastSeen = new Date().toJSON();
+    store.saveSoon();
+  },
+
+  /**
+   * Stop an active experiment, deactivate preference watchers, and optionally
+   * reset the associated preference to its previous value.
+   * @param {string} experimentName
+   * @param {boolean} [resetValue=true]
+   *   If true, reset the preference to its original value.
+   * @rejects {Error}
+   *   If there is no stored experiment with the given name, or if the
+   *   experiment has already expired.
+   */
+  async stop(experimentName, resetValue = true) {
+    log.debug(`PreferenceExperiments.stop(${experimentName})`);
+
+    const store = await ensureStorage();
+    if (!(experimentName in store.data)) {
+      throw new Error(`Could not find a preference experiment named "${experimentName}"`);
+    }
+
+    const experiment = store.data[experimentName];
+    if (experiment.expired) {
+      throw new Error(
+        `Cannot stop preference experiment "${experimentName}" because it is already expired`
+      );
+    }
+
+    if (PreferenceExperiments.hasObserver(experimentName)) {
+      PreferenceExperiments.stopObserver(experimentName);
+    }
+
+    if (resetValue) {
+      const {preferenceName, previousPreferenceValue, preferenceBranchType} = experiment;
+      const preferences = PreferenceBranchType[preferenceBranchType];
+      if (previousPreferenceValue !== undefined) {
+        preferences.set(preferenceName, previousPreferenceValue);
+      } else {
+        // This does nothing if we're on the default branch, which is fine. The
+        // preference will be reset on next restart, and most preferences should
+        // have had a default value set before the experiment anyway.
+        preferences.reset(preferenceName);
+      }
+    }
+
+    experiment.expired = true;
+    store.saveSoon();
+
+    TelemetryEnvironment.setExperimentInactive(experimentName, experiment.branch);
+  },
+
+  /**
+   * Get the experiment object for the named experiment.
+   * @param {string} experimentName
+   * @resolves {Experiment}
+   * @rejects {Error}
+   *   If no preference experiment exists with the given name.
+   */
+  async get(experimentName) {
+    log.debug(`PreferenceExperiments.get(${experimentName})`);
+    const store = await ensureStorage();
+    if (!(experimentName in store.data)) {
+      throw new Error(`Could not find a preference experiment named "${experimentName}"`);
+    }
+
+    // Return a copy so mutating it doesn't affect the storage.
+    return Object.assign({}, store.data[experimentName]);
+  },
+
+  /**
+   * Get a list of all stored experiment objects.
+   * @resolves {Experiment[]}
+   */
+  async getAll() {
+    const store = await ensureStorage();
+
+    // Return copies so that mutating returned experiments doesn't affect the
+    // stored values.
+    return Object.values(store.data).map(experiment => Object.assign({}, experiment));
+  },
+
+  /**
+  * Get a list of experiment objects for all active experiments.
+  * @resolves {Experiment[]}
+  */
+  async getAllActive() {
+    log.debug("PreferenceExperiments.getAllActive()");
+    const store = await ensureStorage();
+
+    // Return copies so mutating them doesn't affect the storage.
+    return Object.values(store.data).filter(e => !e.expired).map(e => Object.assign({}, e));
+  },
+
+  /**
+   * Check if an experiment exists with the given name.
+   * @param {string} experimentName
+   * @resolves {boolean} True if the experiment exists, false if it doesn't.
+   */
+  async has(experimentName) {
+    log.debug(`PreferenceExperiments.has(${experimentName})`);
+    const store = await ensureStorage();
+    return experimentName in store.data;
+  },
+};

browser/extensions/shield-recipe-client/lib/PreferenceFilters.jsm

@@ -0,0 +1,29 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+
+this.EXPORTED_SYMBOLS = ["PreferenceFilters"];
+
+this.PreferenceFilters = {
+  // Compare the value of a given preference. Takes a `default` value as an
+  // optional argument to pass into `Preferences.get`.
+  preferenceValue(prefKey, defaultValue) {
+    return Preferences.get(prefKey, defaultValue);
+  },
+
+  // Compare if the preference is user set.
+  preferenceIsUserSet(prefKey) {
+    return Preferences.isSet(prefKey);
+  },
+
+  // Compare if the preference has _any_ value, whether it's user-set or default.
+  preferenceExists(prefKey) {
+    return Preferences.has(prefKey);
+  },
+};

browser/extensions/shield-recipe-client/lib/RecipeRunner.jsm

@@ -6,24 +6,38 @@
 
 const {utils: Cu} = Components;
 Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/Timer.jsm"); /* globals setTimeout */
-Cu.import("resource://gre/modules/Task.jsm");
-Cu.import("resource://shield-recipe-client/lib/LogManager.jsm");
-Cu.import("resource://shield-recipe-client/lib/NormandyDriver.jsm");
-Cu.import("resource://shield-recipe-client/lib/FilterExpressions.jsm");
-Cu.import("resource://shield-recipe-client/lib/NormandyApi.jsm");
-Cu.import("resource://shield-recipe-client/lib/SandboxManager.jsm");
-Cu.import("resource://shield-recipe-client/lib/ClientEnvironment.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
-Cu.importGlobalProperties(["fetch"]); /* globals fetch */
+Cu.import("resource://shield-recipe-client/lib/LogManager.jsm");
 
+XPCOMUtils.defineLazyServiceGetter(this, "timerManager",
+                                   "@mozilla.org/updates/timer-manager;1",
+                                   "nsIUpdateTimerManager");
 XPCOMUtils.defineLazyModuleGetter(this, "Preferences", "resource://gre/modules/Preferences.jsm");
-XPCOMUtils.defineLazyModuleGetter(this, "Storage", "resource://shield-recipe-client/lib/Storage.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Storage",
+                                  "resource://shield-recipe-client/lib/Storage.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "NormandyDriver",
+                                  "resource://shield-recipe-client/lib/NormandyDriver.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "FilterExpressions",
+                                  "resource://shield-recipe-client/lib/FilterExpressions.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "NormandyApi",
+                                  "resource://shield-recipe-client/lib/NormandyApi.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "SandboxManager",
+                                  "resource://shield-recipe-client/lib/SandboxManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "ClientEnvironment",
+                                  "resource://shield-recipe-client/lib/ClientEnvironment.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "CleanupManager",
+                                  "resource://shield-recipe-client/lib/CleanupManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "ActionSandboxManager",
+                                  "resource://shield-recipe-client/lib/ActionSandboxManager.jsm");
+
+Cu.importGlobalProperties(["fetch"]); /* globals fetch */
 
 this.EXPORTED_SYMBOLS = ["RecipeRunner"];
 
 const log = LogManager.getLogger("recipe-runner");
 const prefs = Services.prefs.getBranch("extensions.shield-recipe-client.");
+const TIMER_NAME = "recipe-client-addon-run";
+const RUN_INTERVAL_PREF = "run_interval_seconds";
 
 this.RecipeRunner = {
   init() {
@@ -31,15 +45,17 @@ this.RecipeRunner = {
       return;
     }
 
-    let delay;
     if (prefs.getBoolPref("dev_mode")) {
-      delay = 0;
-    } else {
-      // startup delay is in seconds
-      delay = prefs.getIntPref("startup_delay_seconds") * 1000;
+      // Run right now in dev mode
+      this.run();
     }
 
-    setTimeout(this.start.bind(this), delay);
+    this.updateRunInterval();
+    CleanupManager.addCleanupHandler(() => timerManager.unregisterTimer(TIMER_NAME));
+
+    // Watch for the run interval to change, and re-register the timer with the new value
+    prefs.addObserver(RUN_INTERVAL_PREF, this);
+    CleanupManager.addCleanupHandler(() => prefs.removeObserver(RUN_INTERVAL_PREF, this));
   },
 
   checkPrefs() {
@@ -63,46 +79,132 @@ this.RecipeRunner = {
     return true;
   },
 
-  start: Task.async(function* () {
+  /**
+   * Watch for preference changes from Services.pref.addObserver.
+   */
+  observe(changedPrefBranch, action, changedPref) {
+    if (action === "nsPref:changed" && changedPref === RUN_INTERVAL_PREF) {
+      this.updateRunInterval();
+    } else {
+      log.debug(`Observer fired with unexpected pref change: ${action} ${changedPref}`);
+    }
+  },
+
+  updateRunInterval() {
+    // Run once every `runInterval` wall-clock seconds. This is managed by setting a "last ran"
+    // timestamp, and running if it is more than `runInterval` seconds ago. Even with very short
+    // intervals, the timer will only fire at most once every few minutes.
+    const runInterval = prefs.getIntPref(RUN_INTERVAL_PREF);
+    timerManager.registerTimer(TIMER_NAME, () => this.run(), runInterval);
+  },
+
+  async run() {
+    this.clearCaches();
     // Unless lazy classification is enabled, prep the classify cache.
     if (!Preferences.get("extensions.shield-recipe-client.experiments.lazy_classify", false)) {
-      yield ClientEnvironment.getClientClassification();
+      await ClientEnvironment.getClientClassification();
     }
 
+    const actionSandboxManagers = await this.loadActionSandboxManagers();
+    Object.values(actionSandboxManagers).forEach(manager => manager.addHold("recipeRunner"));
+
+    // Run pre-execution hooks. If a hook fails, we don't run recipes with that
+    // action to avoid inconsistencies.
+    for (const [actionName, manager] of Object.entries(actionSandboxManagers)) {
+      try {
+        await manager.runAsyncCallback("preExecution");
+        manager.disabled = false;
+      } catch (err) {
+        log.error(`Could not run pre-execution hook for ${actionName}:`, err.message);
+        manager.disabled = true;
+      }
+    }
+
+    // Fetch recipes from the API
     let recipes;
     try {
-      recipes = yield NormandyApi.fetchRecipes({enabled: true});
+      recipes = await NormandyApi.fetchRecipes({enabled: true});
     } catch (e) {
       const apiUrl = prefs.getCharPref("api_url");
       log.error(`Could not fetch recipes from ${apiUrl}: "${e}"`);
       return;
     }
 
+    // Evaluate recipe filters
     const recipesToRun = [];
-
     for (const recipe of recipes) {
-      if (yield this.checkFilter(recipe)) {
+      if (await this.checkFilter(recipe)) {
         recipesToRun.push(recipe);
       }
     }
 
+    // Execute recipes, if we have any.
     if (recipesToRun.length === 0) {
       log.debug("No recipes to execute");
     } else {
       for (const recipe of recipesToRun) {
-        try {
-          log.debug(`Executing recipe "${recipe.name}" (action=${recipe.action})`);
-          yield this.executeRecipe(recipe);
-        } catch (e) {
-          log.error(`Could not execute recipe ${recipe.name}:`, e);
+        const manager = actionSandboxManagers[recipe.action];
+        if (!manager) {
+          log.error(
+            `Could not execute recipe ${recipe.name}:`,
+            `Action ${recipe.action} is either missing or invalid.`
+          );
+        } else if (manager.disabled) {
+          log.warn(
+            `Skipping recipe ${recipe.name} because ${recipe.action} failed during pre-execution.`
+          );
+        } else {
+          try {
+            log.info(`Executing recipe "${recipe.name}" (action=${recipe.action})`);
+            await manager.runAsyncCallback("action", recipe);
+          } catch (e) {
+            log.error(`Could not execute recipe ${recipe.name}:`, e);
+          }
         }
       }
     }
-  }),
 
-  getFilterContext() {
+    // Run post-execution hooks
+    for (const [actionName, manager] of Object.entries(actionSandboxManagers)) {
+      // Skip if pre-execution failed.
+      if (manager.disabled) {
+        log.info(`Skipping post-execution hook for ${actionName} due to earlier failure.`);
+        continue;
+      }
+
+      try {
+        await manager.runAsyncCallback("postExecution");
+      } catch (err) {
+        log.info(`Could not run post-execution hook for ${actionName}:`, err.message);
+      }
+    }
+
+    // Nuke sandboxes
+    Object.values(actionSandboxManagers).forEach(manager => manager.removeHold("recipeRunner"));
+  },
+
+  async loadActionSandboxManagers() {
+    const actions = await NormandyApi.fetchActions();
+    const actionSandboxManagers = {};
+    for (const action of actions) {
+      try {
+        const implementation = await NormandyApi.fetchImplementation(action);
+        actionSandboxManagers[action.name] = new ActionSandboxManager(implementation);
+      } catch (err) {
+        log.warn(`Could not fetch implementation for ${action.name}:`, err);
+      }
+    }
+    return actionSandboxManagers;
+  },
+
+  getFilterContext(recipe) {
     return {
-      normandy: ClientEnvironment.getEnvironment(),
+      normandy: Object.assign(ClientEnvironment.getEnvironment(), {
+        recipe: {
+          id: recipe.id,
+          arguments: recipe.arguments,
+        },
+      }),
     };
   },
 
@@ -113,10 +215,10 @@ this.RecipeRunner = {
    * @return {boolean} The result of evaluating the filter, cast to a bool, or false
    *                   if an error occurred during evaluation.
    */
-  checkFilter: Task.async(function* (recipe) {
-    const context = this.getFilterContext();
+  async checkFilter(recipe) {
+    const context = this.getFilterContext(recipe);
     try {
-      const result = yield FilterExpressions.eval(recipe.filter_expression, context);
+      const result = await FilterExpressions.eval(recipe.filter_expression, context);
       return !!result;
     } catch (err) {
       log.error(`Error checking filter for "${recipe.name}"`);
@@ -124,69 +226,15 @@ this.RecipeRunner = {
       log.error(`Error: "${err}"`);
       return false;
     }
-  }),
+  },
 
   /**
-   * Execute a recipe by fetching it action and executing it.
-   * @param  {Object} recipe A recipe to execute
-   * @promise Resolves when the action has executed
+   * Clear all caches of systems used by RecipeRunner, in preparation
+   * for a clean run.
    */
-  executeRecipe: Task.async(function* (recipe) {
-    const action = yield NormandyApi.fetchAction(recipe.action);
-    const response = yield fetch(action.implementation_url);
-
-    const actionScript = yield response.text();
-    yield this.executeAction(recipe, actionScript);
-  }),
-
-  /**
-   * Execute an action in a sandbox for a specific recipe.
-   * @param  {Object} recipe A recipe to execute
-   * @param  {String} actionScript The JavaScript for the action to execute.
-   * @promise Resolves or rejects when the action has executed or failed.
-   */
-  executeAction(recipe, actionScript) {
-    return new Promise((resolve, reject) => {
-      const sandboxManager = new SandboxManager();
-      const prepScript = `
-        function registerAction(name, Action) {
-          let a = new Action(sandboxedDriver, sandboxedRecipe);
-          a.execute()
-            .then(actionFinished)
-            .catch(actionFailed);
-        };
-
-        this.window = this;
-        this.registerAction = registerAction;
-        this.setTimeout = sandboxedDriver.setTimeout;
-        this.clearTimeout = sandboxedDriver.clearTimeout;
-      `;
-
-      const driver = new NormandyDriver(sandboxManager);
-      sandboxManager.cloneIntoGlobal("sandboxedDriver", driver, {cloneFunctions: true});
-      sandboxManager.cloneIntoGlobal("sandboxedRecipe", recipe);
-
-      // Results are cloned so that they don't become inaccessible when
-      // the sandbox they came from is nuked when the hold is removed.
-      sandboxManager.addGlobal("actionFinished", result => {
-        const clonedResult = Cu.cloneInto(result, {});
-        sandboxManager.removeHold("recipeExecution");
-        resolve(clonedResult);
-      });
-      sandboxManager.addGlobal("actionFailed", err => {
-        Cu.reportError(err);
-
-        // Error objects can't be cloned, so we just copy the message
-        // (which doesn't need to be cloned) to be somewhat useful.
-        const message = err.message;
-        sandboxManager.removeHold("recipeExecution");
-        reject(new Error(message));
-      });
-
-      sandboxManager.addHold("recipeExecution");
-      sandboxManager.evalInSandbox(prepScript);
-      sandboxManager.evalInSandbox(actionScript);
-    });
+  clearCaches() {
+    ClientEnvironment.clearClassifyCache();
+    NormandyApi.clearIndexCache();
   },
 
   /**
@@ -194,18 +242,17 @@ this.RecipeRunner = {
    * API url. This is used mainly by the mock-recipe-server JS that is
    * executed in the browser console.
    */
-  testRun: Task.async(function* (baseApiUrl) {
+  async testRun(baseApiUrl) {
     const oldApiUrl = prefs.getCharPref("api_url");
     prefs.setCharPref("api_url", baseApiUrl);
 
     try {
       Storage.clearAllStorage();
-      ClientEnvironment.clearClassifyCache();
-      NormandyApi.clearIndexCache();
-      yield this.start();
+      this.clearCaches();
+      await this.run();
     } finally {
       prefs.setCharPref("api_url", oldApiUrl);
-      NormandyApi.clearIndexCache();
+      this.clearCaches();
     }
-  }),
+  },
 };

browser/extensions/shield-recipe-client/lib/Sampling.jsm

@@ -5,7 +5,6 @@
 "use strict";
 
 const {utils: Cu} = Components;
-Cu.import("resource://gre/modules/Task.jsm");
 Cu.importGlobalProperties(["crypto", "TextEncoder"]);
 
 this.EXPORTED_SYMBOLS = ["Sampling"];
@@ -73,14 +72,14 @@ this.Sampling = {
   /**
    * @promise A hash of `data`, truncated to the 12 most significant characters.
    */
-  truncatedHash: Task.async(function* (data) {
+  async truncatedHash(data) {
     const hasher = crypto.subtle;
     const input = new TextEncoder("utf-8").encode(JSON.stringify(data));
-    const hash = yield hasher.digest("SHA-256", input);
+    const hash = await hasher.digest("SHA-256", input);
     // truncate hash to 12 characters (2^48), because the full hash is larger
     // than JS can meaningfully represent as a number.
     return Sampling.bufferToHex(hash).slice(0, 12);
-  }),
+  },
 
   /**
    * Sample by splitting the input into two buckets, one with a size (rate) and
@@ -92,12 +91,12 @@ this.Sampling = {
    *                           0.25 returns true 25% of the time.
    * @promises {boolean} True if the input is in the sample.
    */
-  stableSample: Task.async(function* (input, rate) {
-    const inputHash = yield Sampling.truncatedHash(input);
+  async stableSample(input, rate) {
+    const inputHash = await Sampling.truncatedHash(input);
     const samplePoint = Sampling.fractionToKey(rate);
 
     return inputHash < samplePoint;
-  }),
+  },
 
   /**
    * Sample by splitting the input space into a series of buckets, and checking
@@ -114,8 +113,8 @@ this.Sampling = {
    * @param {integer}    total Total number of buckets to group inputs into.
    * @promises {boolean} True if the given input is within the range of buckets
    *                     we're checking. */
-  bucketSample: Task.async(function* (input, start, count, total) {
-    const inputHash = yield Sampling.truncatedHash(input);
+  async bucketSample(input, start, count, total) {
+    const inputHash = await Sampling.truncatedHash(input);
     const wrappedStart = start % total;
     const end = wrappedStart + count;
 
@@ -129,5 +128,46 @@ this.Sampling = {
     }
 
     return Sampling.isHashInBucket(inputHash, wrappedStart, end, total);
-  }),
+  },
+
+  /**
+   * Sample over a list of ratios such that, over the input space, each ratio
+   * has a number of matches in correct proportion to the other ratios.
+   *
+   * For example, given the ratios:
+   *
+   * [1, 2, 3, 4]
+   *
+   * 10% of all inputs will return 0, 20% of all inputs will return 1, 30% will
+   * return 2, and 40% will return 3. You can determine the percent of inputs
+   * that will return an index by dividing the ratio by the sum of all ratios
+   * passed in. In the case above, 4 / (1 + 2 + 3 + 4) == 0.4, or 40% of the
+   * inputs.
+   *
+   * @param {object} input
+   * @param {Array<integer>} ratios
+   * @promises {integer}
+   *   Index of the ratio that matched the input
+   * @rejects {Error}
+   *   If the list of ratios doesn't have at least one element
+   */
+  async ratioSample(input, ratios) {
+    if (ratios.length < 1) {
+      throw new Error(`ratios must be at least 1 element long (got length: ${ratios.length})`);
+    }
+
+    const inputHash = await Sampling.truncatedHash(input);
+    const ratioTotal = ratios.reduce((acc, ratio) => acc + ratio);
+
+    let samplePoint = 0;
+    for (let k = 0; k < ratios.length - 1; k++) {
+      samplePoint += ratios[k];
+      if (inputHash <= Sampling.fractionToKey(samplePoint / ratioTotal)) {
+        return k;
+      }
+    }
+
+    // No need to check the last bucket if the others didn't match.
+    return ratios.length - 1;
+  },
 };

browser/extensions/shield-recipe-client/lib/SandboxManager.jsm

@@ -3,9 +3,20 @@ Cu.import("resource://gre/modules/Services.jsm");
 
 this.EXPORTED_SYMBOLS = ["SandboxManager"];
 
+/**
+ * A wrapper class with helper methods for manipulating a sandbox.
+ *
+ * Along with convenient utility methods, SandboxManagers maintain a list of
+ * "holds", which prevent the sandbox from being nuked until all registered
+ * holds are removed. This allows sandboxes to trigger async operations and
+ * automatically nuke themselves when they're done.
+ */
 this.SandboxManager = class {
   constructor() {
-    this._sandbox = makeSandbox();
+    this._sandbox = new Cu.Sandbox(null, {
+      wantComponents: false,
+      wantGlobalProperties: ["URL", "URLSearchParams"],
+    });
     this.holds = [];
   }
 
@@ -65,14 +76,46 @@ this.SandboxManager = class {
       }
     });
   }
+
+  /**
+   * Wraps a function that returns a Promise from a privileged (i.e. chrome)
+   * context and returns a Promise from this SandboxManager's sandbox. Useful
+   * for exposing privileged functions to the sandbox, since the sandbox can't
+   * access properties on privileged objects, e.g. Promise.then on a privileged
+   * Promise.
+   * @param {Function} wrappedFunction
+   * @param {Object} [options]
+   * @param {boolean} [options.cloneInto=false]
+   *   If true, the value resolved by the privileged Promise is cloned into the
+   *   sandbox before being resolved by the sandbox Promise. Without this, the
+   *   result will be Xray-wrapped.
+   * @param {boolean} [options.cloneArguments=false]
+   *   If true, the arguments passed to wrappedFunction will be cloned into the
+   *   privileged chrome context. If wrappedFunction holds a reference to any of
+   *   its arguments, you will need this to avoid losing access to the arguments
+   *   when the sandbox they originate from is nuked.
+   * @return {Function}
+   */
+  wrapAsync(wrappedFunction, options = {cloneInto: false, cloneArguments: false}) {
+    // In order for `this` to work in wrapped functions, we must return a
+    // non-arrow function, which requires saving a reference to the manager.
+    const sandboxManager = this;
+    return function(...args) {
+      return new sandboxManager.sandbox.Promise((resolve, reject) => {
+        if (options.cloneArguments) {
+          args = Cu.cloneInto(args, {});
+        }
+
+        wrappedFunction.apply(this, args).then(result => {
+          if (options.cloneInto) {
+            result = sandboxManager.cloneInto(result);
+          }
+
+          resolve(result);
+        }, err => {
+          reject(new sandboxManager.sandbox.Error(err.message, err.fileName, err.lineNumber));
+        });
+      });
+    };
+  }
 };
-
-
-function makeSandbox() {
-  const sandbox = new Cu.Sandbox(null, {
-    wantComponents: false,
-    wantGlobalProperties: ["URL", "URLSearchParams"],
-  });
-
-  return sandbox;
-}

browser/extensions/shield-recipe-client/lib/ShieldRecipeClient.jsm

@@ -0,0 +1,105 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const {utils: Cu} = Components;
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "LogManager",
+  "resource://shield-recipe-client/lib/LogManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "RecipeRunner",
+  "resource://shield-recipe-client/lib/RecipeRunner.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "CleanupManager",
+  "resource://shield-recipe-client/lib/CleanupManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "PreferenceExperiments",
+  "resource://shield-recipe-client/lib/PreferenceExperiments.jsm");
+
+this.EXPORTED_SYMBOLS = ["ShieldRecipeClient"];
+
+const REASONS = {
+  APP_STARTUP: 1,      // The application is starting up.
+  APP_SHUTDOWN: 2,     // The application is shutting down.
+  ADDON_ENABLE: 3,     // The add-on is being enabled.
+  ADDON_DISABLE: 4,    // The add-on is being disabled. (Also sent during uninstallation)
+  ADDON_INSTALL: 5,    // The add-on is being installed.
+  ADDON_UNINSTALL: 6,  // The add-on is being uninstalled.
+  ADDON_UPGRADE: 7,    // The add-on is being upgraded.
+  ADDON_DOWNGRADE: 8,  // The add-on is being downgraded.
+};
+const PREF_BRANCH = "extensions.shield-recipe-client.";
+const DEFAULT_PREFS = {
+  api_url: "https://normandy.cdn.mozilla.net/api/v1",
+  dev_mode: false,
+  enabled: true,
+  startup_delay_seconds: 300,
+  "logging.level": Log.Level.Warn,
+  user_id: "",
+  run_interval_seconds: 86400, // 24 hours
+};
+const PREF_DEV_MODE = "extensions.shield-recipe-client.dev_mode";
+const PREF_SELF_SUPPORT_ENABLED = "browser.selfsupport.enabled";
+const PREF_LOGGING_LEVEL = PREF_BRANCH + "logging.level";
+
+let log = null;
+
+/**
+ * Handles startup and shutdown of the entire add-on. Bootsrap.js defers to this
+ * module for most tasks so that we can more easily test startup and shutdown
+ * (bootstrap.js is difficult to import in tests).
+ */
+this.ShieldRecipeClient = {
+  async startup() {
+    ShieldRecipeClient.setDefaultPrefs();
+
+    // Setup logging and listen for changes to logging prefs
+    LogManager.configure(Services.prefs.getIntPref(PREF_LOGGING_LEVEL));
+    Preferences.observe(PREF_LOGGING_LEVEL, LogManager.configure);
+    CleanupManager.addCleanupHandler(
+      () => Preferences.ignore(PREF_LOGGING_LEVEL, LogManager.configure),
+    );
+    log = LogManager.getLogger("bootstrap");
+
+    // Disable self-support, since we replace its behavior.
+    // Self-support only checks its pref on start, so if we disable it, wait
+    // until next startup to run, unless the dev_mode preference is set.
+    if (Preferences.get(PREF_SELF_SUPPORT_ENABLED, true)) {
+      Preferences.set(PREF_SELF_SUPPORT_ENABLED, false);
+      if (!Preferences.get(PREF_DEV_MODE, false)) {
+        return;
+      }
+    }
+
+    // Initialize experiments first to avoid a race between initializing prefs
+    // and recipes rolling back pref changes when experiments end.
+    try {
+      await PreferenceExperiments.init();
+    } catch (err) {
+      log.error("Failed to initialize preference experiments:", err);
+    }
+
+    await RecipeRunner.init();
+  },
+
+  shutdown(reason) {
+    CleanupManager.cleanup();
+
+    // Re-enable self-support if we're being disabled.
+    if (reason === REASONS.ADDON_DISABLE || reason === REASONS.ADDON_UNINSTALL) {
+      Services.prefs.setBoolPref(PREF_SELF_SUPPORT_ENABLED, true);
+    }
+  },
+
+  setDefaultPrefs() {
+    for (const [key, val] of Object.entries(DEFAULT_PREFS)) {
+      const fullKey = PREF_BRANCH + key;
+      // If someone beat us to setting a default, don't overwrite it.
+      if (!Preferences.isSet(fullKey)) {
+        Preferences.set(fullKey, val);
+      }
+    }
+  },
+};

browser/extensions/shield-recipe-client/lib/Storage.jsm

@@ -10,7 +10,6 @@ Cu.import("resource://shield-recipe-client/lib/LogManager.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "JSONFile", "resource://gre/modules/JSONFile.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
-XPCOMUtils.defineLazyModuleGetter(this, "Task", "resource://gre/modules/Task.jsm");
 
 this.EXPORTED_SYMBOLS = ["Storage"];
 
@@ -21,10 +20,10 @@ function loadStorage() {
   if (storePromise === undefined) {
     const path = OS.Path.join(OS.Constants.Path.profileDir, "shield-recipe-client.json");
     const storage = new JSONFile({path});
-    storePromise = Task.spawn(function* () {
-      yield storage.load();
+    storePromise = (async function() {
+      await storage.load();
       return storage;
-    });
+    })();
   }
   return storePromise;
 }
@@ -70,7 +69,7 @@ this.Storage = {
               if (!(prefix in store.data)) {
                 store.data[prefix] = {};
               }
-              store.data[prefix][keySuffix] = value;
+              store.data[prefix][keySuffix] = Cu.cloneInto(value, {});
               store.saveSoon();
               resolve();
             })

browser/extensions/shield-recipe-client/lib/Utils.jsm

@@ -0,0 +1,37 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {utils: Cu} = Components;
+Cu.import("resource://shield-recipe-client/lib/LogManager.jsm");
+
+this.EXPORTED_SYMBOLS = ["Utils"];
+
+const log = LogManager.getLogger("utils");
+
+this.Utils = {
+  /**
+   * Convert an array of objects to an object. Each item is keyed by the value
+   * of the given key on the item.
+   *
+   * > list = [{foo: "bar"}, {foo: "baz"}]
+   * > keyBy(list, "foo") == {bar: {foo: "bar"}, baz: {foo: "baz"}}
+   *
+   * @param  {Array} list
+   * @param  {String} key
+   * @return {Object}
+   */
+  keyBy(list, key) {
+    return list.reduce((map, item) => {
+      if (!(key in item)) {
+        log.warn(`Skipping list due to missing key "${key}".`);
+        return map;
+      }
+
+      map[item[key]] = item;
+      return map;
+    }, {});
+  },
+};

browser/extensions/shield-recipe-client/moz.build

@@ -4,9 +4,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-with Files("**"):
-    BUG_COMPONENT = ("Shield", "Add-on")
-
 DEFINES['MOZ_APP_VERSION'] = CONFIG['MOZ_APP_VERSION']
 DEFINES['MOZ_APP_MAXVERSION'] = CONFIG['MOZ_APP_MAXVERSION']
 

browser/extensions/shield-recipe-client/test/.eslintrc.js

@@ -5,7 +5,8 @@ module.exports = {
     Assert: false,
     add_task: false,
     getRootDirectory: false,
-    gTestPath: false
+    gTestPath: false,
+    Cu: false,
   },
   rules: {
     "spaced-comment": 2,

browser/extensions/shield-recipe-client/test/browser/.eslintrc.js

@@ -17,5 +17,7 @@ module.exports = {
     UUID_REGEX: false,
     withSandboxManager: false,
     withDriver: false,
+    withMockNormandyApi: false,
+    withMockPreferences: false,
   },
 };

browser/extensions/shield-recipe-client/test/browser/action_server.sjs

@@ -0,0 +1,10 @@
+// Returns JS for an action, regardless of the URL.
+function handleRequest(request, response) {
+  // Allow cross-origin, so you can XHR to it!
+  response.setHeader("Access-Control-Allow-Origin", "*", false);
+  // Avoid confusing cache behaviors
+  response.setHeader("Cache-Control", "no-cache", false);
+
+  // Write response body
+  response.write('registerAsyncCallback("action", async () => {});');
+}

browser/extensions/shield-recipe-client/test/browser/browser.ini

@@ -6,5 +6,9 @@ head = head.js
 [browser_Storage.js]
 [browser_Heartbeat.js]
 [browser_RecipeRunner.js]
+support-files =
+  action_server.sjs
 [browser_LogManager.js]
 [browser_ClientEnvironment.js]
+[browser_ShieldRecipeClient.js]
+[browser_PreferenceExperiments.js]

browser/extensions/shield-recipe-client/test/browser/browser_ClientEnvironment.js

@@ -3,69 +3,68 @@
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://gre/modules/TelemetryController.jsm", this);
-Cu.import("resource://gre/modules/Task.jsm", this);
 Cu.import("resource://shield-recipe-client/lib/ClientEnvironment.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/PreferenceExperiments.jsm", this);
 
 
-add_task(function* testTelemetry() {
+add_task(async function testTelemetry() {
   // setup
-  yield TelemetryController.submitExternalPing("testfoo", {foo: 1});
-  yield TelemetryController.submitExternalPing("testbar", {bar: 2});
+  await TelemetryController.submitExternalPing("testfoo", {foo: 1});
+  await TelemetryController.submitExternalPing("testbar", {bar: 2});
   const environment = ClientEnvironment.getEnvironment();
 
   // Test it can access telemetry
-  const telemetry = yield environment.telemetry;
-  is(typeof telemetry, "object", "Telemetry is accessible");
+  const telemetry = await environment.telemetry;
+  is(typeof telemetry, "object", "Telemetry is accesible");
 
   // Test it reads different types of telemetry
   is(telemetry.testfoo.payload.foo, 1, "value 'foo' is in mock telemetry");
   is(telemetry.testbar.payload.bar, 2, "value 'bar' is in mock telemetry");
 });
 
-add_task(function* testUserId() {
+add_task(async function testUserId() {
   let environment = ClientEnvironment.getEnvironment();
 
   // Test that userId is available
   ok(UUID_REGEX.test(environment.userId), "userId available");
 
   // test that it pulls from the right preference
-  yield SpecialPowers.pushPrefEnv({set: [["extensions.shield-recipe-client.user_id", "fake id"]]});
+  await SpecialPowers.pushPrefEnv({set: [["extensions.shield-recipe-client.user_id", "fake id"]]});
   environment = ClientEnvironment.getEnvironment();
   is(environment.userId, "fake id", "userId is pulled from preferences");
 });
 
-add_task(function* testDistribution() {
+add_task(async function testDistribution() {
   let environment = ClientEnvironment.getEnvironment();
 
   // distribution id defaults to "default"
   is(environment.distribution, "default", "distribution has a default value");
 
   // distribution id is read from a preference
-  yield SpecialPowers.pushPrefEnv({set: [["distribution.id", "funnelcake"]]});
+  await SpecialPowers.pushPrefEnv({set: [["distribution.id", "funnelcake"]]});
   environment = ClientEnvironment.getEnvironment();
   is(environment.distribution, "funnelcake", "distribution is read from preferences");
 });
 
 const mockClassify = {country: "FR", request_time: new Date(2017, 1, 1)};
-add_task(ClientEnvironment.withMockClassify(mockClassify, function* testCountryRequestTime() {
+add_task(ClientEnvironment.withMockClassify(mockClassify, async function testCountryRequestTime() {
   const environment = ClientEnvironment.getEnvironment();
 
   // Test that country and request_time pull their data from the server.
-  is(yield environment.country, mockClassify.country, "country is read from the server API");
+  is(await environment.country, mockClassify.country, "country is read from the server API");
   is(
-    yield environment.request_time, mockClassify.request_time,
+    await environment.request_time, mockClassify.request_time,
     "request_time is read from the server API"
   );
 }));
 
-add_task(function* testSync() {
+add_task(async function testSync() {
   let environment = ClientEnvironment.getEnvironment();
   is(environment.syncMobileDevices, 0, "syncMobileDevices defaults to zero");
   is(environment.syncDesktopDevices, 0, "syncDesktopDevices defaults to zero");
   is(environment.syncTotalDevices, 0, "syncTotalDevices defaults to zero");
-  yield SpecialPowers.pushPrefEnv({
+  await SpecialPowers.pushPrefEnv({
     set: [
-      ["services.sync.numClients", 9],
       ["services.sync.clients.devices.mobile", 5],
       ["services.sync.clients.devices.desktop", 4],
     ],
@@ -76,14 +75,40 @@ add_task(function* testSync() {
   is(environment.syncTotalDevices, 9, "syncTotalDevices is read when set");
 });
 
-add_task(function* testDoNotTrack() {
+add_task(async function testDoNotTrack() {
   let environment = ClientEnvironment.getEnvironment();
 
   // doNotTrack defaults to false
   ok(!environment.doNotTrack, "doNotTrack has a default value");
 
   // doNotTrack is read from a preference
-  yield SpecialPowers.pushPrefEnv({set: [["privacy.donottrackheader.enabled", true]]});
+  await SpecialPowers.pushPrefEnv({set: [["privacy.donottrackheader.enabled", true]]});
   environment = ClientEnvironment.getEnvironment();
   ok(environment.doNotTrack, "doNotTrack is read from preferences");
 });
+
+add_task(async function testExperiments() {
+  const active = {name: "active", expired: false};
+  const expired = {name: "expired", expired: true};
+  const getAll = sinon.stub(PreferenceExperiments, "getAll", async () => [active, expired]);
+
+  const environment = ClientEnvironment.getEnvironment();
+  const experiments = await environment.experiments;
+  Assert.deepEqual(
+    experiments.all,
+    ["active", "expired"],
+    "experiments.all returns all stored experiment names",
+  );
+  Assert.deepEqual(
+    experiments.active,
+    ["active"],
+    "experiments.active returns all active experiment names",
+  );
+  Assert.deepEqual(
+    experiments.expired,
+    ["expired"],
+    "experiments.expired returns all expired experiment names",
+  );
+
+  getAll.restore();
+});

browser/extensions/shield-recipe-client/test/browser/browser_EventEmitter.js

@@ -26,7 +26,7 @@ function listenerC(x = 1) {
   evidence.log += "c";
 }
 
-add_task(withSandboxManager(Assert, function* (sandboxManager) {
+add_task(withSandboxManager(Assert, async function(sandboxManager) {
   const eventEmitter = new EventEmitter(sandboxManager);
 
   // Fire an unrelated event, to make sure nothing goes wrong
@@ -51,7 +51,7 @@ add_task(withSandboxManager(Assert, function* (sandboxManager) {
   }, "events are fired async");
 
   // Spin the event loop to run events, so we can safely "off"
-  yield Promise.resolve();
+  await Promise.resolve();
 
   // Check intermediate event results
   Assert.deepEqual(evidence, {
@@ -69,7 +69,7 @@ add_task(withSandboxManager(Assert, function* (sandboxManager) {
   eventEmitter.on("nothing");
 
   // Spin the event loop to run events
-  yield Promise.resolve();
+  await Promise.resolve();
 
   Assert.deepEqual(evidence, {
     a: 111,
@@ -91,13 +91,13 @@ add_task(withSandboxManager(Assert, function* (sandboxManager) {
 
   const data = {count: 0};
   eventEmitter.emit("mutationTest", data);
-  yield Promise.resolve();
+  await Promise.resolve();
 
   is(handlerRunCount, 2, "Mutation handler was executed twice.");
   is(data.count, 0, "Event data cannot be mutated by handlers.");
 }));
 
-add_task(withSandboxManager(Assert, function* sandboxedEmitter(sandboxManager) {
+add_task(withSandboxManager(Assert, async function sandboxedEmitter(sandboxManager) {
   const eventEmitter = new EventEmitter(sandboxManager);
 
   // Event handlers inside the sandbox should be run in response to
@@ -117,7 +117,7 @@ add_task(withSandboxManager(Assert, function* sandboxedEmitter(sandboxManager) {
   eventEmitter.emit("event", 10);
   eventEmitter.emit("eventOnce", 5);
   eventEmitter.emit("eventOnce", 10);
-  yield Promise.resolve();
+  await Promise.resolve();
 
   const eventCounts = sandboxManager.evalInSandbox("this.eventCounts");
   Assert.deepEqual(eventCounts, {

browser/extensions/shield-recipe-client/test/browser/browser_FilterExpressions.js

@@ -2,53 +2,92 @@
 
 Cu.import("resource://shield-recipe-client/lib/FilterExpressions.jsm", this);
 
-add_task(function* () {
+// Basic JEXL tests
+add_task(async function() {
   let val;
   // Test that basic expressions work
-  val = yield FilterExpressions.eval("2+2");
+  val = await FilterExpressions.eval("2+2");
   is(val, 4, "basic expression works");
 
   // Test that multiline expressions work
-  val = yield FilterExpressions.eval(`
+  val = await FilterExpressions.eval(`
     2
     +
     2
   `);
   is(val, 4, "multiline expression works");
 
+  // Test that it reads from the context correctly.
+  val = await FilterExpressions.eval("first + second + 3", {first: 1, second: 2});
+  is(val, 6, "context is available to filter expressions");
+});
+
+// Date tests
+add_task(async function() {
+  let val;
   // Test has a date transform
-  val = yield FilterExpressions.eval('"2016-04-22"|date');
+  val = await FilterExpressions.eval('"2016-04-22"|date');
   const d = new Date(Date.UTC(2016, 3, 22)); // months are 0 based
   is(val.toString(), d.toString(), "Date transform works");
 
   // Test dates are comparable
   const context = {someTime: Date.UTC(2016, 0, 1)};
-  val = yield FilterExpressions.eval('"2015-01-01"|date < someTime', context);
+  val = await FilterExpressions.eval('"2015-01-01"|date < someTime', context);
   ok(val, "dates are comparable with less-than");
-  val = yield FilterExpressions.eval('"2017-01-01"|date > someTime', context);
+  val = await FilterExpressions.eval('"2017-01-01"|date > someTime', context);
   ok(val, "dates are comparable with greater-than");
+});
+
+// Sampling tests
+add_task(async function() {
+  let val;
+  // Test stable sample returns true for matching samples
+  val = await FilterExpressions.eval('["test"]|stableSample(1)');
+  ok(val, "Stable sample returns true for 100% sample");
 
   // Test stable sample returns true for matching samples
-  val = yield FilterExpressions.eval('["test"]|stableSample(1)');
-  is(val, true, "Stable sample returns true for 100% sample");
-
-  // Test stable sample returns true for matching samples
-  val = yield FilterExpressions.eval('["test"]|stableSample(0)');
-  is(val, false, "Stable sample returns false for 0% sample");
+  val = await FilterExpressions.eval('["test"]|stableSample(0)');
+  ok(!val, "Stable sample returns false for 0% sample");
 
   // Test stable sample for known samples
-  val = yield FilterExpressions.eval('["test-1"]|stableSample(0.5)');
-  is(val, true, "Stable sample returns true for a known sample");
-  val = yield FilterExpressions.eval('["test-4"]|stableSample(0.5)');
-  is(val, false, "Stable sample returns false for a known sample");
+  val = await FilterExpressions.eval('["test-1"]|stableSample(0.5)');
+  ok(val, "Stable sample returns true for a known sample");
+  val = await FilterExpressions.eval('["test-4"]|stableSample(0.5)');
+  ok(!val, "Stable sample returns false for a known sample");
 
   // Test bucket sample for known samples
-  val = yield FilterExpressions.eval('["test-1"]|bucketSample(0, 5, 10)');
-  is(val, true, "Bucket sample returns true for a known sample");
-  val = yield FilterExpressions.eval('["test-4"]|bucketSample(0, 5, 10)');
-  is(val, false, "Bucket sample returns false for a known sample");
-
-  // Test that it reads from the context correctly.
-  val = yield FilterExpressions.eval("first + second + 3", {first: 1, second: 2});
-  is(val, 6, "context is available to filter expressions");
+  val = await FilterExpressions.eval('["test-1"]|bucketSample(0, 5, 10)');
+  ok(val, "Bucket sample returns true for a known sample");
+  val = await FilterExpressions.eval('["test-4"]|bucketSample(0, 5, 10)');
+  ok(!val, "Bucket sample returns false for a known sample");
+});
+
+// Preference tests
+add_task(async function() {
+  let val;
+  // Compare the value of the preference
+  await SpecialPowers.pushPrefEnv({set: [["normandy.test.value", 3]]});
+  val = await FilterExpressions.eval('"normandy.test.value"|preferenceValue == 3');
+  ok(val, "preferenceValue expression compares against preference values");
+  val = await FilterExpressions.eval('"normandy.test.value"|preferenceValue == "test"');
+  ok(!val, "preferenceValue expression fails value checks appropriately");
+
+  // preferenceValue can take a default value as an optional argument, which
+  // defaults to `undefined`.
+  val = await FilterExpressions.eval('"normandy.test.default"|preferenceValue(false) == false');
+  ok(val, "preferenceValue takes optional 'default value' param for prefs without set values");
+  val = await FilterExpressions.eval('"normandy.test.value"|preferenceValue(5) == 5');
+  ok(!val, "preferenceValue default param is not returned for prefs with set values");
+
+  // Compare if the preference is user set
+  val = await FilterExpressions.eval('"normandy.test.isSet"|preferenceIsUserSet == true');
+  ok(!val, "preferenceIsUserSet expression determines if preference is set at all");
+  val = await FilterExpressions.eval('"normandy.test.value"|preferenceIsUserSet == true');
+  ok(val, "preferenceIsUserSet expression determines if user's preference has been set");
+
+  // Compare if the preference has _any_ value, whether it's user-set or default,
+  val = await FilterExpressions.eval('"normandy.test.nonexistant"|preferenceExists == true');
+  ok(!val, "preferenceExists expression determines if preference exists at all");
+  val = await FilterExpressions.eval('"normandy.test.value"|preferenceExists == true');
+  ok(val, "preferenceExists expression fails existence check appropriately");
 });

browser/extensions/shield-recipe-client/test/browser/browser_Heartbeat.js

@@ -77,7 +77,7 @@ sandboxManager.addHold("test running");
 // into three batches.
 
 /* Batch #1 - General UI, Stars, and telemetry data */
-add_task(function* () {
+add_task(async function() {
   const targetWindow = Services.wm.getMostRecentWindow("navigator:browser");
   const notificationBox = targetWindow.document.querySelector("#high-priority-global-notificationbox");
 
@@ -104,22 +104,22 @@ add_task(function* () {
   // Check that when clicking the learn more link, a tab opens with the right URL
   const tabOpenPromise = BrowserTestUtils.waitForNewTab(targetWindow.gBrowser);
   learnMoreEl.click();
-  const tab = yield tabOpenPromise;
-  const tabUrl = yield BrowserTestUtils.browserLoaded(
+  const tab = await tabOpenPromise;
+  const tabUrl = await BrowserTestUtils.browserLoaded(
     tab.linkedBrowser, true, url => url && url !== "about:blank");
 
   Assert.equal(tabUrl, "https://example.org/learnmore", "Learn more link opened the right url");
 
   const telemetrySentPromise = assertTelemetrySent(hb, ["offeredTS", "learnMoreTS", "closedTS"]);
   // Close notification to trigger telemetry to be sent
-  yield closeAllNotifications(targetWindow, notificationBox);
-  yield telemetrySentPromise;
-  yield BrowserTestUtils.removeTab(tab);
+  await closeAllNotifications(targetWindow, notificationBox);
+  await telemetrySentPromise;
+  await BrowserTestUtils.removeTab(tab);
 });
 
 
 // Batch #2 - Engagement buttons
-add_task(function* () {
+add_task(async function() {
   const targetWindow = Services.wm.getMostRecentWindow("navigator:browser");
   const notificationBox = targetWindow.document.querySelector("#high-priority-global-notificationbox");
   const hb = new Heartbeat(targetWindow, sandboxManager, {
@@ -140,22 +140,22 @@ add_task(function* () {
   const engagementEl = hb.notice.querySelector(".notification-button");
   const tabOpenPromise = BrowserTestUtils.waitForNewTab(targetWindow.gBrowser);
   engagementEl.click();
-  const tab = yield tabOpenPromise;
-  const tabUrl = yield BrowserTestUtils.browserLoaded(
+  const tab = await tabOpenPromise;
+  const tabUrl = await BrowserTestUtils.browserLoaded(
         tab.linkedBrowser, true, url => url && url !== "about:blank");
   // the postAnswer url gets query parameters appended onto the end, so use Assert.startsWith instead of Assert.equal
   Assert.ok(tabUrl.startsWith("https://example.org/postAnswer"), "Engagement button opened the right url");
 
   const telemetrySentPromise = assertTelemetrySent(hb, ["offeredTS", "engagedTS", "closedTS"]);
   // Close notification to trigger telemetry to be sent
-  yield closeAllNotifications(targetWindow, notificationBox);
-  yield telemetrySentPromise;
-  yield BrowserTestUtils.removeTab(tab);
+  await closeAllNotifications(targetWindow, notificationBox);
+  await telemetrySentPromise;
+  await BrowserTestUtils.removeTab(tab);
 });
 
 // Batch 3 - Closing the window while heartbeat is open
-add_task(function* () {
-  const targetWindow = yield BrowserTestUtils.openNewBrowserWindow();
+add_task(async function() {
+  const targetWindow = await BrowserTestUtils.openNewBrowserWindow();
 
   const hb = new Heartbeat(targetWindow, sandboxManager, {
     testing: true,
@@ -165,16 +165,16 @@ add_task(function* () {
 
   const telemetrySentPromise = assertTelemetrySent(hb, ["offeredTS", "windowClosedTS"]);
   // triggers sending ping to normandy
-  yield BrowserTestUtils.closeWindow(targetWindow);
-  yield telemetrySentPromise;
+  await BrowserTestUtils.closeWindow(targetWindow);
+  await telemetrySentPromise;
 });
 
 
 // Cleanup
-add_task(function* () {
+add_task(async function() {
   // Make sure the sandbox is clean.
   sandboxManager.removeHold("test running");
-  yield sandboxManager.isNuked()
+  await sandboxManager.isNuked()
     .then(() => ok(true, "sandbox is nuked"))
     .catch(e => ok(false, "sandbox is nuked", e));
 });

browser/extensions/shield-recipe-client/test/browser/browser_LogManager.js

@@ -3,7 +3,7 @@
 Cu.import("resource://gre/modules/Log.jsm", this);
 Cu.import("resource://shield-recipe-client/lib/LogManager.jsm", this);
 
-add_task(function*() {
+add_task(async function() {
   // Ensure that configuring the logger affects all generated loggers.
   const firstLogger = LogManager.getLogger("first");
   LogManager.configure(5);
@@ -17,7 +17,7 @@ add_task(function*() {
   ok(logger.appenders.length > 0, true, "Loggers have at least one appender.");
 
   // Ensure our loggers log to the console.
-  yield new Promise(resolve => {
+  await new Promise(resolve => {
     SimpleTest.waitForExplicitFinish();
     SimpleTest.monitorConsole(resolve, [{message: /legend has it/}]);
     logger.warn("legend has it");

browser/extensions/shield-recipe-client/test/browser/browser_NormandyDriver.js

@@ -1,6 +1,6 @@
 "use strict";
 
-add_task(withDriver(Assert, function* uuids(driver) {
+add_task(withDriver(Assert, async function uuids(driver) {
   // Test that it is a UUID
   const uuid1 = driver.uuid();
   ok(UUID_REGEX.test(uuid1), "valid uuid format");
@@ -10,36 +10,35 @@ add_task(withDriver(Assert, function* uuids(driver) {
   isnot(uuid1, uuid2, "uuids are unique");
 }));
 
-add_task(withDriver(Assert, function* userId(driver) {
+add_task(withDriver(Assert, async function userId(driver) {
   // Test that userId is a UUID
   ok(UUID_REGEX.test(driver.userId), "userId is a uuid");
 }));
 
-add_task(withDriver(Assert, function* syncDeviceCounts(driver) {
-  let client = yield driver.client();
+add_task(withDriver(Assert, async function syncDeviceCounts(driver) {
+  let client = await driver.client();
   is(client.syncMobileDevices, 0, "syncMobileDevices defaults to zero");
   is(client.syncDesktopDevices, 0, "syncDesktopDevices defaults to zero");
   is(client.syncTotalDevices, 0, "syncTotalDevices defaults to zero");
 
-  yield SpecialPowers.pushPrefEnv({
+  await SpecialPowers.pushPrefEnv({
     set: [
-      ["services.sync.numClients", 9],
       ["services.sync.clients.devices.mobile", 5],
       ["services.sync.clients.devices.desktop", 4],
     ],
   });
 
-  client = yield driver.client();
+  client = await driver.client();
   is(client.syncMobileDevices, 5, "syncMobileDevices is read when set");
   is(client.syncDesktopDevices, 4, "syncDesktopDevices is read when set");
   is(client.syncTotalDevices, 9, "syncTotalDevices is read when set");
 }));
 
-add_task(withDriver(Assert, function* distribution(driver) {
-  let client = yield driver.client();
+add_task(withDriver(Assert, async function distribution(driver) {
+  let client = await driver.client();
   is(client.distribution, "default", "distribution has a default value");
 
-  yield SpecialPowers.pushPrefEnv({set: [["distribution.id", "funnelcake"]]});
-  client = yield driver.client();
+  await SpecialPowers.pushPrefEnv({set: [["distribution.id", "funnelcake"]]});
+  client = await driver.client();
   is(client.distribution, "funnelcake", "distribution is read from preferences");
 }));

browser/extensions/shield-recipe-client/test/browser/browser_PreferenceExperiments.js

@@ -0,0 +1,662 @@
+"use strict";
+
+Cu.import("resource://gre/modules/Preferences.jsm", this);
+Cu.import("resource://gre/modules/TelemetryEnvironment.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/PreferenceExperiments.jsm", this);
+
+// Save ourselves some typing
+const {withMockExperiments} = PreferenceExperiments;
+const DefaultPreferences = new Preferences({defaultBranch: true});
+
+function experimentFactory(attrs) {
+  return Object.assign({
+    name: "fakename",
+    branch: "fakebranch",
+    expired: false,
+    lastSeen: new Date().toJSON(),
+    preferenceName: "fake.preference",
+    preferenceValue: "falkevalue",
+    preferenceType: "string",
+    previousPreferenceValue: "oldfakevalue",
+    preferenceBranchType: "default",
+  }, attrs);
+}
+
+// clearAllExperimentStorage
+add_task(withMockExperiments(async function(experiments) {
+  experiments["test"] = experimentFactory({name: "test"});
+  ok(await PreferenceExperiments.has("test"), "Mock experiment is detected.");
+  await PreferenceExperiments.clearAllExperimentStorage();
+  ok(
+    !(await PreferenceExperiments.has("test")),
+    "clearAllExperimentStorage removed all stored experiments",
+  );
+}));
+
+// start should throw if an experiment with the given name already exists
+add_task(withMockExperiments(async function(experiments) {
+  experiments["test"] = experimentFactory({name: "test"});
+  await Assert.rejects(
+    PreferenceExperiments.start({
+      name: "test",
+      branch: "branch",
+      preferenceName: "fake.preference",
+      preferenceValue: "value",
+      preferenceType: "string",
+      preferenceBranchType: "default",
+    }),
+    "start threw an error due to a conflicting experiment name",
+  );
+}));
+
+// start should throw if an experiment for the given preference is active
+add_task(withMockExperiments(async function(experiments) {
+  experiments["test"] = experimentFactory({name: "test", preferenceName: "fake.preference"});
+  await Assert.rejects(
+    PreferenceExperiments.start({
+      name: "different",
+      branch: "branch",
+      preferenceName: "fake.preference",
+      preferenceValue: "value",
+      preferenceType: "string",
+      preferenceBranchType: "default",
+    }),
+    "start threw an error due to an active experiment for the given preference",
+  );
+}));
+
+// start should throw if an invalid preferenceBranchType is given
+add_task(withMockExperiments(async function() {
+  await Assert.rejects(
+    PreferenceExperiments.start({
+      name: "test",
+      branch: "branch",
+      preferenceName: "fake.preference",
+      preferenceValue: "value",
+      preferenceType: "string",
+      preferenceBranchType: "invalid",
+    }),
+    "start threw an error due to an invalid preference branch type",
+  );
+}));
+
+// start should save experiment data, modify the preference, and register a
+// watcher.
+add_task(withMockExperiments(withMockPreferences(async function(experiments, mockPreferences) {
+  const startObserver = sinon.stub(PreferenceExperiments, "startObserver");
+  mockPreferences.set("fake.preference", "oldvalue", "default");
+  mockPreferences.set("fake.preference", "uservalue", "user");
+
+  await PreferenceExperiments.start({
+    name: "test",
+    branch: "branch",
+    preferenceName: "fake.preference",
+    preferenceValue: "newvalue",
+    preferenceBranchType: "default",
+    preferenceType: "string",
+  });
+  ok("test" in experiments, "start saved the experiment");
+  ok(
+    startObserver.calledWith("test", "fake.preference", "newvalue"),
+    "start registered an observer",
+  );
+
+  const expectedExperiment = {
+    name: "test",
+    branch: "branch",
+    expired: false,
+    preferenceName: "fake.preference",
+    preferenceValue: "newvalue",
+    preferenceType: "string",
+    previousPreferenceValue: "oldvalue",
+    preferenceBranchType: "default",
+  };
+  const experiment = {};
+  Object.keys(expectedExperiment).forEach(key => experiment[key] = experiments.test[key]);
+  Assert.deepEqual(experiment, expectedExperiment, "start saved the experiment");
+
+  is(
+    DefaultPreferences.get("fake.preference"),
+    "newvalue",
+    "start modified the default preference",
+  );
+  is(
+    Preferences.get("fake.preference"),
+    "uservalue",
+    "start did not modify the user preference",
+  );
+
+  startObserver.restore();
+})));
+
+// start should modify the user preference for the user branch type
+add_task(withMockExperiments(withMockPreferences(async function(experiments, mockPreferences) {
+  const startObserver = sinon.stub(PreferenceExperiments, "startObserver");
+  mockPreferences.set("fake.preference", "oldvalue", "user");
+  mockPreferences.set("fake.preference", "olddefaultvalue", "default");
+
+  await PreferenceExperiments.start({
+    name: "test",
+    branch: "branch",
+    preferenceName: "fake.preference",
+    preferenceValue: "newvalue",
+    preferenceType: "string",
+    preferenceBranchType: "user",
+  });
+  ok(
+    startObserver.calledWith("test", "fake.preference", "newvalue"),
+    "start registered an observer",
+  );
+
+  const expectedExperiment = {
+    name: "test",
+    branch: "branch",
+    expired: false,
+    preferenceName: "fake.preference",
+    preferenceValue: "newvalue",
+    preferenceType: "string",
+    previousPreferenceValue: "oldvalue",
+    preferenceBranchType: "user",
+  };
+
+  const experiment = {};
+  Object.keys(expectedExperiment).forEach(key => experiment[key] = experiments.test[key]);
+  Assert.deepEqual(experiment, expectedExperiment, "start saved the experiment");
+
+  Assert.notEqual(
+    DefaultPreferences.get("fake.preference"),
+    "newvalue",
+    "start did not modify the default preference",
+  );
+  is(Preferences.get("fake.preference"), "newvalue", "start modified the user preference");
+
+  startObserver.restore();
+})));
+
+// start should detect if a new preference value type matches the previous value type
+add_task(withMockPreferences(async function(mockPreferences) {
+  mockPreferences.set("fake.type_preference", "oldvalue");
+
+  await Assert.rejects(
+    PreferenceExperiments.start({
+      name: "test",
+      branch: "branch",
+      preferenceName: "fake.type_preference",
+      preferenceBranchType: "user",
+      preferenceValue: 12345,
+      preferenceType: "integer",
+    }),
+    "start threw error for incompatible preference type"
+  );
+}));
+
+
+// startObserver should throw if an observer for the experiment is already
+// active.
+add_task(withMockExperiments(async function() {
+  PreferenceExperiments.startObserver("test", "fake.preference", "newvalue");
+  Assert.throws(
+    () => PreferenceExperiments.startObserver("test", "another.fake", "othervalue"),
+    "startObserver threw due to a conflicting active observer",
+  );
+  PreferenceExperiments.stopAllObservers();
+}));
+
+// startObserver should register an observer that calls stop when a preference
+// changes from its experimental value.
+add_task(withMockExperiments(withMockPreferences(async function(mockExperiments, mockPreferences) {
+  const stop = sinon.stub(PreferenceExperiments, "stop");
+  mockPreferences.set("fake.preference", "startvalue");
+
+  // NOTE: startObserver does not modify the pref
+  PreferenceExperiments.startObserver("test", "fake.preference", "experimentvalue");
+
+  // Setting it to the experimental value should not trigger the call.
+  Preferences.set("fake.preference", "experimentvalue");
+  ok(!stop.called, "Changing to the experimental pref value did not trigger the observer");
+
+  // Setting it to something different should trigger the call.
+  Preferences.set("fake.preference", "newvalue");
+  ok(stop.called, "Changing to a different value triggered the observer");
+
+  PreferenceExperiments.stopAllObservers();
+  stop.restore();
+})));
+
+add_task(withMockExperiments(async function testHasObserver() {
+  PreferenceExperiments.startObserver("test", "fake.preference", "experimentValue");
+
+  ok(await PreferenceExperiments.hasObserver("test"), "hasObserver detects active observers");
+  ok(
+    !(await PreferenceExperiments.hasObserver("missing")),
+    "hasObserver doesn't detect inactive observers",
+  );
+
+  PreferenceExperiments.stopAllObservers();
+}));
+
+// stopObserver should throw if there is no observer active for it to stop.
+add_task(withMockExperiments(async function() {
+  Assert.throws(
+    () => PreferenceExperiments.stopObserver("neveractive", "another.fake", "othervalue"),
+    "stopObserver threw because there was not matching active observer",
+  );
+}));
+
+// stopObserver should cancel an active observer.
+add_task(withMockExperiments(withMockPreferences(async function(mockExperiments, mockPreferences) {
+  const stop = sinon.stub(PreferenceExperiments, "stop");
+  mockPreferences.set("fake.preference", "startvalue");
+
+  PreferenceExperiments.startObserver("test", "fake.preference", "experimentvalue");
+  PreferenceExperiments.stopObserver("test");
+
+  // Setting the preference now that the observer is stopped should not call
+  // stop.
+  Preferences.set("fake.preference", "newvalue");
+  ok(!stop.called, "stopObserver successfully removed the observer");
+
+  // Now that the observer is stopped, start should be able to start a new one
+  // without throwing.
+  try {
+    PreferenceExperiments.startObserver("test", "fake.preference", "experimentvalue");
+  } catch (err) {
+    ok(false, "startObserver did not throw an error for an observer that was already stopped");
+  }
+
+  PreferenceExperiments.stopAllObservers();
+  stop.restore();
+})));
+
+// stopAllObservers
+add_task(withMockExperiments(withMockPreferences(async function(mockExperiments, mockPreferences) {
+  const stop = sinon.stub(PreferenceExperiments, "stop");
+  mockPreferences.set("fake.preference", "startvalue");
+  mockPreferences.set("other.fake.preference", "startvalue");
+
+  PreferenceExperiments.startObserver("test", "fake.preference", "experimentvalue");
+  PreferenceExperiments.startObserver("test2", "other.fake.preference", "experimentvalue");
+  PreferenceExperiments.stopAllObservers();
+
+  // Setting the preference now that the observers are stopped should not call
+  // stop.
+  Preferences.set("fake.preference", "newvalue");
+  Preferences.set("other.fake.preference", "newvalue");
+  ok(!stop.called, "stopAllObservers successfully removed all observers");
+
+  // Now that the observers are stopped, start should be able to start new
+  // observers without throwing.
+  try {
+    PreferenceExperiments.startObserver("test", "fake.preference", "experimentvalue");
+    PreferenceExperiments.startObserver("test2", "other.fake.preference", "experimentvalue");
+  } catch (err) {
+    ok(false, "startObserver did not throw an error for an observer that was already stopped");
+  }
+
+  PreferenceExperiments.stopAllObservers();
+  stop.restore();
+})));
+
+// markLastSeen should throw if it can't find a matching experiment
+add_task(withMockExperiments(async function() {
+  await Assert.rejects(
+    PreferenceExperiments.markLastSeen("neveractive"),
+    "markLastSeen threw because there was not a matching experiment",
+  );
+}));
+
+// markLastSeen should update the lastSeen date
+add_task(withMockExperiments(async function(experiments) {
+  const oldDate = new Date(1988, 10, 1).toJSON();
+  experiments["test"] = experimentFactory({name: "test", lastSeen: oldDate});
+  await PreferenceExperiments.markLastSeen("test");
+  Assert.notEqual(
+    experiments["test"].lastSeen,
+    oldDate,
+    "markLastSeen updated the experiment lastSeen date",
+  );
+}));
+
+// stop should throw if an experiment with the given name doesn't exist
+add_task(withMockExperiments(async function() {
+  await Assert.rejects(
+    PreferenceExperiments.stop("test"),
+    "stop threw an error because there are no experiments with the given name",
+  );
+}));
+
+// stop should throw if the experiment is already expired
+add_task(withMockExperiments(async function(experiments) {
+  experiments["test"] = experimentFactory({name: "test", expired: true});
+  await Assert.rejects(
+    PreferenceExperiments.stop("test"),
+    "stop threw an error because the experiment was already expired",
+  );
+}));
+
+// stop should mark the experiment as expired, stop its observer, and revert the
+// preference value.
+add_task(withMockExperiments(withMockPreferences(async function(experiments, mockPreferences) {
+  const stopObserver = sinon.spy(PreferenceExperiments, "stopObserver");
+
+  mockPreferences.set("fake.preference", "experimentvalue", "default");
+  experiments["test"] = experimentFactory({
+    name: "test",
+    expired: false,
+    preferenceName: "fake.preference",
+    preferenceValue: "experimentvalue",
+    preferenceType: "string",
+    previousPreferenceValue: "oldvalue",
+    preferenceBranchType: "default",
+  });
+  PreferenceExperiments.startObserver("test", "fake.preference", "experimentvalue");
+
+  await PreferenceExperiments.stop("test");
+  ok(stopObserver.calledWith("test"), "stop removed an observer");
+  is(experiments["test"].expired, true, "stop marked the experiment as expired");
+  is(
+    DefaultPreferences.get("fake.preference"),
+    "oldvalue",
+    "stop reverted the preference to its previous value",
+  );
+
+  stopObserver.restore();
+  PreferenceExperiments.stopAllObservers();
+})));
+
+// stop should also support user pref experiments
+add_task(withMockExperiments(withMockPreferences(async function(experiments, mockPreferences) {
+  const stopObserver = sinon.stub(PreferenceExperiments, "stopObserver");
+  mockPreferences.set("fake.preference", "experimentvalue", "user");
+  experiments["test"] = experimentFactory({
+    name: "test",
+    expired: false,
+    preferenceName: "fake.preference",
+    preferenceValue: "experimentvalue",
+    preferenceType: "string",
+    previousPreferenceValue: "oldvalue",
+    preferenceBranchType: "user",
+  });
+  PreferenceExperiments.startObserver("test", "fake.preference", "experimentvalue");
+
+  await PreferenceExperiments.stop("test");
+  ok(stopObserver.calledWith("test"), "stop removed an observer");
+  is(experiments["test"].expired, true, "stop marked the experiment as expired");
+  is(
+    Preferences.get("fake.preference"),
+    "oldvalue",
+    "stop reverted the preference to its previous value",
+  );
+
+  stopObserver.restore();
+})));
+
+// stop should not call stopObserver if there is no observer registered.
+add_task(withMockExperiments(withMockPreferences(async function(experiments) {
+  const stopObserver = sinon.spy(PreferenceExperiments, "stopObserver");
+  experiments["test"] = experimentFactory({name: "test", expired: false});
+
+  await PreferenceExperiments.stop("test");
+  ok(!stopObserver.called, "stop did not bother to stop an observer that wasn't active");
+
+  stopObserver.restore();
+  PreferenceExperiments.stopAllObservers();
+})));
+
+// stop should remove a preference that had no value prior to an experiment for user prefs
+add_task(withMockExperiments(withMockPreferences(async function(experiments, mockPreferences) {
+  const stopObserver = sinon.stub(PreferenceExperiments, "stopObserver");
+  mockPreferences.set("fake.preference", "experimentvalue", "user");
+  experiments["test"] = experimentFactory({
+    name: "test",
+    expired: false,
+    preferenceName: "fake.preference",
+    preferenceValue: "experimentvalue",
+    preferenceType: "string",
+    previousPreferenceValue: undefined,
+    preferenceBranchType: "user",
+  });
+
+  await PreferenceExperiments.stop("test");
+  ok(
+    !Preferences.isSet("fake.preference"),
+    "stop removed the preference that had no value prior to the experiment",
+  );
+
+  stopObserver.restore();
+})));
+
+// stop should not modify a preference if resetValue is false
+add_task(withMockExperiments(withMockPreferences(async function(experiments, mockPreferences) {
+  const stopObserver = sinon.stub(PreferenceExperiments, "stopObserver");
+  mockPreferences.set("fake.preference", "customvalue", "default");
+  experiments["test"] = experimentFactory({
+    name: "test",
+    expired: false,
+    preferenceName: "fake.preference",
+    preferenceValue: "experimentvalue",
+    preferenceType: "string",
+    previousPreferenceValue: "oldvalue",
+    peferenceBranchType: "default",
+  });
+
+  await PreferenceExperiments.stop("test", false);
+  is(
+    DefaultPreferences.get("fake.preference"),
+    "customvalue",
+    "stop did not modify the preference",
+  );
+
+  stopObserver.restore();
+})));
+
+// get should throw if no experiment exists with the given name
+add_task(withMockExperiments(async function() {
+  await Assert.rejects(
+    PreferenceExperiments.get("neverexisted"),
+    "get rejects if no experiment with the given name is found",
+  );
+}));
+
+// get
+add_task(withMockExperiments(async function(experiments) {
+  const experiment = experimentFactory({name: "test"});
+  experiments["test"] = experiment;
+
+  const fetchedExperiment = await PreferenceExperiments.get("test");
+  Assert.deepEqual(fetchedExperiment, experiment, "get fetches the correct experiment");
+
+  // Modifying the fetched experiment must not edit the data source.
+  fetchedExperiment.name = "othername";
+  is(experiments["test"].name, "test", "get returns a copy of the experiment");
+}));
+
+add_task(withMockExperiments(async function testGetAll(experiments) {
+  const experiment1 = experimentFactory({name: "experiment1"});
+  const experiment2 = experimentFactory({name: "experiment2", disabled: true});
+  experiments["experiment1"] = experiment1;
+  experiments["experiment2"] = experiment2;
+
+  const fetchedExperiments = await PreferenceExperiments.getAll();
+  is(fetchedExperiments.length, 2, "getAll returns a list of all stored experiments");
+  Assert.deepEqual(
+    fetchedExperiments.find(e => e.name === "experiment1"),
+    experiment1,
+    "getAll returns a list with the correct experiments",
+  );
+  const fetchedExperiment2 = fetchedExperiments.find(e => e.name === "experiment2");
+  Assert.deepEqual(
+    fetchedExperiment2,
+    experiment2,
+    "getAll returns a list with the correct experiments, including disabled ones",
+  );
+
+  fetchedExperiment2.name = "othername";
+  is(experiment2.name, "experiment2", "getAll returns copies of the experiments");
+}));
+
+add_task(withMockExperiments(withMockPreferences(async function testGetAllActive(experiments) {
+  experiments["active"] = experimentFactory({
+    name: "active",
+    expired: false,
+  });
+  experiments["inactive"] = experimentFactory({
+    name: "inactive",
+    expired: true,
+  });
+
+  const activeExperiments = await PreferenceExperiments.getAllActive();
+  Assert.deepEqual(
+    activeExperiments,
+    [experiments["active"]],
+    "getAllActive only returns active experiments",
+  );
+
+  activeExperiments[0].name = "newfakename";
+  Assert.notEqual(
+    experiments["active"].name,
+    "newfakename",
+    "getAllActive returns copies of stored experiments",
+  );
+})));
+
+// has
+add_task(withMockExperiments(async function(experiments) {
+  experiments["test"] = experimentFactory({name: "test"});
+  ok(await PreferenceExperiments.has("test"), "has returned true for a stored experiment");
+  ok(!(await PreferenceExperiments.has("missing")), "has returned false for a missing experiment");
+}));
+
+// init should set the default preference value for active, default experiments
+add_task(withMockExperiments(withMockPreferences(async function testInit(experiments, mockPreferences) {
+  experiments["user"] = experimentFactory({
+    name: "user",
+    preferenceName: "user",
+    preferenceValue: true,
+    preferenceType: "boolean",
+    expired: false,
+    preferenceBranchType: "user",
+  });
+  experiments["default"] = experimentFactory({
+    name: "default",
+    preferenceName: "default",
+    preferenceValue: true,
+    preferenceType: "boolean",
+    expired: false,
+    preferenceBranchType: "default",
+  });
+  experiments["expireddefault"] = experimentFactory({
+    name: "expireddefault",
+    preferenceName: "expireddefault",
+    preferenceValue: true,
+    preferenceType: "boolean",
+    expired: true,
+    preferenceBranchType: "default",
+  });
+
+  for (const experiment of Object.values(experiments)) {
+    mockPreferences.set(experiment.preferenceName, false, "default");
+  }
+
+  await PreferenceExperiments.init();
+
+  is(DefaultPreferences.get("user"), false, "init ignored a user pref experiment");
+  is(
+    DefaultPreferences.get("default"),
+    true,
+    "init set the value for a default pref experiment",
+  );
+  is(
+    DefaultPreferences.get("expireddefault"),
+    false,
+    "init ignored an expired default pref experiment",
+  );
+})));
+
+// init should register telemetry experiments
+add_task(withMockExperiments(withMockPreferences(async function testInit(experiments, mockPreferences) {
+  const setActiveStub = sinon.stub(TelemetryEnvironment, "setExperimentActive");
+  const startObserverStub = sinon.stub(PreferenceExperiments, "startObserver");
+  mockPreferences.set("fake.pref", "experiment value");
+
+  experiments["test"] = experimentFactory({
+    name: "test",
+    branch: "branch",
+    preferenceName: "fake.pref",
+    preferenceValue: "experiment value",
+    expired: false,
+    preferenceBranchType: "default",
+  });
+
+  await PreferenceExperiments.init();
+  ok(setActiveStub.calledWith("test", "branch"), "Experiment is registered by init");
+  startObserverStub.restore();
+  setActiveStub.restore();
+})));
+
+// starting and stopping experiments should register in telemetry
+add_task(withMockExperiments(async function testInitTelemetry() {
+  const setActiveStub = sinon.stub(TelemetryEnvironment, "setExperimentActive");
+  const setInactiveStub = sinon.stub(TelemetryEnvironment, "setExperimentInactive");
+
+  await PreferenceExperiments.start({
+    name: "test",
+    branch: "branch",
+    preferenceName: "fake.preference",
+    preferenceValue: "value",
+    preferenceType: "string",
+    preferenceBranchType: "default",
+  });
+
+  ok(setActiveStub.calledWith("test", "branch"), "Experiment is registerd by start()");
+  await PreferenceExperiments.stop("test");
+  ok(setInactiveStub.calledWith("test", "branch"), "Experiment is unregisterd by stop()");
+
+  setActiveStub.restore();
+  setInactiveStub.restore();
+}));
+
+// Experiments shouldn't be recorded by init() in telemetry if they are expired
+add_task(withMockExperiments(async function testInitTelemetryExpired(experiments) {
+  const setActiveStub = sinon.stub(TelemetryEnvironment, "setExperimentActive");
+  experiments["experiment1"] = experimentFactory({name: "expired", branch: "branch", expired: true});
+  await PreferenceExperiments.init();
+  ok(!setActiveStub.called, "Expired experiment is not registered by init");
+  setActiveStub.restore();
+}));
+
+// Experiments should end if the preference has been changed when init() is called
+add_task(withMockExperiments(withMockPreferences(async function testInitChanges(experiments, mockPreferences) {
+  const stopStub = sinon.stub(PreferenceExperiments, "stop");
+  mockPreferences.set("fake.preference", "experiment value", "default");
+  experiments["test"] = experimentFactory({
+    name: "test",
+    preferenceName: "fake.preference",
+    preferenceValue: "experiment value",
+  });
+  mockPreferences.set("fake.preference", "changed value");
+  await PreferenceExperiments.init();
+  ok(stopStub.calledWith("test"), "Experiment is stopped because value changed");
+  ok(Preferences.get("fake.preference"), "changed value", "Preference value was not changed");
+  stopStub.restore();
+})));
+
+
+// init should register an observer for experiments
+add_task(withMockExperiments(withMockPreferences(async function testInitRegistersObserver(experiments, mockPreferences) {
+  const startObserver = sinon.stub(PreferenceExperiments, "startObserver");
+  mockPreferences.set("fake.preference", "experiment value", "default");
+  experiments["test"] = experimentFactory({
+    name: "test",
+    preferenceName: "fake.preference",
+    preferenceValue: "experiment value",
+  });
+  await PreferenceExperiments.init();
+
+  ok(
+    startObserver.calledWith("test", "fake.preference", "experiment value"),
+    "init registered an observer",
+  );
+
+  startObserver.restore();
+})));

browser/extensions/shield-recipe-client/test/browser/browser_RecipeRunner.js

@@ -1,93 +1,15 @@
 "use strict";
 
+Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://shield-recipe-client/lib/RecipeRunner.jsm", this);
 Cu.import("resource://shield-recipe-client/lib/ClientEnvironment.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/CleanupManager.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/NormandyApi.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/ActionSandboxManager.jsm", this);
 
-add_task(function* execute() {
-  // Test that RecipeRunner can execute a basic recipe/action and return
-  // the result of execute.
-  const recipe = {
-    foo: "bar",
-  };
-  const actionScript = `
-    class TestAction {
-      constructor(driver, recipe) {
-        this.recipe = recipe;
-      }
-
-      execute() {
-        return new Promise(resolve => {
-          resolve({foo: this.recipe.foo});
-        });
-      }
-    }
-
-    registerAction('test-action', TestAction);
-  `;
-
-  const result = yield RecipeRunner.executeAction(recipe, actionScript);
-  is(result.foo, "bar", "Recipe executed correctly");
-});
-
-add_task(function* error() {
-  // Test that RecipeRunner rejects with error messages from within the
-  // sandbox.
-  const actionScript = `
-    class TestAction {
-      execute() {
-        return new Promise((resolve, reject) => {
-          reject(new Error("ERROR MESSAGE"));
-        });
-      }
-    }
-
-    registerAction('test-action', TestAction);
-  `;
-
-  let gotException = false;
-  try {
-    yield RecipeRunner.executeAction({}, actionScript);
-  } catch (err) {
-    gotException = true;
-    is(err.message, "ERROR MESSAGE", "RecipeRunner throws errors from the sandbox correctly.");
-  }
-  ok(gotException, "RecipeRunner threw an error from the sandbox.");
-});
-
-add_task(function* globalObject() {
-  // Test that window is an alias for the global object, and that it
-  // has some expected functions available on it.
-  const actionScript = `
-    window.setOnWindow = "set";
-    this.setOnGlobal = "set";
-
-    class TestAction {
-      execute() {
-        return new Promise(resolve => {
-          resolve({
-            setOnWindow: setOnWindow,
-            setOnGlobal: window.setOnGlobal,
-            setTimeoutExists: setTimeout !== undefined,
-            clearTimeoutExists: clearTimeout !== undefined,
-          });
-        });
-      }
-    }
-
-    registerAction('test-action', TestAction);
-  `;
-
-  const result = yield RecipeRunner.executeAction({}, actionScript);
-  Assert.deepEqual(result, {
-    setOnWindow: "set",
-    setOnGlobal: "set",
-    setTimeoutExists: true,
-    clearTimeoutExists: true,
-  }, "sandbox.window is the global object and has expected functions.");
-});
-
-add_task(function* getFilterContext() {
-  const context = RecipeRunner.getFilterContext();
+add_task(async function getFilterContext() {
+  const recipe = {id: 17, arguments: {foo: "bar"}, unrelated: false};
+  const context = RecipeRunner.getFilterContext(recipe);
 
   // Test for expected properties in the filter expression context.
   const expectedNormandyKeys = [
@@ -98,6 +20,7 @@ add_task(function* getFilterContext() {
     "isDefaultBrowser",
     "locale",
     "plugins",
+    "recipe",
     "request_time",
     "searchEngine",
     "syncDesktopDevices",
@@ -111,38 +34,190 @@ add_task(function* getFilterContext() {
   for (const key of expectedNormandyKeys) {
     ok(key in context.normandy, `normandy.${key} is available`);
   }
+
+  is(
+    context.normandy.recipe.id,
+    recipe.id,
+    "normandy.recipe is the recipe passed to getFilterContext",
+  );
+  delete recipe.unrelated;
+  Assert.deepEqual(
+    context.normandy.recipe,
+    recipe,
+    "normandy.recipe drops unrecognized attributes from the recipe",
+  );
 });
 
-add_task(function* checkFilter() {
+add_task(async function checkFilter() {
   const check = filter => RecipeRunner.checkFilter({filter_expression: filter});
 
   // Errors must result in a false return value.
-  ok(!(yield check("invalid ( + 5yntax")), "Invalid filter expressions return false");
+  ok(!(await check("invalid ( + 5yntax")), "Invalid filter expressions return false");
 
   // Non-boolean filter results result in a true return value.
-  ok(yield check("[1, 2, 3]"), "Non-boolean filter expressions return true");
+  ok(await check("[1, 2, 3]"), "Non-boolean filter expressions return true");
+
+  // The given recipe must be available to the filter context.
+  const recipe = {filter_expression: "normandy.recipe.id == 7", id: 7};
+  ok(await RecipeRunner.checkFilter(recipe), "The recipe is available in the filter context");
+  recipe.id = 4;
+  ok(!(await RecipeRunner.checkFilter(recipe)), "The recipe is available in the filter context");
 });
 
-add_task(function* testStart() {
+add_task(withMockNormandyApi(async function testClientClassificationCache() {
   const getStub = sinon.stub(ClientEnvironment, "getClientClassification")
     .returns(Promise.resolve(false));
 
+  await SpecialPowers.pushPrefEnv({set: [
+    ["extensions.shield-recipe-client.api_url",
+     "https://example.com/selfsupport-dummy"],
+  ]});
+
   // When the experiment pref is false, eagerly call getClientClassification.
-  yield SpecialPowers.pushPrefEnv({set: [
+  await SpecialPowers.pushPrefEnv({set: [
     ["extensions.shield-recipe-client.experiments.lazy_classify", false],
   ]});
   ok(!getStub.called, "getClientClassification hasn't been called");
-  yield RecipeRunner.start();
-  ok(getStub.called, "getClientClassfication was called eagerly");
+  await RecipeRunner.run();
+  ok(getStub.called, "getClientClassification was called eagerly");
 
   // When the experiment pref is true, do not eagerly call getClientClassification.
-  yield SpecialPowers.pushPrefEnv({set: [
+  await SpecialPowers.pushPrefEnv({set: [
     ["extensions.shield-recipe-client.experiments.lazy_classify", true],
   ]});
   getStub.reset();
   ok(!getStub.called, "getClientClassification hasn't been called");
-  yield RecipeRunner.start();
-  ok(!getStub.called, "getClientClassfication was not called eagerly");
+  await RecipeRunner.run();
+  ok(!getStub.called, "getClientClassification was not called eagerly");
 
   getStub.restore();
+}));
+
+/**
+ * Mocks RecipeRunner.loadActionSandboxManagers for testing run.
+ */
+async function withMockActionSandboxManagers(actions, testFunction) {
+  const managers = {};
+  for (const action of actions) {
+    managers[action.name] = new ActionSandboxManager("");
+    sinon.stub(managers[action.name], "runAsyncCallback");
+  }
+
+  const loadActionSandboxManagers = sinon.stub(
+    RecipeRunner,
+    "loadActionSandboxManagers",
+    async () => managers,
+  );
+  await testFunction(managers);
+  loadActionSandboxManagers.restore();
+}
+
+add_task(withMockNormandyApi(async function testRun(mockApi) {
+  const matchAction = {name: "matchAction"};
+  const noMatchAction = {name: "noMatchAction"};
+  mockApi.actions = [matchAction, noMatchAction];
+
+  const matchRecipe = {action: "matchAction", filter_expression: "true"};
+  const noMatchRecipe = {action: "noMatchAction", filter_expression: "false"};
+  const missingRecipe = {action: "missingAction", filter_expression: "true"};
+  mockApi.recipes = [matchRecipe, noMatchRecipe, missingRecipe];
+
+  await withMockActionSandboxManagers(mockApi.actions, async managers => {
+    const matchManager = managers["matchAction"];
+    const noMatchManager = managers["noMatchAction"];
+
+    await RecipeRunner.run();
+
+    // match should be called for preExecution, action, and postExecution
+    sinon.assert.calledWith(matchManager.runAsyncCallback, "preExecution");
+    sinon.assert.calledWith(matchManager.runAsyncCallback, "action", matchRecipe);
+    sinon.assert.calledWith(matchManager.runAsyncCallback, "postExecution");
+
+    // noMatch should be called for preExecution and postExecution, and skipped
+    // for action since the filter expression does not match.
+    sinon.assert.calledWith(noMatchManager.runAsyncCallback, "preExecution");
+    sinon.assert.neverCalledWith(noMatchManager.runAsyncCallback, "action", noMatchRecipe);
+    sinon.assert.calledWith(noMatchManager.runAsyncCallback, "postExecution");
+
+    // missing is never called at all due to no matching action/manager.
+    await matchManager.isNuked();
+    await noMatchManager.isNuked();
+  });
+}));
+
+add_task(withMockNormandyApi(async function testRunPreExecutionFailure(mockApi) {
+  const passAction = {name: "passAction"};
+  const failAction = {name: "failAction"};
+  mockApi.actions = [passAction, failAction];
+
+  const passRecipe = {action: "passAction", filter_expression: "true"};
+  const failRecipe = {action: "failAction", filter_expression: "true"};
+  mockApi.recipes = [passRecipe, failRecipe];
+
+  await withMockActionSandboxManagers(mockApi.actions, async managers => {
+    const passManager = managers["passAction"];
+    const failManager = managers["failAction"];
+    failManager.runAsyncCallback.returns(Promise.reject(new Error("oh no")));
+
+    await RecipeRunner.run();
+
+    // pass should be called for preExecution, action, and postExecution
+    sinon.assert.calledWith(passManager.runAsyncCallback, "preExecution");
+    sinon.assert.calledWith(passManager.runAsyncCallback, "action", passRecipe);
+    sinon.assert.calledWith(passManager.runAsyncCallback, "postExecution");
+
+    // fail should only be called for preExecution, since it fails during that
+    sinon.assert.calledWith(failManager.runAsyncCallback, "preExecution");
+    sinon.assert.neverCalledWith(failManager.runAsyncCallback, "action", failRecipe);
+    sinon.assert.neverCalledWith(failManager.runAsyncCallback, "postExecution");
+
+    await passManager.isNuked();
+    await failManager.isNuked();
+  });
+}));
+
+add_task(withMockNormandyApi(async function testLoadActionSandboxManagers(mockApi) {
+  mockApi.actions = [
+    {name: "normalAction"},
+    {name: "missingImpl"},
+  ];
+  mockApi.implementations["normalAction"] = "window.scriptRan = true";
+
+  const managers = await RecipeRunner.loadActionSandboxManagers();
+  ok("normalAction" in managers, "Actions with implementations have managers");
+  ok(!("missingImpl" in managers), "Actions without implementations are skipped");
+
+  const normalManager = managers["normalAction"];
+  ok(
+    await normalManager.evalInSandbox("window.scriptRan"),
+    "Implementations are run in the sandbox",
+  );
+}));
+
+add_task(async function testStartup() {
+  const runStub = sinon.stub(RecipeRunner, "run");
+  const addCleanupHandlerStub = sinon.stub(CleanupManager, "addCleanupHandler");
+  const updateRunIntervalStub = sinon.stub(RecipeRunner, "updateRunInterval");
+
+  // in dev mode
+  await SpecialPowers.pushPrefEnv({set: [["extensions.shield-recipe-client.dev_mode", true]]});
+  RecipeRunner.init();
+  ok(runStub.called, "RecipeRunner.run is called immediately when in dev mode");
+  ok(addCleanupHandlerStub.called, "A cleanup function is registered when in dev mode");
+  ok(updateRunIntervalStub.called, "A timer is registered when in dev mode");
+
+  runStub.reset();
+  addCleanupHandlerStub.reset();
+  updateRunIntervalStub.reset();
+
+  // not in dev mode
+  await SpecialPowers.pushPrefEnv({set: [["extensions.shield-recipe-client.dev_mode", false]]});
+  RecipeRunner.init();
+  ok(!runStub.called, "RecipeRunner.run is not called immediately when not in dev mode");
+  ok(addCleanupHandlerStub.called, "A cleanup function is registered when not in dev mode");
+  ok(updateRunIntervalStub.called, "A timer is registered when not in dev mode");
+
+  runStub.restore();
+  addCleanupHandlerStub.restore();
+  updateRunIntervalStub.restore();
 });

browser/extensions/shield-recipe-client/test/browser/browser_ShieldRecipeClient.js

@@ -0,0 +1,30 @@
+"use strict";
+
+Cu.import("resource://shield-recipe-client/lib/ShieldRecipeClient.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/RecipeRunner.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/PreferenceExperiments.jsm", this);
+
+add_task(async function testStartup() {
+  sinon.stub(RecipeRunner, "init");
+  sinon.stub(PreferenceExperiments, "init");
+
+  await ShieldRecipeClient.startup();
+  ok(PreferenceExperiments.init.called, "startup calls PreferenceExperiments.init");
+  ok(RecipeRunner.init.called, "startup calls RecipeRunner.init");
+
+  PreferenceExperiments.init.restore();
+  RecipeRunner.init.restore();
+});
+
+add_task(async function testStartupPrefInitFail() {
+  sinon.stub(RecipeRunner, "init");
+  sinon.stub(PreferenceExperiments, "init").returns(Promise.reject(new Error("oh no")));
+
+  await ShieldRecipeClient.startup();
+  ok(PreferenceExperiments.init.called, "startup calls PreferenceExperiments.init");
+  // Even if PreferenceExperiments.init fails, RecipeRunner.init should be called.
+  ok(RecipeRunner.init.called, "startup calls RecipeRunner.init");
+
+  PreferenceExperiments.init.restore();
+  RecipeRunner.init.restore();
+});

browser/extensions/shield-recipe-client/test/browser/browser_Storage.js

@@ -1,46 +1,69 @@
 "use strict";
 
 Cu.import("resource://shield-recipe-client/lib/Storage.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/SandboxManager.jsm", this);
 
-const fakeSandbox = {Promise};
-const store1 = Storage.makeStorage("prefix1", fakeSandbox);
-const store2 = Storage.makeStorage("prefix2", fakeSandbox);
+add_task(async function() {
+  const fakeSandbox = {Promise};
+  const store1 = Storage.makeStorage("prefix1", fakeSandbox);
+  const store2 = Storage.makeStorage("prefix2", fakeSandbox);
 
-add_task(function* () {
   // Make sure values return null before being set
-  Assert.equal(yield store1.getItem("key"), null);
-  Assert.equal(yield store2.getItem("key"), null);
+  Assert.equal(await store1.getItem("key"), null);
+  Assert.equal(await store2.getItem("key"), null);
 
   // Set values to check
-  yield store1.setItem("key", "value1");
-  yield store2.setItem("key", "value2");
+  await store1.setItem("key", "value1");
+  await store2.setItem("key", "value2");
 
   // Check that they are available
-  Assert.equal(yield store1.getItem("key"), "value1");
-  Assert.equal(yield store2.getItem("key"), "value2");
+  Assert.equal(await store1.getItem("key"), "value1");
+  Assert.equal(await store2.getItem("key"), "value2");
 
   // Remove them, and check they are gone
-  yield store1.removeItem("key");
-  yield store2.removeItem("key");
-  Assert.equal(yield store1.getItem("key"), null);
-  Assert.equal(yield store2.getItem("key"), null);
+  await store1.removeItem("key");
+  await store2.removeItem("key");
+  Assert.equal(await store1.getItem("key"), null);
+  Assert.equal(await store2.getItem("key"), null);
 
   // Check that numbers are stored as numbers (not strings)
-  yield store1.setItem("number", 42);
-  Assert.equal(yield store1.getItem("number"), 42);
+  await store1.setItem("number", 42);
+  Assert.equal(await store1.getItem("number"), 42);
 
   // Check complex types work
   const complex = {a: 1, b: [2, 3], c: {d: 4}};
-  yield store1.setItem("complex", complex);
-  Assert.deepEqual(yield store1.getItem("complex"), complex);
+  await store1.setItem("complex", complex);
+  Assert.deepEqual(await store1.getItem("complex"), complex);
 
   // Check that clearing the storage removes data from multiple
   // prefixes.
-  yield store1.setItem("removeTest", 1);
-  yield store2.setItem("removeTest", 2);
-  Assert.equal(yield store1.getItem("removeTest"), 1);
-  Assert.equal(yield store2.getItem("removeTest"), 2);
-  yield Storage.clearAllStorage();
-  Assert.equal(yield store1.getItem("removeTest"), null);
-  Assert.equal(yield store2.getItem("removeTest"), null);
+  await store1.setItem("removeTest", 1);
+  await store2.setItem("removeTest", 2);
+  Assert.equal(await store1.getItem("removeTest"), 1);
+  Assert.equal(await store2.getItem("removeTest"), 2);
+  await Storage.clearAllStorage();
+  Assert.equal(await store1.getItem("removeTest"), null);
+  Assert.equal(await store2.getItem("removeTest"), null);
+});
+
+add_task(async function testSandboxValueStorage() {
+  const manager1 = new SandboxManager();
+  const manager2 = new SandboxManager();
+  const store1 = Storage.makeStorage("testSandboxValueStorage", manager1.sandbox);
+  const store2 = Storage.makeStorage("testSandboxValueStorage", manager2.sandbox);
+  manager1.addGlobal("store", store1);
+  manager2.addGlobal("store", store2);
+  manager1.addHold("testing");
+  manager2.addHold("testing");
+
+  await manager1.evalInSandbox("store.setItem('foo', {foo: 'bar'});");
+  manager1.removeHold("testing");
+  await manager1.isNuked();
+
+  const objectMatches = await manager2.evalInSandbox(`
+    store.getItem("foo").then(item => item.foo === "bar");
+  `);
+  ok(objectMatches, "Values persisted in a store survive after their originating sandbox is nuked");
+
+  manager2.removeHold("testing");
 });

browser/extensions/shield-recipe-client/test/browser/head.js

@@ -1,14 +1,23 @@
 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 
+
+Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://shield-recipe-client/lib/SandboxManager.jsm", this);
 Cu.import("resource://shield-recipe-client/lib/NormandyDriver.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/NormandyApi.jsm", this);
+Cu.import("resource://shield-recipe-client/lib/Utils.jsm", this);
 
 // Load mocking/stubbing library, sinon
 // docs: http://sinonjs.org/docs/
 const loader = Cc["@mozilla.org/moz/jssubscript-loader;1"].getService(Ci.mozIJSSubScriptLoader);
 loader.loadSubScript("resource://testing-common/sinon-1.16.1.js");
 
-registerCleanupFunction(function*() {
+// Make sinon assertions fail in a way that mochitest understands
+sinon.assert.fail = function(message) {
+  ok(false, message);
+};
+
+registerCleanupFunction(async function() {
   // Cleanup window or the test runner will throw an error
   delete window.sinon;
   delete window.setImmediate;
@@ -18,23 +27,95 @@ registerCleanupFunction(function*() {
 
 this.UUID_REGEX = /[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/;
 
-this.withSandboxManager = function(Assert, testGenerator) {
-  return function* inner() {
+this.withSandboxManager = function(Assert, testFunction) {
+  return async function inner() {
     const sandboxManager = new SandboxManager();
     sandboxManager.addHold("test running");
 
-    yield testGenerator(sandboxManager);
+    await testFunction(sandboxManager);
 
     sandboxManager.removeHold("test running");
-    yield sandboxManager.isNuked()
+    await sandboxManager.isNuked()
       .then(() => Assert.ok(true, "sandbox is nuked"))
       .catch(e => Assert.ok(false, "sandbox is nuked", e));
   };
 };
 
-this.withDriver = function(Assert, testGenerator) {
-  return withSandboxManager(Assert, function* inner(sandboxManager) {
+this.withDriver = function(Assert, testFunction) {
+  return withSandboxManager(Assert, async function inner(sandboxManager) {
     const driver = new NormandyDriver(sandboxManager);
-    yield testGenerator(driver);
+    await testFunction(driver);
   });
 };
+
+this.withMockNormandyApi = function(testFunction) {
+  return async function inner(...args) {
+    const mockApi = {actions: [], recipes: [], implementations: {}};
+
+    sinon.stub(NormandyApi, "fetchActions", async () => mockApi.actions);
+    sinon.stub(NormandyApi, "fetchRecipes", async () => mockApi.recipes);
+    sinon.stub(NormandyApi, "fetchImplementation", async action => {
+      const impl = mockApi.implementations[action.name];
+      if (!impl) {
+        throw new Error("Missing");
+      }
+      return impl;
+    });
+
+    await testFunction(mockApi, ...args);
+
+    NormandyApi.fetchActions.restore();
+    NormandyApi.fetchRecipes.restore();
+    NormandyApi.fetchImplementation.restore();
+  };
+};
+
+const preferenceBranches = {
+  user: Preferences,
+  default: new Preferences({defaultBranch: true}),
+};
+
+this.withMockPreferences = function(testFunction) {
+  return async function inner(...args) {
+    const prefManager = new MockPreferences();
+    try {
+      await testFunction(...args, prefManager);
+    } finally {
+      prefManager.cleanup();
+    }
+  };
+};
+
+class MockPreferences {
+  constructor() {
+    this.oldValues = {user: {}, default: {}};
+  }
+
+  set(name, value, branch = "user") {
+    this.preserve(name, branch);
+    preferenceBranches[branch].set(name, value);
+  }
+
+  preserve(name, branch) {
+    if (!(name in this.oldValues[branch])) {
+      const preferenceBranch = preferenceBranches[branch];
+      this.oldValues[branch][name] = {
+        oldValue: preferenceBranch.get(name),
+        existed: preferenceBranch.has(name),
+      };
+    }
+  }
+
+  cleanup() {
+    for (const [branchName, values] of Object.entries(this.oldValues)) {
+      const preferenceBranch = preferenceBranches[branchName];
+      for (const [name, {oldValue, existed}] of Object.entries(values)) {
+        if (existed) {
+          preferenceBranch.set(name, oldValue);
+        } else {
+          preferenceBranch.reset(name);
+        }
+      }
+    }
+  }
+}

browser/extensions/shield-recipe-client/test/unit/.eslintrc.js

@@ -4,5 +4,12 @@ module.exports = {
   globals: {
     do_get_file: false,
     equal: false,
+    Cu: false,
+    ok: false,
+    load: false,
+    do_register_cleanup: false,
+    sinon: false,
+    notEqual: false,
+    deepEqual: false,
   },
 };

browser/extensions/shield-recipe-client/test/unit/echo_server.sjs

@@ -0,0 +1,21 @@
+/**
+ * Reads an HTTP status code and response body from the querystring and sends
+ * back a matching response.
+ */
+function handleRequest(request, response) {
+  // Allow cross-origin, so you can XHR to it!
+  response.setHeader("Access-Control-Allow-Origin", "*", false);
+  // Avoid confusing cache behaviors
+  response.setHeader("Cache-Control", "no-cache", false);
+
+  const params = request.queryString.split("&");
+  for (const param of params) {
+    const [key, value] = param.split("=");
+    if (key === "status") {
+      response.setStatusLine(null, value);
+    } else if (key === "body") {
+      response.write(value);
+    }
+  }
+  response.write("");
+}

browser/extensions/shield-recipe-client/test/unit/test_ActionSandboxManager.js

@@ -0,0 +1,169 @@
+"use strict";
+
+Cu.import("resource://shield-recipe-client/lib/ActionSandboxManager.jsm");
+Cu.import("resource://shield-recipe-client/lib/NormandyDriver.jsm");
+
+async function withManager(script, testFunction) {
+  const manager = new ActionSandboxManager(script);
+  manager.addHold("testing");
+  await testFunction(manager);
+  manager.removeHold("testing");
+}
+
+add_task(async function testMissingCallbackName() {
+  await withManager("1 + 1", async manager => {
+    equal(
+      await manager.runAsyncCallback("missingCallback"),
+      undefined,
+      "runAsyncCallback returns undefined when given a missing callback name",
+    );
+  });
+});
+
+add_task(async function testCallback() {
+  const script = `
+    registerAsyncCallback("testCallback", async function(normandy) {
+      return 5;
+    });
+  `;
+
+  await withManager(script, async manager => {
+    const result = await manager.runAsyncCallback("testCallback");
+    equal(result, 5, "runAsyncCallback executes the named callback inside the sandbox");
+  });
+});
+
+add_task(async function testArguments() {
+  const script = `
+    registerAsyncCallback("testCallback", async function(normandy, a, b) {
+      return a + b;
+    });
+  `;
+
+  await withManager(script, async manager => {
+    const result = await manager.runAsyncCallback("testCallback", 4, 6);
+    equal(result, 10, "runAsyncCallback passes arguments to the callback");
+  });
+});
+
+add_task(async function testCloning() {
+  const script = `
+    registerAsyncCallback("testCallback", async function(normandy, obj) {
+      return {foo: "bar", baz: obj.baz};
+    });
+  `;
+
+  await withManager(script, async manager => {
+    const result = await manager.runAsyncCallback("testCallback", {baz: "biff"});
+
+    deepEqual(
+      result,
+      {foo: "bar", baz: "biff"},
+      (
+        "runAsyncCallback clones arguments into the sandbox and return values into the " +
+        "context it was called from"
+      ),
+    );
+  });
+});
+
+add_task(async function testError() {
+  const script = `
+    registerAsyncCallback("testCallback", async function(normandy) {
+      throw new Error("WHY")
+    });
+  `;
+
+  await withManager(script, async manager => {
+    try {
+      await manager.runAsyncCallback("testCallback");
+      ok(false, "runAsnycCallbackFromScript throws errors when raised by the sandbox");
+    } catch (err) {
+      equal(err.message, "WHY", "runAsnycCallbackFromScript clones error messages");
+    }
+  });
+});
+
+add_task(async function testDriver() {
+  const script = `
+    registerAsyncCallback("testCallback", async function(normandy) {
+      return normandy;
+    });
+  `;
+
+  await withManager(script, async manager => {
+    const sandboxDriver = await manager.runAsyncCallback("testCallback");
+    const referenceDriver = new NormandyDriver(manager);
+    equal(
+      sandboxDriver.constructor.name,
+      "NormandyDriver",
+      "runAsyncCallback passes a driver as the first parameter",
+    );
+    for (const prop in referenceDriver) {
+      ok(prop in sandboxDriver, "runAsyncCallback passes a driver as the first parameter");
+    }
+  });
+});
+
+add_task(async function testGlobalObject() {
+  // Test that window is an alias for the global object, and that it
+  // has some expected functions available on it.
+  const script = `
+    window.setOnWindow = "set";
+    this.setOnGlobal = "set";
+
+    registerAsyncCallback("testCallback", async function(normandy) {
+      return {
+        setOnWindow: setOnWindow,
+        setOnGlobal: window.setOnGlobal,
+        setTimeoutExists: setTimeout !== undefined,
+        clearTimeoutExists: clearTimeout !== undefined,
+      };
+    });
+  `;
+
+  await withManager(script, async manager => {
+    const result = await manager.runAsyncCallback("testCallback");
+    Assert.deepEqual(result, {
+      setOnWindow: "set",
+      setOnGlobal: "set",
+      setTimeoutExists: true,
+      clearTimeoutExists: true,
+    }, "sandbox.window is the global object and has expected functions.");
+  });
+});
+
+add_task(async function testRegisterActionShim() {
+  const recipe = {
+    foo: "bar",
+  };
+  const script = `
+    class TestAction {
+      constructor(driver, recipe) {
+        this.driver = driver;
+        this.recipe = recipe;
+      }
+
+      execute() {
+        return new Promise(resolve => {
+          resolve({
+            foo: this.recipe.foo,
+            isDriver: "log" in this.driver,
+          });
+        });
+      }
+    }
+
+    registerAction('test-action', TestAction);
+  `;
+
+  await withManager(script, async manager => {
+    const result = await manager.runAsyncCallback("action", recipe);
+    equal(result.foo, "bar", "registerAction registers an async callback for actions");
+    equal(
+      result.isDriver,
+      true,
+      "registerAction passes the driver to the action class constructor",
+    );
+  });
+});

browser/extensions/shield-recipe-client/test/unit/test_NormandyApi.js

@@ -3,48 +3,28 @@
 /* globals Cu */
 
 Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://testing-common/httpd.js"); /* globals HttpServer */
 Cu.import("resource://gre/modules/osfile.jsm", this); /* globals OS */
 Cu.import("resource://shield-recipe-client/lib/NormandyApi.jsm", this);
 
-class PrefManager {
-  constructor() {
-    this.oldValues = {};
-  }
-
-  setCharPref(name, value) {
-    if (!(name in this.oldValues)) {
-      this.oldValues[name] = Services.prefs.getCharPref(name);
-    }
-    Services.prefs.setCharPref(name, value);
-  }
-
-  cleanup() {
-    for (const name of Object.keys(this.oldValues)) {
-      Services.prefs.setCharPref(name, this.oldValues[name]);
-    }
-  }
-}
+load("utils.js"); /* globals withMockPreferences */
 
 function withServer(server, task) {
-  return function* inner() {
+  return withMockPreferences(async function inner(preferences) {
     const serverUrl = `http://localhost:${server.identity.primaryPort}`;
-    const prefManager = new PrefManager();
-    prefManager.setCharPref("extensions.shield-recipe-client.api_url", `${serverUrl}/api/v1`);
-    prefManager.setCharPref(
+    preferences.set("extensions.shield-recipe-client.api_url", `${serverUrl}/api/v1`);
+    preferences.set(
       "security.content.signature.root_hash",
       // Hash of the key that signs the normandy dev certificates
       "4C:35:B1:C3:E3:12:D9:55:E7:78:ED:D0:A7:E7:8A:38:83:04:EF:01:BF:FA:03:29:B2:46:9F:3C:C5:EC:36:04"
     );
 
     try {
-      yield task(serverUrl);
+      await task(serverUrl);
     } finally {
-      prefManager.cleanup();
-      yield new Promise(resolve => server.stop(resolve));
+      await new Promise(resolve => server.stop(resolve));
     }
-  };
+  });
 }
 
 function makeScriptServer(scriptPath) {
@@ -63,7 +43,7 @@ function makeMockApiServer() {
   const server = new HttpServer();
   server.registerDirectory("/", do_get_file("mock_api"));
 
-  server.setIndexHandler(Task.async(function* (request, response) {
+  server.setIndexHandler(async function(request, response) {
     response.processAsync();
     const dir = request.getProperty("directory");
     const index = dir.clone();
@@ -77,7 +57,7 @@ function makeMockApiServer() {
     }
 
     try {
-      const contents = yield OS.File.read(index.path, {encoding: "utf-8"});
+      const contents = await OS.File.read(index.path, {encoding: "utf-8"});
       response.write(contents);
     } catch (e) {
       response.setStatusLine("1.1", 500, "Server error");
@@ -85,7 +65,7 @@ function makeMockApiServer() {
     } finally {
       response.finish();
     }
-  }));
+  });
 
   server.start(-1);
   return server;
@@ -95,62 +75,92 @@ function withMockApiServer(task) {
   return withServer(makeMockApiServer(), task);
 }
 
-add_task(withMockApiServer(function* test_get(serverUrl) {
+add_task(withMockApiServer(async function test_get(serverUrl) {
   // Test that NormandyApi can fetch from the test server.
-  const response = yield NormandyApi.get(`${serverUrl}/api/v1`);
-  const data = yield response.json();
+  const response = await NormandyApi.get(`${serverUrl}/api/v1`);
+  const data = await response.json();
   equal(data["recipe-list"], "/api/v1/recipe/", "Expected data in response");
 }));
 
-add_task(withMockApiServer(function* test_getApiUrl(serverUrl) {
+add_task(withMockApiServer(async function test_getApiUrl(serverUrl) {
   const apiBase = `${serverUrl}/api/v1`;
   // Test that NormandyApi can use the self-describing API's index
-  const recipeListUrl = yield NormandyApi.getApiUrl("action-list");
+  const recipeListUrl = await NormandyApi.getApiUrl("action-list");
   equal(recipeListUrl, `${apiBase}/action/`, "Can retrieve action-list URL from API");
 }));
 
-add_task(withMockApiServer(function* test_fetchRecipes() {
-  const recipes = yield NormandyApi.fetchRecipes();
+add_task(withMockApiServer(async function test_fetchRecipes() {
+  const recipes = await NormandyApi.fetchRecipes();
   equal(recipes.length, 1);
   equal(recipes[0].name, "system-addon-test");
 }));
 
-add_task(withMockApiServer(function* test_classifyClient() {
-  const classification = yield NormandyApi.classifyClient();
+add_task(withMockApiServer(async function test_classifyClient() {
+  const classification = await NormandyApi.classifyClient();
   Assert.deepEqual(classification, {
     country: "US",
     request_time: new Date("2017-02-22T17:43:24.657841Z"),
   });
 }));
 
-add_task(withMockApiServer(function* test_fetchAction() {
-  const action = yield NormandyApi.fetchAction("show-heartbeat");
-  equal(action.name, "show-heartbeat");
+add_task(withMockApiServer(async function test_fetchActions() {
+  const actions = await NormandyApi.fetchActions();
+  equal(actions.length, 2);
+  const actionNames = actions.map(a => a.name);
+  ok(actionNames.includes("console-log"));
+  ok(actionNames.includes("show-heartbeat"));
 }));
 
-add_task(withScriptServer("test_server.sjs", function* test_getTestServer(serverUrl) {
+add_task(withScriptServer("query_server.sjs", async function test_getTestServer(serverUrl) {
   // Test that NormandyApi can fetch from the test server.
-  const response = yield NormandyApi.get(serverUrl);
-  const data = yield response.json();
+  const response = await NormandyApi.get(serverUrl);
+  const data = await response.json();
   Assert.deepEqual(data, {queryString: {}, body: {}}, "NormandyApi returned incorrect server data.");
 }));
 
-add_task(withScriptServer("test_server.sjs", function* test_getQueryString(serverUrl) {
+add_task(withScriptServer("query_server.sjs", async function test_getQueryString(serverUrl) {
   // Test that NormandyApi can send query string parameters to the test server.
-  const response = yield NormandyApi.get(serverUrl, {foo: "bar", baz: "biff"});
-  const data = yield response.json();
+  const response = await NormandyApi.get(serverUrl, {foo: "bar", baz: "biff"});
+  const data = await response.json();
   Assert.deepEqual(
     data, {queryString: {foo: "bar", baz: "biff"}, body: {}},
     "NormandyApi sent an incorrect query string."
   );
 }));
 
-add_task(withScriptServer("test_server.sjs", function* test_postData(serverUrl) {
+add_task(withScriptServer("query_server.sjs", async function test_postData(serverUrl) {
   // Test that NormandyApi can POST JSON-formatted data to the test server.
-  const response = yield NormandyApi.post(serverUrl, {foo: "bar", baz: "biff"});
-  const data = yield response.json();
+  const response = await NormandyApi.post(serverUrl, {foo: "bar", baz: "biff"});
+  const data = await response.json();
   Assert.deepEqual(
     data, {queryString: {}, body: {foo: "bar", baz: "biff"}},
     "NormandyApi sent an incorrect query string."
   );
 }));
+
+add_task(withScriptServer("echo_server.sjs", async function test_fetchImplementation(serverUrl) {
+  const action = {
+    implementation_url: `${serverUrl}?status=200&body=testcontent`,
+  };
+  equal(
+    await NormandyApi.fetchImplementation(action),
+    "testcontent",
+    "fetchImplementation fetches the content at the correct URL",
+  );
+}));
+
+add_task(withScriptServer(
+  "echo_server.sjs",
+  async function test_fetchImplementationFail(serverUrl) {
+    const action = {
+      implementation_url: `${serverUrl}?status=500&body=servererror`,
+    };
+
+    try {
+      await NormandyApi.fetchImplementation(action);
+      ok(false, "fetchImplementation throws for non-200 response status codes");
+    } catch (err) {
+      // pass
+    }
+  },
+));

browser/extensions/shield-recipe-client/test/unit/test_Sampling.js

@@ -4,27 +4,44 @@
 
 Cu.import("resource://shield-recipe-client/lib/Sampling.jsm", this);
 
-add_task(function* testStableSample() {
+add_task(async function testStableSample() {
   // Absolute samples
-  equal(yield Sampling.stableSample("test", 1), true, "stableSample returns true for 100% sample");
-  equal(yield Sampling.stableSample("test", 0), false, "stableSample returns false for 0% sample");
+  equal(await Sampling.stableSample("test", 1), true, "stableSample returns true for 100% sample");
+  equal(await Sampling.stableSample("test", 0), false, "stableSample returns false for 0% sample");
 
   // Known samples. The numbers are nonces to make the tests pass
-  equal(yield Sampling.stableSample("test-0", 0.5), true, "stableSample returns true for known matching sample");
-  equal(yield Sampling.stableSample("test-1", 0.5), false, "stableSample returns false for known non-matching sample");
+  equal(await Sampling.stableSample("test-0", 0.5), true, "stableSample returns true for known matching sample");
+  equal(await Sampling.stableSample("test-1", 0.5), false, "stableSample returns false for known non-matching sample");
 });
 
-add_task(function* testBucketSample() {
+add_task(async function testBucketSample() {
   // Absolute samples
-  equal(yield Sampling.bucketSample("test", 0, 10, 10), true, "bucketSample returns true for 100% sample");
-  equal(yield Sampling.bucketSample("test", 0, 0, 10), false, "bucketSample returns false for 0% sample");
+  equal(await Sampling.bucketSample("test", 0, 10, 10), true, "bucketSample returns true for 100% sample");
+  equal(await Sampling.bucketSample("test", 0, 0, 10), false, "bucketSample returns false for 0% sample");
 
   // Known samples. The numbers are nonces to make the tests pass
-  equal(yield Sampling.bucketSample("test-0", 0, 5, 10), true, "bucketSample returns true for known matching sample");
-  equal(yield Sampling.bucketSample("test-1", 0, 5, 10), false, "bucketSample returns false for known non-matching sample");
+  equal(await Sampling.bucketSample("test-0", 0, 5, 10), true, "bucketSample returns true for known matching sample");
+  equal(await Sampling.bucketSample("test-1", 0, 5, 10), false, "bucketSample returns false for known non-matching sample");
 });
 
-add_task(function* testFractionToKey() {
+add_task(async function testRatioSample() {
+  // Invalid input
+  Assert.rejects(Sampling.ratioSample("test", []), "ratioSample rejects for a list with no ratios");
+
+  // Absolute samples
+  equal(await Sampling.ratioSample("test", [1]), 0, "ratioSample returns 0 for a list with only 1 ratio");
+  equal(
+    await Sampling.ratioSample("test", [0, 0, 1, 0]),
+    2,
+    "ratioSample returns the only non-zero bucket if all other buckets are zero"
+  );
+
+  // Known samples. The numbers are nonces to make the tests pass
+  equal(await Sampling.ratioSample("test-0", [1, 1]), 0, "ratioSample returns the correct index for known matching sample");
+  equal(await Sampling.ratioSample("test-1", [1, 1]), 1, "ratioSample returns the correct index for known non-matching sample");
+});
+
+add_task(async function testFractionToKey() {
   // Test that results are always 12 character hexadecimal strings.
   const expected_regex = /[0-9a-f]{12}/;
   const count = 100;
@@ -38,12 +55,12 @@ add_task(function* testFractionToKey() {
   equal(successes, count, "fractionToKey makes keys the right length");
 });
 
-add_task(function* testTruncatedHash() {
+add_task(async function testTruncatedHash() {
   const expected_regex = /[0-9a-f]{12}/;
   const count = 100;
   let successes = 0;
   for (let i = 0; i < count; i++) {
-    const h = yield Sampling.truncatedHash(Math.random());
+    const h = await Sampling.truncatedHash(Math.random());
     if (expected_regex.test(h)) {
       successes++;
     }
@@ -51,7 +68,7 @@ add_task(function* testTruncatedHash() {
   equal(successes, count, "truncatedHash makes hashes the right length");
 });
 
-add_task(function* testBufferToHex() {
+add_task(async function testBufferToHex() {
   const data = new ArrayBuffer(4);
   const view = new DataView(data);
   view.setUint8(0, 0xff);

browser/extensions/shield-recipe-client/test/unit/test_SandboxManager.js

@@ -0,0 +1,110 @@
+"use strict";
+
+Cu.import("resource://shield-recipe-client/lib/SandboxManager.jsm");
+
+// wrapAsync should wrap privileged Promises with Promises that are usable by
+// the sandbox.
+add_task(function* () {
+  const manager = new SandboxManager();
+  manager.addHold("testing");
+
+  manager.cloneIntoGlobal("driver", {
+    async privileged() {
+      return "privileged";
+    },
+    wrapped: manager.wrapAsync(async function() {
+      return "wrapped";
+    }),
+    aValue: "aValue",
+    wrappedThis: manager.wrapAsync(async function() {
+      return this.aValue;
+    }),
+  }, {cloneFunctions: true});
+
+  // Assertion helpers
+  manager.addGlobal("ok", ok);
+  manager.addGlobal("equal", equal);
+
+  const sandboxResult = yield new Promise(resolve => {
+    manager.addGlobal("resolve", result => resolve(result));
+    manager.evalInSandbox(`
+      // Unwrapped privileged promises are not accessible in the sandbox
+      try {
+        const privilegedResult = driver.privileged().then(() => false);
+        ok(false, "The sandbox could not use a privileged Promise");
+      } catch (err) { }
+
+      // Wrapped functions return promises that the sandbox can access.
+      const wrappedResult = driver.wrapped();
+      ok("then" in wrappedResult);
+
+      // Resolve the Promise around the sandbox with the wrapped result to test
+      // that the Promise in the sandbox works.
+      wrappedResult.then(resolve);
+    `);
+  });
+  equal(sandboxResult, "wrapped", "wrapAsync methods return Promises that work in the sandbox");
+
+  yield manager.evalInSandbox(`
+    (async function sandboxTest() {
+      equal(
+        await driver.wrappedThis(),
+        "aValue",
+        "wrapAsync preserves the behavior of the this keyword",
+      );
+    })();
+  `);
+
+  manager.removeHold("testing");
+});
+
+// wrapAsync cloning options
+add_task(function* () {
+  const manager = new SandboxManager();
+  manager.addHold("testing");
+
+  // clonedArgument stores the argument passed to cloned(), which we use to test
+  // that arguments from within the sandbox are cloned outside.
+  let clonedArgument = null;
+  manager.cloneIntoGlobal("driver", {
+    uncloned: manager.wrapAsync(async function() {
+      return {value: "uncloned"};
+    }),
+    cloned: manager.wrapAsync(async function(argument) {
+      clonedArgument = argument;
+      return {value: "cloned"};
+    }, {cloneInto: true, cloneArguments: true}),
+  }, {cloneFunctions: true});
+
+  // Assertion helpers
+  manager.addGlobal("ok", ok);
+  manager.addGlobal("deepEqual", deepEqual);
+
+  yield new Promise(resolve => {
+    manager.addGlobal("resolve", resolve);
+    manager.evalInSandbox(`
+      (async function() {
+        // The uncloned return value should be privileged and inaccesible.
+        const uncloned = await driver.uncloned();
+        ok(!("value" in uncloned), "The sandbox could not use an uncloned return value");
+
+        // The cloned return value should be usable.
+        deepEqual(
+          await driver.cloned({value: "insidesandbox"}),
+          {value: "cloned"},
+          "The sandbox could use the cloned return value",
+        );
+      })().then(resolve);
+    `);
+  });
+
+  // Removing the hold nukes the sandbox. Afterwards, because cloned() has the
+  // cloneArguments option, the clonedArgument variable should still be
+  // accessible.
+  manager.removeHold("testing");
+  deepEqual(
+    clonedArgument,
+    {value: "insidesandbox"},
+    "cloneArguments allowed an argument from within the sandbox to persist after it was nuked",
+  );
+});

browser/extensions/shield-recipe-client/test/unit/test_Utils.js

@@ -0,0 +1,21 @@
+"use strict";
+/* globals Cu */
+
+Cu.import("resource://shield-recipe-client/lib/Utils.jsm");
+
+add_task(async function testKeyBy() {
+  const list = [];
+  deepEqual(Utils.keyBy(list, "foo"), {});
+
+  const foo = {name: "foo", value: 1};
+  list.push(foo);
+  deepEqual(Utils.keyBy(list, "name"), {foo});
+
+  const bar = {name: "bar", value: 2};
+  list.push(bar);
+  deepEqual(Utils.keyBy(list, "name"), {foo, bar});
+
+  const missingKey = {value: 7};
+  list.push(missingKey);
+  deepEqual(Utils.keyBy(list, "name"), {foo, bar}, "keyBy skips items that are missing the key");
+});

browser/extensions/shield-recipe-client/test/unit/utils.js

@@ -0,0 +1,51 @@
+"use strict";
+/* eslint-disable no-unused-vars */
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+
+const preferenceBranches = {
+  user: Preferences,
+  default: new Preferences({defaultBranch: true}),
+};
+
+// duplicated from test/browser/head.js until we move everything over to mochitests.
+function withMockPreferences(testFunction) {
+  return async function inner(...args) {
+    const prefManager = new MockPreferences();
+    try {
+      await testFunction(...args, prefManager);
+    } finally {
+      prefManager.cleanup();
+    }
+  };
+}
+
+class MockPreferences {
+  constructor() {
+    this.oldValues = {user: {}, default: {}};
+  }
+
+  set(name, value, branch = "user") {
+    this.preserve(name, branch);
+    preferenceBranches[branch].set(name, value);
+  }
+
+  preserve(name, branch) {
+    if (!(name in this.oldValues[branch])) {
+      this.oldValues[branch][name] = preferenceBranches[branch].get(name, undefined);
+    }
+  }
+
+  cleanup() {
+    for (const [branchName, values] of Object.entries(this.oldValues)) {
+      const preferenceBranch = preferenceBranches[branchName];
+      for (const [name, value] of Object.entries(values)) {
+        if (value !== undefined) {
+          preferenceBranch.set(name, value);
+        } else {
+          preferenceBranch.reset(name);
+        }
+      }
+    }
+  }
+}

browser/extensions/shield-recipe-client/test/unit/xpc_head.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const {interfaces: Ci, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 
 Cu.import("resource://gre/modules/Services.jsm");
 
@@ -17,3 +17,12 @@ if (!extensionDir.exists()) {
   extensionDir.append(EXTENSION_ID + ".xpi");
 }
 Components.manager.addBootstrappedManifestLocation(extensionDir);
+
+// Load Sinon for mocking/stubbing during tests.
+// Sinon assumes that setTimeout and friends are available, and looks for a
+// global object named self during initialization.
+Cu.import("resource://gre/modules/Timer.jsm");
+const self = {}; // eslint-disable-line no-unused-vars
+
+const loader = Cc["@mozilla.org/moz/jssubscript-loader;1"].getService(Ci.mozIJSSubScriptLoader);
+loader.loadSubScript("resource://testing-common/sinon-1.16.1.js");

browser/extensions/shield-recipe-client/test/unit/xpcshell.ini

@@ -2,7 +2,10 @@
 head = xpc_head.js
 support-files =
   mock_api/**
-  test_server.sjs
+  query_server.sjs
+  echo_server.sjs
+  utils.js
 
 [test_NormandyApi.js]
 [test_Sampling.js]
+[test_SandboxManager.js]

browser/locales/en-US/chrome/browser/browser.properties

@@ -731,6 +731,8 @@ emeNotifications.drmContentPlaying.message2 = Some audio or video on this site u
 emeNotifications.drmContentPlaying.button.label = Configure…
 emeNotifications.drmContentPlaying.button.accesskey = C
 
+# LOCALIZATION NOTE(emeNotifications.drmContentDisabled.message): NB: inserted via innerHTML, so please don't use <, > or & in this string. %S will be the 'learn more' link
+emeNotifications.drmContentDisabled.message = You must enable DRM to play some audio or video on this page. %S
 emeNotifications.drmContentDisabled.button.label = Enable DRM
 emeNotifications.drmContentDisabled.button.accesskey = E
 # LOCALIZATION NOTE(emeNotifications.drmContentDisabled.learnMoreLabel): NB: inserted via innerHTML, so please don't use <, > or & in this string.

browser/locales/search/list.json

@@ -460,6 +460,13 @@
         ]
       }
     },
+    "lo": {
+      "default": {
+        "visibleDefaultEngines": [
+          "google", "yahoo", "bing", "ddg", "wikipedia-lo", "twitter"
+        ]
+      }
+    },
     "lt": {
       "default": {
         "visibleDefaultEngines": [

browser/locales/searchplugins/wikipedia-lo.xml

@@ -0,0 +1,19 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/">
+<ShortName>ວິກິພີເດຍ (lo)</ShortName>
+<Description>ວິກິພີເດຍ, ສາລານຸກົມເສລີ</Description>
+<InputEncoding>UTF-8</InputEncoding>
+<Image width="16" height="16">resource://search-plugins/images/wikipedia.ico</Image>
+<Url type="application/x-suggestions+json" method="GET" template="https://lo.wikipedia.org/w/api.php">
+  <Param name="action" value="opensearch"/>
+  <Param name="search" value="{searchTerms}"/>
+</Url>
+<Url type="text/html" method="GET" template="https://lo.wikipedia.org/wiki/ພິເສດ:ຊອກຫາ"
+     resultdomain="wikipedia.org" rel="searchform">
+  <Param name="search" value="{searchTerms}"/>
+  <Param name="sourceid" value="Mozilla-search"/>
+</Url>
+</SearchPlugin>

browser/modules/moz.build

@@ -26,7 +26,7 @@ with Files("test/browser/*SitePermissions*"):
     BUG_COMPONENT = ("Firefox", "Site Identity and Permission Panels")
 
 with Files("test/browser/browser_UnsubmittedCrashHandler.js"):
-    BUG_COMPONENT = ("Toolkit", "Breakpad Integration")
+    BUG_COMPONENT = ("Toolkit", "Crash Reporting")
 
 with Files("test/browser/browser_bug1319078.js"):
     BUG_COMPONENT = ("Core", "DOM: Core & HTML")
@@ -62,7 +62,7 @@ with Files("*Telemetry.jsm"):
     BUG_COMPONENT = ("Toolkit", "Telemetry")
 
 with Files("ContentCrashHandlers.jsm"):
-    BUG_COMPONENT = ("Toolkit", "Breakpad Integration")
+    BUG_COMPONENT = ("Toolkit", "Crash Reporting")
 
 with Files("ContentSearch.jsm"):
     BUG_COMPONENT = ("Firefox", "Search")

browser/themes/shared/customizableui/panelUI.inc.css

@@ -1203,6 +1203,14 @@ menuitem.panel-subview-footer@menuStateActive@,
   padding-left: 6px;
 }
 
+/* Yeah, the ids are ugly, but this should be reasonably performant, and
+ * using a tagname as the last item would be less so.
+ */
+#widget-overflow-fixed-list:empty + #widget-overflow-fixed-separator {
+  display: none;
+}
+
+#widget-overflow-scroller > toolbarseparator,
 .PanelUI-subView menuseparator,
 .PanelUI-subView toolbarseparator,
 .cui-widget-panelview menuseparator {
@@ -1428,7 +1436,7 @@ toolbarpaletteitem[haswideitem][place="panel"] + toolbarpaletteitem[haswideitem]
   margin-bottom: 10px;
 }
 
-#widget-overflow-list {
+.widget-overflow-list {
   width: @menuPanelWidth@;
   padding-left: 10px;
   padding-right: 10px;
@@ -1455,7 +1463,7 @@ toolbaritem[overflowedItem=true],
   padding-inline-start: .5em;
 }
 
-#widget-overflow-list > .toolbaritem-combined-buttons {
+.widget-overflow-list > .toolbaritem-combined-buttons {
   min-height: 28px;
 }
 

build/moz.configure/toolchain.configure

@@ -1008,3 +1008,4 @@ def security_hardening_cflags(value, c_compiler):
         return '-fstack-protector-strong'
 
 add_old_configure_assignment('HARDENING_CFLAGS', security_hardening_cflags)
+imply_option('--enable-pie', depends_if('--enable-hardening')(lambda v: v))

devtools/client/framework/moz.build

@@ -26,6 +26,7 @@ DevToolsModules(
     'selection.js',
     'sidebar.js',
     'source-map-service.js',
+    'source-map-url-service.js',
     'target-from-url.js',
     'target.js',
     'toolbox-highlighter-utils.js',

devtools/client/framework/source-map-url-service.js

@@ -0,0 +1,89 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+/**
+ * A simple service to track source actors and keep a mapping between
+ * original URLs and objects holding the source actor's ID (which is
+ * used as a cookie by the devtools-source-map service) and the source
+ * map URL.
+ *
+ * @param {object} target
+ *        The object the toolbox is debugging.
+ * @param {SourceMapService} sourceMapService
+ *        The devtools-source-map functions
+ */
+function SourceMapURLService(target, sourceMapService) {
+  this._target = target;
+  this._sourceMapService = sourceMapService;
+  this._urls = new Map();
+
+  this._onSourceUpdated = this._onSourceUpdated.bind(this);
+  this.reset = this.reset.bind(this);
+
+  target.on("source-updated", this._onSourceUpdated);
+  target.on("will-navigate", this.reset);
+}
+
+/**
+ * Reset the service.  This flushes the internal cache.
+ */
+SourceMapURLService.prototype.reset = function () {
+  this._urls.clear();
+};
+
+/**
+ * Shut down the service, unregistering its event listeners and
+ * flushing the cache.  After this call the service will no longer
+ * function.
+ */
+SourceMapURLService.prototype.destroy = function () {
+  this.reset();
+  this._target.off("source-updated", this._onSourceUpdated);
+  this._target.off("will-navigate", this.reset);
+  this._target = this._urls = null;
+};
+
+/**
+ * A helper function that is called when a new source is available.
+ */
+SourceMapURLService.prototype._onSourceUpdated = function (_, sourceEvent) {
+  let { source } = sourceEvent;
+  let { generatedUrl, url, actor: id, sourceMapURL } = source;
+
+  // As long as the actor is also handling source maps, we want the
+  // generated URL if it is available.  This will be going away in bug 1349354.
+  let seenUrl = generatedUrl || url;
+  this._urls.set(seenUrl, { id, url: seenUrl, sourceMapURL });
+};
+
+/**
+ * Look up the original position for a given location.  This returns a
+ * promise resolving to either the original location, or null if the
+ * given location is not source-mapped.  If a location is returned, it
+ * is of the same form as devtools-source-map's |getOriginalLocation|.
+ *
+ * @param {String} url
+ *        The URL to map.
+ * @param {number} line
+ *        The line number to map.
+ * @param {number} column
+ *        The column number to map.
+ * @return Promise
+ *        A promise resolving either to the original location, or null.
+ */
+SourceMapURLService.prototype.originalPositionFor = async function (url, line, column) {
+  const urlInfo = this._urls.get(url);
+  if (!urlInfo) {
+    return null;
+  }
+  // Call getOriginalURLs to make sure the source map has been
+  // fetched.  We don't actually need the result of this though.
+  await this._sourceMapService.getOriginalURLs(urlInfo);
+  const location = { sourceId: urlInfo.id, line, column, sourceUrl: url };
+  let resolvedLocation = await this._sourceMapService.getOriginalLocation(location);
+  return resolvedLocation === location ? null : resolvedLocation;
+};
+
+exports.SourceMapURLService = SourceMapURLService;

devtools/client/framework/test/browser_source_map-01.js

@@ -24,42 +24,23 @@ const DEBUGGER_ROOT = "http://example.com/browser/devtools/client/debugger/test/
 const PAGE_URL = `${DEBUGGER_ROOT}doc_empty-tab-01.html`;
 const JS_URL = `${URL_ROOT}code_binary_search.js`;
 const COFFEE_URL = `${URL_ROOT}code_binary_search.coffee`;
-const { SourceMapService } = require("devtools/client/framework/source-map-service");
-const { serialize } = require("devtools/client/framework/location-store");
 
 add_task(function* () {
   const toolbox = yield openNewTabAndToolbox(PAGE_URL, "jsdebugger");
-  const service = new SourceMapService(toolbox.target);
-  let aggregator = new Map();
-
-  function onUpdate(e, oldLoc, newLoc) {
-    if (oldLoc.line === 6) {
-      checkLoc1(oldLoc, newLoc);
-    } else if (oldLoc.line === 8) {
-      checkLoc2(oldLoc, newLoc);
-    } else {
-      throw new Error(`Unexpected location update: ${JSON.stringify(oldLoc)}`);
-    }
-    aggregator.set(serialize(oldLoc), newLoc);
-  }
-
-  let loc1 = { url: JS_URL, line: 6 };
-  let loc2 = { url: JS_URL, line: 8, column: 3 };
-
-  service.subscribe(loc1, onUpdate);
-  service.subscribe(loc2, onUpdate);
+  const service = toolbox.sourceMapURLService;
 
   // Inject JS script
   let sourceShown = waitForSourceShown(toolbox.getCurrentPanel(), "code_binary_search");
   yield createScript(JS_URL);
   yield sourceShown;
 
-  yield waitUntil(() => aggregator.size === 2);
+  let loc1 = { url: JS_URL, line: 6 };
+  let newLoc1 = yield service.originalPositionFor(loc1.url, loc1.line);
+  checkLoc1(loc1, newLoc1);
 
-  aggregator = Array.from(aggregator.values());
-
-  ok(aggregator.find(i => i.url === COFFEE_URL && i.line === 4), "found first updated location");
-  ok(aggregator.find(i => i.url === COFFEE_URL && i.line === 6), "found second updated location");
+  let loc2 = { url: JS_URL, line: 8, column: 3 };
+  let newLoc2 = yield service.originalPositionFor(loc2.url, loc2.line, loc2.column);
+  checkLoc2(loc2, newLoc2);
 
   yield toolbox.destroy();
   gBrowser.removeCurrentTab();
@@ -72,7 +53,7 @@ function checkLoc1(oldLoc, newLoc) {
   is(oldLoc.url, JS_URL, "Correct url for JS:6");
   is(newLoc.line, 4, "Correct line for JS:6 -> COFFEE");
   is(newLoc.column, 2, "Correct column for JS:6 -> COFFEE -- handles falsy column entries");
-  is(newLoc.url, COFFEE_URL, "Correct url for JS:6 -> COFFEE");
+  is(newLoc.sourceUrl, COFFEE_URL, "Correct url for JS:6 -> COFFEE");
 }
 
 function checkLoc2(oldLoc, newLoc) {
@@ -81,7 +62,7 @@ function checkLoc2(oldLoc, newLoc) {
   is(oldLoc.url, JS_URL, "Correct url for JS:8:3");
   is(newLoc.line, 6, "Correct line for JS:8:3 -> COFFEE");
   is(newLoc.column, 10, "Correct column for JS:8:3 -> COFFEE");
-  is(newLoc.url, COFFEE_URL, "Correct url for JS:8:3 -> COFFEE");
+  is(newLoc.sourceUrl, COFFEE_URL, "Correct url for JS:8:3 -> COFFEE");
 }
 
 function createScript(url) {

devtools/client/framework/toolbox.js

@@ -62,6 +62,8 @@ loader.lazyRequireGetter(this, "ToolboxButtons",
   "devtools/client/definitions", true);
 loader.lazyRequireGetter(this, "SourceMapService",
   "devtools/client/framework/source-map-service", true);
+loader.lazyRequireGetter(this, "SourceMapURLService",
+  "devtools/client/framework/source-map-url-service", true);
 loader.lazyRequireGetter(this, "HUDService",
   "devtools/client/webconsole/hudservice");
 loader.lazyRequireGetter(this, "viewSource",
@@ -536,7 +538,8 @@ Toolbox.prototype = {
 
   /**
    * A common access point for the client-side mapping service for source maps that
-   * any panel can use.
+   * any panel can use.  This is a "low-level" API that connects to
+   * the source map worker.
    */
   get sourceMapService() {
     if (!Services.prefs.getBoolPref("devtools.source-map.client-service.enabled")) {
@@ -552,6 +555,25 @@ Toolbox.prototype = {
     return this._sourceMapService;
   },
 
+  /**
+   * Clients wishing to use source maps but that want the toolbox to
+   * track the source actor mapping can use this source map service.
+   * This is a higher-level service than the one returned by
+   * |sourceMapService|, in that it automatically tracks source actor
+   * IDs.
+   */
+  get sourceMapURLService() {
+    if (this._sourceMapURLService) {
+      return this._sourceMapURLService;
+    }
+    let sourceMaps = this.sourceMapService;
+    if (!sourceMaps) {
+      return null;
+    }
+    this._sourceMapURLService = new SourceMapURLService(this._target, sourceMaps);
+    return this._sourceMapURLService;
+  },
+
   // Return HostType id for telemetry
   _getTelemetryHostId: function () {
     switch (this.hostType) {
@@ -2299,6 +2321,11 @@ Toolbox.prototype = {
       this._deprecatedServerSourceMapService = null;
     }
 
+    if (this._sourceMapURLService) {
+      this._sourceMapURLService.destroy();
+      this._sourceMapURLService = null;
+    }
+
     if (this._sourceMapService) {
       this._sourceMapService.stopSourceMapWorker();
       this._sourceMapService = null;

devtools/client/shared/components/frame.js

@@ -52,26 +52,29 @@ module.exports = createClass({
   componentWillMount() {
     const sourceMapService = this.props.sourceMapService;
     if (sourceMapService) {
-      const source = this.getSource();
-      sourceMapService.subscribe(source, this.onSourceUpdated);
-    }
-  },
-
-  componentWillUnmount() {
-    const sourceMapService = this.props.sourceMapService;
-    if (sourceMapService) {
-      const source = this.getSource();
-      sourceMapService.unsubscribe(source, this.onSourceUpdated);
+      const { source, line, column } = this.props.frame;
+      sourceMapService.originalPositionFor(source, line, column)
+        .then(resolvedLocation => {
+          if (resolvedLocation) {
+            this.onSourceUpdated(resolvedLocation);
+          }
+        });
     }
   },
 
   /**
    * Component method to update the FrameView when a resolved location is available
-   * @param event
-   * @param location
+   * @param {Location} resolvedLocation
+   *        the resolved location as found via a source map
    */
-  onSourceUpdated(event, location, resolvedLocation) {
-    const frame = this.getFrame(resolvedLocation);
+  onSourceUpdated(resolvedLocation) {
+    const { sourceUrl, line, column } = resolvedLocation;
+    const frame = {
+      source: sourceUrl,
+      line,
+      column,
+      functionDisplayName: this.props.frame.functionDisplayName,
+    };
     this.setState({
       frame,
       isSourceMapped: true,
@@ -79,33 +82,17 @@ module.exports = createClass({
   },
 
   /**
-   * Utility method to convert the Frame object to the
-   * Source Object model required by SourceMapService
-   * @param frame
-   * @returns {{url: *, line: *, column: *}}
+   * Utility method to convert the Frame object model to the
+   * object model required by the onClick callback.
+   * @param Frame frame
+   * @returns {{url: *, line: *, column: *, functionDisplayName: *}}
    */
-  getSource(frame) {
-    frame = frame || this.props.frame;
+  getSourceForClick(frame) {
     const { source, line, column } = frame;
     return {
       url: source,
       line,
       column,
-    };
-  },
-
-  /**
-   * Utility method to convert the Source object model to the
-   * Frame object model required by FrameView class.
-   * @param source
-   * @returns {{source: *, line: *, column: *, functionDisplayName: *}}
-   */
-  getFrame(source) {
-    const { url, line, column } = source;
-    return {
-      source: url,
-      line,
-      column,
       functionDisplayName: this.props.frame.functionDisplayName,
     };
   },
@@ -224,7 +211,7 @@ module.exports = createClass({
       sourceEl = dom.a({
         onClick: e => {
           e.preventDefault();
-          onClick(this.getSource(frame));
+          onClick(this.getSourceForClick(frame));
         },
         href: source,
         className: "frame-link-source",

devtools/client/shared/components/test/mochitest/test_frame_01.html

@@ -287,12 +287,43 @@ window.onload = Task.async(function* () {
       tooltip: "View source in Debugger → http://www.cnn.com/:1",
     });
 
+    const resolvedLocation = {
+      sourceId: "whatever",
+      line: 23,
+      sourceUrl: "https://bugzilla.mozilla.org/original.js",
+    };
+    let mockSourceMapService = {
+      originalPositionFor: function () {
+	// Return a phony promise-like thing that resolves
+	// immediately.
+	return {
+	  then: function (consequence) {
+	    consequence(resolvedLocation);
+	  },
+	};
+      },
+    };
+    yield checkFrameComponent({
+      frame: {
+	line: 97,
+	source: "https://bugzilla.mozilla.org/bundle.js",
+      },
+      sourceMapService: mockSourceMapService,
+    }, {
+      file: "original.js",
+      line: resolvedLocation.line,
+      shouldLink: true,
+      tooltip: "View source in Debugger → https://bugzilla.mozilla.org/original.js:23",
+      source: "https://bugzilla.mozilla.org/original.js",
+    });
+
     function* checkFrameComponent(input, expected) {
       let props = Object.assign({ onClick: () => {} }, input);
       let frame = ReactDOM.render(Frame(props), window.document.body);
       let el = ReactDOM.findDOMNode(frame);
       let { source } = input.frame;
       checkFrameString(Object.assign({ el, source }, expected));
+      ReactDOM.unmountComponentAtNode(window.document.body);
     }
 
   } catch (e) {

devtools/client/shared/widgets/widgets.css

@@ -54,7 +54,6 @@
   display: none;
 }
 
-.variables-view-scope,
 .variable-or-property {
   -moz-user-focus: normal;
 }

devtools/client/webconsole/new-console-output/new-console-output-wrapper.js

@@ -92,8 +92,7 @@ NewConsoleOutputWrapper.prototype = {
             return panel.panelWin.NetMonitorController.inspectRequest(requestId);
           });
         },
-        sourceMapService:
-          this.toolbox ? this.toolbox._deprecatedServerSourceMapService : null,
+        sourceMapService: this.toolbox ? this.toolbox.sourceMapURLService : null,
         highlightDomElement: (grip, options = {}) => {
           return this.toolbox.highlighterUtils
             ? this.toolbox.highlighterUtils.highlightDomValueGrip(grip, options)

devtools/client/webconsole/new-console-output/selectors/messages.js

@@ -113,36 +113,14 @@ function matchSearchFilters(message, filters) {
     || isTextInParameters(text, message.parameters)
     // Look for a match in location.
     || isTextInFrame(text, message.frame)
-    // Look for a match in stacktrace.
-    || (
-      Array.isArray(message.stacktrace) &&
-      message.stacktrace.some(frame => isTextInFrame(text,
-        // isTextInFrame expect the properties of the frame object to be in the same
-        // order they are rendered in the Frame component.
-        {
-          functionName: frame.functionName ||
-            l10n.getStr("stacktrace.anonymousFunction"),
-          filename: frame.filename,
-          lineNumber: frame.lineNumber,
-          columnNumber: frame.columnNumber
-        }))
-    )
+    // Look for a match in net events.
+    || isTextInNetEvent(text, message.request)
+    // Look for a match in stack-trace.
+    || isTextInStackTrace(text, message.stacktrace)
     // Look for a match in messageText.
-    || (message.messageText !== null
-          && message.messageText.toLocaleLowerCase().includes(text.toLocaleLowerCase()))
-    // Look for a match in parameters. Currently only checks value grips.
-    || (message.parameters !== null
-        && message.parameters.join("").toLocaleLowerCase()
-            .includes(text.toLocaleLowerCase()))
+    || isTextInMessageText(text, message.messageText)
     // Look for a match in notes.
-    || (Array.isArray(message.notes) && message.notes.some(note =>
-          // Look for a match in location.
-          isTextInFrame(text, note.frame)
-          // Look for a match in messageBody.
-          || (note.messageBody !== null
-                && note.messageBody.toLocaleLowerCase()
-                     .includes(text.toLocaleLowerCase()))
-        ))
+    || isTextInNotes(text, message.notes)
   );
 }
 
@@ -173,6 +151,68 @@ function isTextInParameters(text, parameters) {
   );
 }
 
+/**
+ * Returns true if given text is included in provided net event grip.
+ */
+function isTextInNetEvent(text, request) {
+  if (!request) {
+    return false;
+  }
+
+  text = text.toLocaleLowerCase();
+
+  let method = request.method.toLocaleLowerCase();
+  let url = request.url.toLocaleLowerCase();
+  return method.includes(text) || url.includes(text);
+}
+
+/**
+ * Returns true if given text is included in provided stack trace.
+ */
+function isTextInStackTrace(text, stacktrace) {
+  if (!Array.isArray(stacktrace)) {
+    return false;
+  }
+
+  // isTextInFrame expect the properties of the frame object to be in the same
+  // order they are rendered in the Frame component.
+  return stacktrace.some(frame => isTextInFrame(text, {
+    functionName: frame.functionName || l10n.getStr("stacktrace.anonymousFunction"),
+    filename: frame.filename,
+    lineNumber: frame.lineNumber,
+    columnNumber: frame.columnNumber
+  }));
+}
+
+/**
+ * Returns true if given text is included in `messageText` field.
+ */
+function isTextInMessageText(text, messageText) {
+  if (!messageText) {
+    return false;
+  }
+
+  return messageText.toLocaleLowerCase().includes(text.toLocaleLowerCase());
+}
+
+/**
+ * Returns true if given text is included in notes.
+ */
+function isTextInNotes(text, notes) {
+  if (!Array.isArray(notes)) {
+    return false;
+  }
+
+  return notes.some(note =>
+    // Look for a match in location.
+    isTextInFrame(text, note.frame) ||
+    // Look for a match in messageBody.
+    (note.messageBody &&
+        note.messageBody.toLocaleLowerCase()
+          .includes(text.toLocaleLowerCase()))
+  );
+}
+
 /**
  * Get a flat array of all the grips and their properties.
  *

devtools/client/webconsole/new-console-output/test/store/search.test.js

@@ -58,6 +58,14 @@ describe("Searching in grips", () => {
       expect(getAllMessages(store.getState()).size).toEqual(1);
     });
   });
+
+  describe("Search in logs with net messages", () => {
+    it("matches on network messages", () => {
+      store.dispatch(actions.filterToggle("net"));
+      store.dispatch(actions.filterTextSet("get"));
+      expect(getAllMessages(store.getState()).size).toEqual(1);
+    });
+  });
 });
 
 function prepareBaseStore() {
@@ -70,6 +78,7 @@ function prepareBaseStore() {
     "console.log('myregex', /a.b.c/)",
     "console.map('mymap')",
     "console.log('myobject', {red: 'redValue', green: 'greenValue', blue: 'blueValue'});",
+    "GET request",
   ]);
 
   return store;

devtools/client/webconsole/webconsole.js

@@ -2626,7 +2626,7 @@ WebConsoleFrame.prototype = {
       frame: { source, line, column },
       showEmptyPathAsHost: true,
       onClick,
-      sourceMapService: toolbox ? toolbox._deprecatedServerSourceMapService : null,
+      sourceMapService: toolbox ? toolbox.sourceMapURLService : null,
     }), locationNode);
 
     return locationNode;

dom/base/Element.cpp

@@ -150,6 +150,8 @@
 #include "nsDOMStringMap.h"
 #include "DOMIntersectionObserver.h"
 
+#include "nsISpeculativeConnect.h"
+
 using namespace mozilla;
 using namespace mozilla::dom;
 
@@ -3066,6 +3068,13 @@ Element::PostHandleEventForLinks(EventChainPostVisitor& aVisitor)
 
           EventStateManager::SetActiveManager(
             aVisitor.mPresContext->EventStateManager(), this);
+
+          // OK, we're pretty sure we're going to load, so warm up a speculative
+          // connection to be sure we have one ready when we open the channel.
+          nsCOMPtr<nsISpeculativeConnect> sc =
+            do_QueryInterface(nsContentUtils::GetIOService());
+          nsCOMPtr<nsIInterfaceRequestor> ir = do_QueryInterface(handler);
+          sc->SpeculativeConnect2(absURI, NodePrincipal(), ir);
         }
       }
     }

dom/base/Navigator.cpp

@@ -69,6 +69,7 @@
 #include "nsStreamUtils.h"
 #include "WidgetUtils.h"
 #include "nsIPresentationService.h"
+#include "nsIScriptError.h"
 
 #include "mozilla/dom/MediaDevices.h"
 #include "MediaManager.h"
@@ -95,6 +96,7 @@
 
 #include "mozilla/EMEUtils.h"
 #include "mozilla/DetailedPromise.h"
+#include "mozilla/Unused.h"
 
 namespace mozilla {
 namespace dom {
@@ -2024,6 +2026,22 @@ Navigator::RequestMediaKeySystemAccess(const nsAString& aKeySystem,
   Telemetry::Accumulate(Telemetry::MEDIA_EME_SECURE_CONTEXT,
                         mWindow->IsSecureContext());
 
+  if (!mWindow->IsSecureContext()) {
+    nsIDocument* doc = mWindow->GetExtantDoc();
+    nsString uri;
+    if (doc) {
+      Unused << doc->GetDocumentURI(uri);
+    }
+    const char16_t* params[] = { uri.get() };
+    nsContentUtils::ReportToConsole(nsIScriptError::warningFlag,
+                                    NS_LITERAL_CSTRING("Media"),
+                                    doc,
+                                    nsContentUtils::eDOM_PROPERTIES,
+                                    "MediaEMEInsecureContextDeprecatedWarning",
+                                    params,
+                                    ArrayLength(params));
+  }
+
   nsCOMPtr<nsIGlobalObject> go = do_QueryInterface(mWindow);
   RefPtr<DetailedPromise> promise =
     DetailedPromise::Create(go, aRv,

dom/base/nsIDocument.h

@@ -1234,11 +1234,13 @@ public:
    * sheets for this document, returns the index that aSheet should
    * be inserted at to maintain document ordering.
    *
+   * Type T has to cast to StyleSheet*.
+   *
    * Defined in nsIDocumentInlines.h.
    */
   template<typename T>
-  size_t FindDocStyleSheetInsertionPoint(const nsTArray<RefPtr<T>>& aDocSheets,
-                                         T* aSheet);
+  size_t FindDocStyleSheetInsertionPoint(const nsTArray<T>& aDocSheets,
+                                         mozilla::StyleSheet* aSheet);
 
   /**
    * Get this document's CSSLoader.  This is guaranteed to not return null.

dom/base/nsIDocumentInlines.h

@@ -19,8 +19,8 @@ nsIDocument::GetBodyElement()
 template<typename T>
 size_t
 nsIDocument::FindDocStyleSheetInsertionPoint(
-    const nsTArray<RefPtr<T>>& aDocSheets,
-    T* aSheet)
+    const nsTArray<T>& aDocSheets,
+    mozilla::StyleSheet* aSheet)
 {
   nsStyleSheetService* sheetService = nsStyleSheetService::GetInstance();
 
@@ -30,13 +30,12 @@ nsIDocument::FindDocStyleSheetInsertionPoint(
   int32_t count = aDocSheets.Length();
   int32_t index;
   for (index = 0; index < count; index++) {
-    T* sheet = aDocSheets[index];
+    mozilla::StyleSheet* sheet = static_cast<mozilla::StyleSheet*>(
+      aDocSheets[index]);
     int32_t sheetDocIndex = GetIndexOfStyleSheet(sheet);
     if (sheetDocIndex > newDocIndex)
       break;
 
-    mozilla::StyleSheet* sheetHandle = sheet;
-
     // If the sheet is not owned by the document it can be an author
     // sheet registered at nsStyleSheetService or an additional author
     // sheet on the document, which means the new
@@ -45,11 +44,11 @@ nsIDocument::FindDocStyleSheetInsertionPoint(
       if (sheetService) {
         auto& authorSheets =
           *sheetService->AuthorStyleSheets(GetStyleBackendType());
-        if (authorSheets.IndexOf(sheetHandle) != authorSheets.NoIndex) {
+        if (authorSheets.IndexOf(sheet) != authorSheets.NoIndex) {
           break;
         }
       }
-      if (sheetHandle == GetFirstAdditionalAuthorSheet()) {
+      if (sheet == GetFirstAdditionalAuthorSheet()) {
         break;
       }
     }

dom/locales/en-US/chrome/dom/dom.properties

@@ -145,6 +145,8 @@ MediaRecorderMultiTracksNotSupported=MediaRecorder does not support recording mu
 MediaStreamAddTrackDifferentAudioChannel=MediaStreamTrack %S could not be added since it belongs to a different AudioChannel.
 # LOCALIZATION NOTE: Do not translate "MediaStream", "stop()" and "MediaStreamTrack"
 MediaStreamStopDeprecatedWarning=MediaStream.stop() is deprecated and will soon be removed. Use MediaStreamTrack.stop() instead.
+# LOCALIZATION NOTE: %S is the URL of the web page which is not served on HTTPS and thus is not encrypted and considered insecure.
+MediaEMEInsecureContextDeprecatedWarning=Using Encrypted Media Extensions at %S on an insecure (i.e. non-HTTPS) context is deprecated and will soon be removed. You should consider switching to a secure origin such as HTTPS.
 # LOCALIZATION NOTE: Do not translate "DOMException", "code" and "name"
 DOMExceptionCodeWarning=Use of DOMException’s code attribute is deprecated. Use name instead.
 # LOCALIZATION NOTE: Do not translate "__exposedProps__"

js/src/gc/Tracer.cpp

@@ -331,26 +331,6 @@ JS_GetTraceThingInfo(char* buf, size_t bufsize, JSTracer* trc, void* thing,
         return;
 
     switch (kind) {
-      case JS::TraceKind::Object:
-      {
-        name = static_cast<JSObject*>(thing)->getClass()->name;
-        break;
-      }
-
-      case JS::TraceKind::Script:
-        name = "script";
-        break;
-
-      case JS::TraceKind::String:
-        name = ((JSString*)thing)->isDependent()
-               ? "substring"
-               : "string";
-        break;
-
-      case JS::TraceKind::Symbol:
-        name = "symbol";
-        break;
-
       case JS::TraceKind::BaseShape:
         name = "base_shape";
         break;
@@ -363,12 +343,44 @@ JS_GetTraceThingInfo(char* buf, size_t bufsize, JSTracer* trc, void* thing,
         name = "lazyscript";
         break;
 
+      case JS::TraceKind::Null:
+        name = "null_pointer";
+        break;
+
+      case JS::TraceKind::Object:
+      {
+        name = static_cast<JSObject*>(thing)->getClass()->name;
+        break;
+      }
+
+      case JS::TraceKind::ObjectGroup:
+        name = "object_group";
+        break;
+
+      case JS::TraceKind::RegExpShared:
+        name = "reg_exp_shared";
+        break;
+
+      case JS::TraceKind::Scope:
+        name = "scope";
+        break;
+
+      case JS::TraceKind::Script:
+        name = "script";
+        break;
+
       case JS::TraceKind::Shape:
         name = "shape";
         break;
 
-      case JS::TraceKind::ObjectGroup:
-        name = "object_group";
+      case JS::TraceKind::String:
+        name = ((JSString*)thing)->isDependent()
+               ? "substring"
+               : "string";
+        break;
+
+      case JS::TraceKind::Symbol:
+        name = "symbol";
         break;
 
       default:

js/src/jit/ExecutableAllocator.h

@@ -259,7 +259,7 @@ class ExecutableAllocator
     {
         __clear_cache(code, reinterpret_cast<char*>(code) + size);
     }
-#elif defined(JS_CODEGEN_ARM) && defined(XP_IOS)
+#elif (defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)) && defined(XP_IOS)
     static void cacheFlush(void* code, size_t size)
     {
         sys_icache_invalidate(code, size);
@@ -297,7 +297,7 @@ class ExecutableAllocator
                 : "r0", "r1", "r2");
         }
     }
-#elif defined(JS_CODEGEN_ARM64) && (defined(__linux__) || defined(ANDROID)) && defined(__GNUC__)
+#elif defined(JS_CODEGEN_ARM64)
     static void cacheFlush(void* code, size_t size)
     {
         vixl::CPU::EnsureIAndDCacheCoherency(code, size);

js/src/wasm/WasmSignalHandlers.cpp

@@ -108,7 +108,19 @@ class AutoSetHandlingSegFault
 # define R12_sig(p) ((p)->sc_r12)
 # define R13_sig(p) ((p)->sc_r13)
 # define R14_sig(p) ((p)->sc_r14)
-# define R15_sig(p) ((p)->sc_r15)
+# if defined(__arm__)
+#  define R15_sig(p) ((p)->sc_pc)
+# else
+#  define R15_sig(p) ((p)->sc_r15)
+# endif
+# if defined(__aarch64__)
+#  define EPC_sig(p) ((p)->sc_elr)
+#  define RFP_sig(p) ((p)->sc_x[29])
+# endif
+# if defined(__mips__)
+#  define EPC_sig(p) ((p)->sc_pc)
+#  define RFP_sig(p) ((p)->sc_regs[30])
+# endif
 #elif defined(__linux__) || defined(__sun)
 # if defined(__linux__)
 #  define XMM_sig(p,i) ((p)->uc_mcontext.fpregs->_xmm[i])
@@ -171,6 +183,14 @@ class AutoSetHandlingSegFault
 # define R13_sig(p) ((p)->uc_mcontext.__gregs[_REG_R13])
 # define R14_sig(p) ((p)->uc_mcontext.__gregs[_REG_R14])
 # define R15_sig(p) ((p)->uc_mcontext.__gregs[_REG_R15])
+# if defined(__aarch64__)
+#  define EPC_sig(p) ((p)->uc_mcontext.__gregs[_REG_PC])
+#  define RFP_sig(p) ((p)->uc_mcontext.__gregs[_REG_X29])
+# endif
+# if defined(__mips__)
+#  define EPC_sig(p) ((p)->uc_mcontext.__gregs[_REG_EPC])
+#  define RFP_sig(p) ((p)->uc_mcontext.__gregs[_REG_S8])
+# endif
 #elif defined(__DragonFly__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 # if defined(__DragonFly__)
 #  define XMM_sig(p,i) (((union savefpu*)(p)->uc_mcontext.mc_fpregs)->sv_xmm.sv_xmm[i])
@@ -200,6 +220,14 @@ class AutoSetHandlingSegFault
 # else
 #  define R15_sig(p) ((p)->uc_mcontext.mc_r15)
 # endif
+# if defined(__FreeBSD__) && defined(__aarch64__)
+#  define EPC_sig(p) ((p)->uc_mcontext.mc_gpregs.gp_elr)
+#  define RFP_sig(p) ((p)->uc_mcontext.mc_gpregs.gp_x[29])
+# endif
+# if defined(__FreeBSD__) && defined(__mips__)
+#  define EPC_sig(p) ((p)->uc_mcontext.mc_pc)
+#  define RFP_sig(p) ((p)->uc_mcontext.mc_regs[30])
+# endif
 #elif defined(XP_DARWIN)
 # define EIP_sig(p) ((p)->uc_mcontext->__ss.__eip)
 # define EBP_sig(p) ((p)->uc_mcontext->__ss.__ebp)

layout/base/AccessibleCaretManager.cpp

@@ -1029,8 +1029,10 @@ AccessibleCaretManager::FlushLayout() const
 
 nsIFrame*
 AccessibleCaretManager::GetFrameForFirstRangeStartOrLastRangeEnd(
-  nsDirection aDirection, int32_t* aOutOffset, nsINode** aOutNode,
-  int32_t* aOutNodeOffset) const
+  nsDirection aDirection,
+  int32_t* aOutOffset,
+  nsIContent** aOutContent,
+  int32_t* aOutContentOffset) const
 {
   if (!mPresShell) {
     return nullptr;
@@ -1096,11 +1098,11 @@ AccessibleCaretManager::GetFrameForFirstRangeStartOrLastRangeEnd(
   }
 
   if (startFrame) {
-    if (aOutNode) {
-      *aOutNode = startNode.get();
+    if (aOutContent) {
+      startContent.forget(aOutContent);
     }
-    if (aOutNodeOffset) {
-      *aOutNodeOffset = nodeOffset;
+    if (aOutContentOffset) {
+      *aOutContentOffset = nodeOffset;
     }
   }
 
@@ -1119,16 +1121,17 @@ AccessibleCaretManager::RestrictCaretDraggingOffsets(
 
   nsDirection dir = mActiveCaret == mFirstCaret.get() ? eDirPrevious : eDirNext;
   int32_t offset = 0;
-  nsINode* node = nullptr;
+  nsCOMPtr<nsIContent> content;
   int32_t contentOffset = 0;
   nsIFrame* frame =
-    GetFrameForFirstRangeStartOrLastRangeEnd(dir, &offset, &node, &contentOffset);
+    GetFrameForFirstRangeStartOrLastRangeEnd(dir, &offset,
+                                             getter_AddRefs(content),
+                                             &contentOffset);
 
   if (!frame) {
     return false;
   }
 
-  nsCOMPtr<nsIContent> content = do_QueryInterface(node);
 
   // Compare the active caret's new position (aOffsets) to the inactive caret's
   // position.

layout/base/AccessibleCaretManager.h

@@ -181,11 +181,13 @@ protected:
 
   // If aDirection is eDirNext, get the frame for the range start in the first
   // range from the current selection, and return the offset into that frame as
-  // well as the range start node and the node offset. Otherwise, get the frame
-  // and offset for the range end in the last range instead.
+  // well as the range start content and the content offset. Otherwise, get the
+  // frame and the offset for the range end in the last range instead.
   nsIFrame* GetFrameForFirstRangeStartOrLastRangeEnd(
-    nsDirection aDirection, int32_t* aOutOffset, nsINode** aOutNode = nullptr,
-    int32_t* aOutNodeOffset = nullptr) const;
+    nsDirection aDirection,
+    int32_t* aOutOffset,
+    nsIContent** aOutContent = nullptr,
+    int32_t* aOutContentOffset = nullptr) const;
 
   nsresult DragCaretInternal(const nsPoint& aPoint);
   nsPoint AdjustDragBoundary(const nsPoint& aPoint) const;

layout/base/nsDocumentViewer.cpp

@@ -18,7 +18,7 @@
 #include "nsIContentViewerContainer.h"
 #include "nsIContentViewer.h"
 #include "nsIDocumentViewerPrint.h"
-#include "nsIDOMBeforeUnloadEvent.h"
+#include "mozilla/dom/BeforeUnloadEvent.h"
 #include "nsIDocument.h"
 #include "nsPresContext.h"
 #include "nsIPresShell.h"
@@ -1185,12 +1185,13 @@ nsDocumentViewer::PermitUnloadInternal(bool *aShouldPrompt,
 
   // Now, fire an BeforeUnload event to the document and see if it's ok
   // to unload...
-  nsCOMPtr<nsIDOMDocument> domDoc = do_QueryInterface(mDocument);
-  nsCOMPtr<nsIDOMEvent> event;
-  domDoc->CreateEvent(NS_LITERAL_STRING("beforeunloadevent"),
-                      getter_AddRefs(event));
-  nsCOMPtr<nsIDOMBeforeUnloadEvent> beforeUnload = do_QueryInterface(event);
-  NS_ENSURE_STATE(beforeUnload);
+  nsIPresShell* shell = mDocument->GetShell();
+  nsPresContext* presContext = nullptr;
+  if (shell) {
+    presContext = shell->GetPresContext();
+  }
+  RefPtr<BeforeUnloadEvent> event =
+    new BeforeUnloadEvent(mDocument, presContext, nullptr);
   event->InitEvent(NS_LITERAL_STRING("beforeunload"), false, true);
 
   // Dispatching to |window|, but using |document| as the target.
@@ -1222,7 +1223,7 @@ nsDocumentViewer::PermitUnloadInternal(bool *aShouldPrompt,
 
   nsCOMPtr<nsIDocShell> docShell(mContainer);
   nsAutoString text;
-  beforeUnload->GetReturnValue(text);
+  event->GetReturnValue(text);
 
   // NB: we nullcheck mDocument because it might now be dead as a result of
   // the event being dispatched.

layout/forms/nsComboboxControlFrame.cpp

@@ -779,9 +779,10 @@ nsComboboxControlFrame::GetIntrinsicISize(nsRenderingContext* aRenderingContext,
   }
 
   // add room for the dropmarker button if there is one
-  if ((!IsThemed() ||
+  const nsStyleDisplay* disp = StyleDisplay();
+  if ((!IsThemed(disp) ||
        presContext->GetTheme()->ThemeNeedsComboboxDropmarker()) &&
-      StyleDisplay()->UsedAppearance() != NS_THEME_NONE) {
+      disp->UsedAppearance() != NS_THEME_NONE) {
     displayISize += scrollbarWidth;
   }
 

layout/forms/nsRangeFrame.cpp

@@ -258,7 +258,8 @@ nsRangeFrame::BuildDisplayList(nsDisplayListBuilder*   aBuilder,
                                const nsRect&           aDirtyRect,
                                const nsDisplayListSet& aLists)
 {
-  if (IsThemed()) {
+  const nsStyleDisplay* disp = StyleDisplay();
+  if (IsThemed(disp)) {
     DisplayBorderBackgroundOutline(aBuilder, aLists);
     // Only create items for the thumb. Specifically, we do not want
     // the track to paint, since *our* background is used to paint
@@ -298,7 +299,6 @@ nsRangeFrame::BuildDisplayList(nsDisplayListBuilder*   aBuilder,
     return;
   }
 
-  const nsStyleDisplay *disp = StyleDisplay();
   if (IsThemed(disp) &&
       PresContext()->GetTheme()->ThemeDrawsFocusForWidget(disp->UsedAppearance())) {
     return; // the native theme displays its own visual indication of focus

layout/reftests/bugs/1348481-1.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<link id="target" rel="stylesheet" href="data:text/css,">
+<link rel="stylesheet" href="data:text/css,div { color: green }">
+<link rel="stylesheet" href="data:text/css,">
+<div>This should be green</div>
+<script>
+  onload = function() {
+    var link = document.createElement("link");
+    link.rel = "stylesheet";
+    link.href = "data:text/css,div { color: red; }";
+    link.title = "turnitred";
+    var target = document.getElementById("target");
+    target.parentNode.insertBefore(link, target);
+  }
+</script>

layout/reftests/bugs/1348481-2.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<link rel="stylesheet" href="data:text/css,div { color: green }">
+<link rel="stylesheet" href="data:text/css,div { color: green }">
+<div>This should be green</div>
+<script>
+  onload = function() {
+    var links = document.getElementsByTagName("link");
+    links[0].remove();
+  }
+</script>

layout/reftests/bugs/1348481-3.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<link rel="stylesheet" href="data:text/css,div { color: green }">
+<link rel="stylesheet" href="data:text/css,div { color: green }">
+<div>This should be green</div>
+<script>
+  onload = function() {
+    var links = document.getElementsByTagName("link");
+    links[0].sheet.cssRules[0].style.color = "red";
+  }
+</script>

layout/reftests/bugs/1348481-ref.html

@@ -0,0 +1,7 @@
+<!DOCTYPE html>
+<style>
+div {
+  color: green;
+}
+</style>
+<div>This should be green</div>

layout/reftests/bugs/reftest.list

@@ -1361,7 +1361,7 @@ pref(browser.display.focus_ring_width,1) == 491180-2.html 491180-2-ref.html
 == 491323-1.xul 491323-1-ref.xul
 == 492239-1.xul 492239-1-ref.xul
 == 492661-1.html 492661-1-ref.html
-fails-if(stylo) == 493968-1.html 493968-1-ref.html
+== 493968-1.html 493968-1-ref.html
 == 494667-1.html 494667-1-ref.html
 == 494667-2.html 494667-2-ref.html
 == 495274-1.html 495274-1-ref.html
@@ -1990,6 +1990,9 @@ fuzzy(2,40000) == 1316719-1c.html 1316719-1-ref.html
 skip-if(Android) != 1318769-1.html 1318769-1-ref.html
 fails-if(stylo) == 1322512-1.html 1322512-1-ref.html
 == 1330051.svg 1330051-ref.svg
+== 1348481-1.html 1348481-ref.html
+== 1348481-2.html 1348481-ref.html
+fails-if(stylo) == 1348481-3.html 1348481-ref.html
 == 1352464-1.html 1352464-1-ref.html
 == 1358375-1.html 1358375-ref.html
 == 1358375-2.html 1358375-ref.html

layout/style/ServoBindingList.h

@@ -50,14 +50,25 @@ SERVO_BINDING_FUNC(Servo_StyleSet_RebuildData, void,
                    RawServoStyleSetBorrowed set)
 SERVO_BINDING_FUNC(Servo_StyleSet_Drop, void, RawServoStyleSetOwned set)
 SERVO_BINDING_FUNC(Servo_StyleSet_AppendStyleSheet, void,
-                   RawServoStyleSetBorrowed set, RawServoStyleSheetBorrowed sheet, bool flush)
+                   RawServoStyleSetBorrowed set,
+                   RawServoStyleSheetBorrowed sheet,
+                   uint32_t unique_id,
+                   bool flush)
 SERVO_BINDING_FUNC(Servo_StyleSet_PrependStyleSheet, void,
-                   RawServoStyleSetBorrowed set, RawServoStyleSheetBorrowed sheet, bool flush)
+                   RawServoStyleSetBorrowed set,
+                   RawServoStyleSheetBorrowed sheet,
+                   uint32_t unique_id,
+                   bool flush)
 SERVO_BINDING_FUNC(Servo_StyleSet_RemoveStyleSheet, void,
-                   RawServoStyleSetBorrowed set, RawServoStyleSheetBorrowed sheet, bool flush)
+                   RawServoStyleSetBorrowed set,
+                   uint32_t unique_id,
+                   bool flush)
 SERVO_BINDING_FUNC(Servo_StyleSet_InsertStyleSheetBefore, void,
-                   RawServoStyleSetBorrowed set, RawServoStyleSheetBorrowed sheet,
-                   RawServoStyleSheetBorrowed reference, bool flush)
+                   RawServoStyleSetBorrowed set,
+                   RawServoStyleSheetBorrowed sheet,
+                   uint32_t unique_id,
+                   uint32_t before_unique_id,
+                   bool flush)
 SERVO_BINDING_FUNC(Servo_StyleSet_FlushStyleSheets, void, RawServoStyleSetBorrowed set)
 SERVO_BINDING_FUNC(Servo_StyleSet_NoteStyleSheetsChanged, void,
                    RawServoStyleSetBorrowed set, bool author_style_disabled)

layout/style/ServoStyleSet.cpp

@@ -31,6 +31,7 @@ using namespace mozilla::dom;
 ServoStyleSet::ServoStyleSet()
   : mPresContext(nullptr)
   , mBatching(0)
+  , mUniqueIDCounter(0)
   , mAllowResolveStaleStyles(false)
   , mAuthorStyleDisabled(false)
 {
@@ -51,15 +52,22 @@ ServoStyleSet::Init(nsPresContext* aPresContext)
 
   // Now that we have an mRawSet, go ahead and notify about whatever stylesheets
   // we have so far.
-  for (auto& sheetArray : mSheets) {
-    for (auto& sheet : sheetArray) {
+  for (auto& entryArray : mEntries) {
+    for (auto& entry : entryArray) {
       // There's no guarantee this will create a list on the servo side whose
       // ordering matches the list that would have been created had all those
-      // sheets been appended/prepended/etc after we had mRawSet.  But hopefully
-      // that's OK (e.g. because servo doesn't care about the relative ordering
-      // of sheets from different cascade levels in the list?).
-      MOZ_ASSERT(sheet->RawSheet(), "We should only append non-null raw sheets.");
-      Servo_StyleSet_AppendStyleSheet(mRawSet.get(), sheet->RawSheet(), false);
+      // sheets been appended/prepended/etc after we had mRawSet. That's okay
+      // because Servo only needs to maintain relative ordering within a sheet
+      // type, which this preserves.
+
+      // Set the uniqueIDs as we go.
+      entry.uniqueID = ++mUniqueIDCounter;
+
+      MOZ_ASSERT(entry.sheet->RawSheet(), "We should only append non-null raw sheets.");
+      Servo_StyleSet_AppendStyleSheet(mRawSet.get(),
+                                      entry.sheet->RawSheet(),
+                                      entry.uniqueID,
+                                      false);
     }
   }
 
@@ -547,14 +555,20 @@ ServoStyleSet::AppendStyleSheet(SheetType aType,
   MOZ_ASSERT(aSheet);
   MOZ_ASSERT(aSheet->IsApplicable());
   MOZ_ASSERT(nsStyleSet::IsCSSSheetType(aType));
-
   MOZ_ASSERT(aSheet->RawSheet(), "Raw sheet should be in place before insertion.");
-  mSheets[aType].RemoveElement(aSheet);
-  mSheets[aType].AppendElement(aSheet);
+
+  // If we were already tracking aSheet, the newUniqueID will be the same
+  // as the oldUniqueID. In that case, Servo will remove aSheet from its
+  // original position as part of the call to Servo_StyleSet_AppendStyleSheet.
+  uint32_t oldUniqueID = RemoveSheetOfType(aType, aSheet);
+  uint32_t newUniqueID = AppendSheetOfType(aType, aSheet, oldUniqueID);
 
   if (mRawSet) {
     // Maintain a mirrored list of sheets on the servo side.
-    Servo_StyleSet_AppendStyleSheet(mRawSet.get(), aSheet->RawSheet(), !mBatching);
+    Servo_StyleSet_AppendStyleSheet(mRawSet.get(),
+                                    aSheet->RawSheet(),
+                                    newUniqueID,
+                                    !mBatching);
   }
 
   return NS_OK;
@@ -567,14 +581,20 @@ ServoStyleSet::PrependStyleSheet(SheetType aType,
   MOZ_ASSERT(aSheet);
   MOZ_ASSERT(aSheet->IsApplicable());
   MOZ_ASSERT(nsStyleSet::IsCSSSheetType(aType));
-
   MOZ_ASSERT(aSheet->RawSheet(), "Raw sheet should be in place before insertion.");
-  mSheets[aType].RemoveElement(aSheet);
-  mSheets[aType].InsertElementAt(0, aSheet);
+
+  // If we were already tracking aSheet, the newUniqueID will be the same
+  // as the oldUniqueID. In that case, Servo will remove aSheet from its
+  // original position as part of the call to Servo_StyleSet_PrependStyleSheet.
+  uint32_t oldUniqueID = RemoveSheetOfType(aType, aSheet);
+  uint32_t newUniqueID = PrependSheetOfType(aType, aSheet, oldUniqueID);
 
   if (mRawSet) {
     // Maintain a mirrored list of sheets on the servo side.
-    Servo_StyleSet_PrependStyleSheet(mRawSet.get(), aSheet->RawSheet(), !mBatching);
+    Servo_StyleSet_PrependStyleSheet(mRawSet.get(),
+                                     aSheet->RawSheet(),
+                                     newUniqueID,
+                                     !mBatching);
   }
 
   return NS_OK;
@@ -587,11 +607,10 @@ ServoStyleSet::RemoveStyleSheet(SheetType aType,
   MOZ_ASSERT(aSheet);
   MOZ_ASSERT(nsStyleSet::IsCSSSheetType(aType));
 
-  mSheets[aType].RemoveElement(aSheet);
-
-  if (mRawSet) {
+  uint32_t uniqueID = RemoveSheetOfType(aType, aSheet);
+  if (mRawSet && uniqueID) {
     // Maintain a mirrored list of sheets on the servo side.
-    Servo_StyleSet_RemoveStyleSheet(mRawSet.get(), aSheet->RawSheet(), !mBatching);
+    Servo_StyleSet_RemoveStyleSheet(mRawSet.get(), uniqueID, !mBatching);
   }
 
   return NS_OK;
@@ -606,19 +625,23 @@ ServoStyleSet::ReplaceSheets(SheetType aType,
   // to express. If the need ever arises, we can easily make this more efficent,
   // probably by aligning the representations better between engines.
 
+  // Remove all the existing sheets first.
   if (mRawSet) {
-    for (ServoStyleSheet* sheet : mSheets[aType]) {
-      Servo_StyleSet_RemoveStyleSheet(mRawSet.get(), sheet->RawSheet(), false);
+    for (const Entry& entry : mEntries[aType]) {
+      Servo_StyleSet_RemoveStyleSheet(mRawSet.get(), entry.uniqueID, false);
     }
   }
+  mEntries[aType].Clear();
 
-  mSheets[aType].Clear();
-  mSheets[aType].AppendElements(aNewSheets);
-
-  if (mRawSet) {
-    for (ServoStyleSheet* sheet : mSheets[aType]) {
+  // Add in all the new sheets.
+  for (auto& sheet : aNewSheets) {
+    uint32_t uniqueID = AppendSheetOfType(aType, sheet);
+    if (mRawSet) {
       MOZ_ASSERT(sheet->RawSheet(), "Raw sheet should be in place before replacement.");
-      Servo_StyleSet_AppendStyleSheet(mRawSet.get(), sheet->RawSheet(), false);
+      Servo_StyleSet_AppendStyleSheet(mRawSet.get(),
+                                      sheet->RawSheet(),
+                                      uniqueID,
+                                      false);
     }
   }
 
@@ -637,21 +660,31 @@ ServoStyleSet::InsertStyleSheetBefore(SheetType aType,
   MOZ_ASSERT(aNewSheet);
   MOZ_ASSERT(aReferenceSheet);
   MOZ_ASSERT(aNewSheet->IsApplicable());
-
-  mSheets[aType].RemoveElement(aNewSheet);
-  size_t idx = mSheets[aType].IndexOf(aReferenceSheet);
-  if (idx == mSheets[aType].NoIndex) {
-    return NS_ERROR_INVALID_ARG;
-  }
+  MOZ_ASSERT(aNewSheet != aReferenceSheet, "Can't place sheet before itself.");
+  MOZ_ASSERT(aNewSheet->RawSheet(), "Raw sheet should be in place before insertion.");
   MOZ_ASSERT(aReferenceSheet->RawSheet(), "Reference sheet should have a raw sheet.");
 
-  MOZ_ASSERT(aNewSheet->RawSheet(), "Raw sheet should be in place before insertion.");
-  mSheets[aType].InsertElementAt(idx, aNewSheet);
+  uint32_t beforeUniqueID = FindSheetOfType(aType, aReferenceSheet);
+  if (beforeUniqueID == 0) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  // If we were already tracking aNewSheet, the newUniqueID will be the same
+  // as the oldUniqueID. In that case, Servo will remove aNewSheet from its
+  // original position as part of the call to Servo_StyleSet_InsertStyleSheetBefore.
+  uint32_t oldUniqueID = RemoveSheetOfType(aType, aNewSheet);
+  uint32_t newUniqueID = InsertSheetOfType(aType,
+                                           aNewSheet,
+                                           beforeUniqueID,
+                                           oldUniqueID);
 
   if (mRawSet) {
     // Maintain a mirrored list of sheets on the servo side.
-    Servo_StyleSet_InsertStyleSheetBefore(mRawSet.get(), aNewSheet->RawSheet(),
-                                          aReferenceSheet->RawSheet(), !mBatching);
+    Servo_StyleSet_InsertStyleSheetBefore(mRawSet.get(),
+                                          aNewSheet->RawSheet(),
+                                          newUniqueID,
+                                          beforeUniqueID,
+                                          !mBatching);
   }
 
   return NS_OK;
@@ -661,7 +694,7 @@ int32_t
 ServoStyleSet::SheetCount(SheetType aType) const
 {
   MOZ_ASSERT(nsStyleSet::IsCSSSheetType(aType));
-  return mSheets[aType].Length();
+  return mEntries[aType].Length();
 }
 
 ServoStyleSheet*
@@ -669,7 +702,7 @@ ServoStyleSet::StyleSheetAt(SheetType aType,
                             int32_t aIndex) const
 {
   MOZ_ASSERT(nsStyleSet::IsCSSSheetType(aType));
-  return mSheets[aType][aIndex];
+  return mEntries[aType][aIndex].sheet;
 }
 
 nsresult
@@ -687,23 +720,39 @@ ServoStyleSet::AddDocStyleSheet(ServoStyleSheet* aSheet,
 
   RefPtr<StyleSheet> strong(aSheet);
 
-  nsTArray<RefPtr<ServoStyleSheet>>& sheetsArray = mSheets[SheetType::Doc];
-
-  sheetsArray.RemoveElement(aSheet);
+  uint32_t oldUniqueID = RemoveSheetOfType(SheetType::Doc, aSheet);
 
   size_t index =
-    aDocument->FindDocStyleSheetInsertionPoint(sheetsArray, aSheet);
-  sheetsArray.InsertElementAt(index, aSheet);
+    aDocument->FindDocStyleSheetInsertionPoint(mEntries[SheetType::Doc], aSheet);
 
-  if (mRawSet) {
-    // Maintain a mirrored list of sheets on the servo side.
-    ServoStyleSheet* followingSheet = sheetsArray.SafeElementAt(index + 1);
-    if (followingSheet) {
-      MOZ_ASSERT(followingSheet->RawSheet(), "Every mSheets element should have a raw sheet");
-      Servo_StyleSet_InsertStyleSheetBefore(mRawSet.get(), aSheet->RawSheet(),
-                                            followingSheet->RawSheet(), !mBatching);
-    } else {
-      Servo_StyleSet_AppendStyleSheet(mRawSet.get(), aSheet->RawSheet(), !mBatching);
+  if (index < mEntries[SheetType::Doc].Length()) {
+    // This case is insert before.
+    uint32_t beforeUniqueID = mEntries[SheetType::Doc][index].uniqueID;
+    uint32_t newUniqueID = InsertSheetOfType(SheetType::Doc,
+                                             aSheet,
+                                             beforeUniqueID,
+                                             oldUniqueID);
+
+    if (mRawSet) {
+      // Maintain a mirrored list of sheets on the servo side.
+      Servo_StyleSet_InsertStyleSheetBefore(mRawSet.get(),
+                                            aSheet->RawSheet(),
+                                            newUniqueID,
+                                            beforeUniqueID,
+                                            !mBatching);
+    }
+  } else {
+    // This case is append.
+    uint32_t newUniqueID = AppendSheetOfType(SheetType::Doc,
+                                             aSheet,
+                                             oldUniqueID);
+
+    if (mRawSet) {
+      // Maintain a mirrored list of sheets on the servo side.
+      Servo_StyleSet_AppendStyleSheet(mRawSet.get(),
+                                      aSheet->RawSheet(),
+                                      newUniqueID,
+                                      !mBatching);
     }
   }
 
@@ -985,4 +1034,69 @@ ServoStyleSet::ResolveForDeclarations(
                                                aDeclarations).Consume();
 }
 
+uint32_t
+ServoStyleSet::FindSheetOfType(SheetType aType,
+                               ServoStyleSheet* aSheet)
+{
+  for (const auto& entry : mEntries[aType]) {
+    if (entry.sheet == aSheet) {
+      return entry.uniqueID;
+    }
+  }
+  return 0;
+}
+
+uint32_t
+ServoStyleSet::PrependSheetOfType(SheetType aType,
+                                  ServoStyleSheet* aSheet,
+                                  uint32_t aReuseUniqueID)
+{
+  Entry* entry = mEntries[aType].InsertElementAt(0);
+  entry->uniqueID = aReuseUniqueID ? aReuseUniqueID : ++mUniqueIDCounter;
+  entry->sheet = aSheet;
+  return entry->uniqueID;
+}
+
+uint32_t
+ServoStyleSet::AppendSheetOfType(SheetType aType,
+                                 ServoStyleSheet* aSheet,
+                                 uint32_t aReuseUniqueID)
+{
+  Entry* entry = mEntries[aType].AppendElement();
+  entry->uniqueID = aReuseUniqueID ? aReuseUniqueID : ++mUniqueIDCounter;
+  entry->sheet = aSheet;
+  return entry->uniqueID;
+}
+
+uint32_t
+ServoStyleSet::InsertSheetOfType(SheetType aType,
+                                 ServoStyleSheet* aSheet,
+                                 uint32_t aBeforeUniqueID,
+                                 uint32_t aReuseUniqueID)
+{
+  for (uint32_t i = 0; i < mEntries[aType].Length(); ++i) {
+    if (mEntries[aType][i].uniqueID == aBeforeUniqueID) {
+      Entry* entry = mEntries[aType].InsertElementAt(i);
+      entry->uniqueID = aReuseUniqueID ? aReuseUniqueID : ++mUniqueIDCounter;
+      entry->sheet = aSheet;
+      return entry->uniqueID;
+    }
+  }
+  return 0;
+}
+
+uint32_t
+ServoStyleSet::RemoveSheetOfType(SheetType aType,
+                                 ServoStyleSheet* aSheet)
+{
+  for (uint32_t i = 0; i < mEntries[aType].Length(); ++i) {
+    if (mEntries[aType][i].sheet == aSheet) {
+      uint32_t uniqueID = mEntries[aType][i].uniqueID;
+      mEntries[aType].RemoveElementAt(i);
+      return uniqueID;
+    }
+  }
+  return 0;
+}
+
 bool ServoStyleSet::sInServoTraversal = false;

layout/style/ServoStyleSet.h

@@ -352,11 +352,40 @@ private:
   already_AddRefed<ServoComputedValues> ResolveStyleLazily(dom::Element* aElement,
                                                            nsIAtom* aPseudoTag);
 
+  uint32_t FindSheetOfType(SheetType aType,
+                           ServoStyleSheet* aSheet);
+
+  uint32_t PrependSheetOfType(SheetType aType,
+                              ServoStyleSheet* aSheet,
+                              uint32_t aReuseUniqueID = 0);
+
+  uint32_t AppendSheetOfType(SheetType aType,
+                             ServoStyleSheet* aSheet,
+                             uint32_t aReuseUniqueID = 0);
+
+  uint32_t InsertSheetOfType(SheetType aType,
+                             ServoStyleSheet* aSheet,
+                             uint32_t aBeforeUniqueID,
+                             uint32_t aReuseUniqueID = 0);
+
+  uint32_t RemoveSheetOfType(SheetType aType,
+                             ServoStyleSheet* aSheet);
+
+  struct Entry {
+    uint32_t uniqueID;
+    RefPtr<ServoStyleSheet> sheet;
+
+    // Provide a cast operator to simplify calling
+    // nsIDocument::FindDocStyleSheetInsertionPoint.
+    operator ServoStyleSheet*() const { return sheet; }
+  };
+
   nsPresContext* mPresContext;
   UniquePtr<RawServoStyleSet> mRawSet;
   EnumeratedArray<SheetType, SheetType::Count,
-                  nsTArray<RefPtr<ServoStyleSheet>>> mSheets;
+                  nsTArray<Entry>> mEntries;
   int32_t mBatching;
+  uint32_t mUniqueIDCounter;
   bool mAllowResolveStaleStyles;
   bool mAuthorStyleDisabled;
 

layout/style/StyleSetHandleInlines.h

@@ -185,7 +185,7 @@ StyleSetHandle::Ptr::InsertStyleSheetBefore(SheetType aType,
   FORWARD_CONCRETE(
     InsertStyleSheetBefore,
     (aType, aNewSheet->AsGecko(), aReferenceSheet->AsGecko()),
-    (aType, aReferenceSheet->AsServo(), aReferenceSheet->AsServo()));
+    (aType, aNewSheet->AsServo(), aReferenceSheet->AsServo()));
 }
 
 int32_t

layout/style/test/stylo-failures.md

@@ -30,7 +30,7 @@ to mochitest command.
 ## Failures
 
 * Media query support:
-  * test_media_queries.html [156]
+  * test_media_queries.html [38]
   * test_media_queries_dynamic.html [6]
   * test_media_queries_dynamic_xbl.html [2]
   * test_webkit_device_pixel_ratio.html: -webkit-device-pixel-ratio [3]
@@ -128,13 +128,6 @@ to mochitest command.
     * test_property_syntax_errors.html `font-variant-alternates` [2]
     * test_value_storage.html `font-variant` [176]
     * test_specified_value_serialization.html `bug-721136` [1]
-  * touch-action property servo/servo#16372
-    * test_compute_data_with_start_struct.html `touch-action` [2]
-    * test_inherit_computation.html `touch-action` [2]
-    * test_inherit_storage.html `touch-action` [2]
-    * test_initial_computation.html `touch-action` [4]
-    * test_initial_storage.html `touch-action` [4]
-    * test_value_storage.html `touch-action` [14]
 * Properties implemented but not in geckolib:
   * font-feature-settings property servo/servo#15975
     * test_compute_data_with_start_struct.html `font-feature-settings` [2]
@@ -145,11 +138,8 @@ to mochitest command.
     * test_value_storage.html `font-feature-settings` [112]
   * image-orientation property bug 1341758
     * test_value_storage.html `image-orientation` [40]
-  * flexbox / grid position properties servo/servo#15001
-    * ... `justify-` [5]
 * Stylesheet cloning is somehow busted bug 1348481
-  * test_selectors.html `cloned correctly` [157]
-  * ... `matched clone` [204]
+  * test_selectors.html `matched clone` [3]
 * Unsupported prefixed values
   * moz-prefixed gradient functions bug 1337655
     * test_value_storage.html `-moz-linear-gradient` [322]
@@ -274,7 +264,6 @@ to mochitest command.
 
 ## Unknown / Unsure
 
-* test_additional_sheets.html: one sub-test cascade order is wrong [1]
 * test_selectors_on_anonymous_content.html: xbl and :nth-child [1]
 * test_parse_rule.html `rgb(0, 128, 0)`: color properties not getting computed [5]