Merge mozilla-central to mozilla-inbound. a=merge on a CLOSED TREE

accessible/tests/mochitest/events/docload/test_docload_root.html

@@ -73,6 +73,9 @@
             {
               role: ROLE_CHROME_WINDOW
             },
+            {
+              role: ROLE_CHROME_WINDOW
+            },
             {
               role: ROLE_CHROME_WINDOW
             }

accessible/tests/mochitest/events/docload/test_docload_shutdown.html

@@ -89,6 +89,9 @@
             {
               role: ROLE_CHROME_WINDOW
             },
+            {
+              role: ROLE_CHROME_WINDOW
+            },
             {
               role: ROLE_CHROME_WINDOW
             }

browser/app/profile/firefox.js

@@ -1729,7 +1729,7 @@ pref("app.normandy.dev_mode", false);
 pref("app.normandy.enabled", true);
 pref("app.normandy.first_run", true);
 pref("app.normandy.logging.level", 50); // Warn
-pref("app.normandy.run_interval_seconds", 86400); // 24 hours
+pref("app.normandy.run_interval_seconds", 21600); // 6 hours
 pref("app.normandy.shieldLearnMoreUrl", "https://support.mozilla.org/1/firefox/%VERSION%/%OS%/%LOCALE%/shield");
 #ifdef MOZ_DATA_REPORTING
 pref("app.shield.optoutstudies.enabled", true);

browser/base/content/browser-fullZoom.js

@@ -205,10 +205,21 @@ var FullZoom = {
       return;
     }
 
-    // Avoid the cps roundtrip and apply the default/global pref.
     if (aURI.spec == "about:blank") {
-      this._applyPrefToZoom(undefined, browser,
-                            this._notifyOnLocationChange.bind(this, browser));
+      if (!browser.contentPrincipal || browser.contentPrincipal.isNullPrincipal) {
+        // For an about:blank with a null principal, zooming any amount does not
+        // make any sense - so simply do 100%.
+        this._applyPrefToZoom(1, browser,
+                              this._notifyOnLocationChange.bind(this, browser));
+      } else {
+        // If it's not a null principal, there may be content loaded into it,
+        // so use the global pref. This will avoid a cps2 roundtrip if we've
+        // already loaded the global pref once. Really, this should probably
+        // use the contentPrincipal's origin if it's an http(s) principal.
+        // (See bug 1457597)
+        this._applyPrefToZoom(undefined, browser,
+                              this._notifyOnLocationChange.bind(this, browser));
+      }
       return;
     }
 

browser/base/content/browser-menubar.inc

@@ -96,15 +96,13 @@
 #ifdef XP_WIN
                           label="&quitApplicationCmdWin2.label;"
                           accesskey="&quitApplicationCmdWin2.accesskey;"
-#else
-#ifdef XP_MACOSX
+#elifdef XP_MACOSX
                           label="&quitApplicationCmdMac2.label;"
 #else
                           label="&quitApplicationCmd.label;"
                           accesskey="&quitApplicationCmd.accesskey;"
 #endif
                           key="key_quitApplication"
-#endif
                           command="cmd_quitApplication"/>
               </menupopup>
             </menu>

browser/base/content/browser-plugins.js

@@ -96,11 +96,8 @@ var gPluginHandler = {
 
   // Callback for user clicking on the link in a click-to-play plugin
   // (where the plugin has an update)
-  openPluginUpdatePage(pluginTag) {
-    let url = Blocklist.getPluginInfoURL(pluginTag);
-    if (!url) {
-      url = Blocklist.getPluginBlocklistURL(pluginTag);
-    }
+  async openPluginUpdatePage(pluginTag) {
+    let url = await Blocklist.getPluginBlockURL(pluginTag);
     openTrustedLinkIn(url, "tab");
   },
 
@@ -271,20 +268,6 @@ var gPluginHandler = {
       if (pluginData.has(pluginInfo.permissionString)) {
         continue;
       }
-
-      // If a block contains an infoURL, we should always prefer that to the default
-      // URL that we construct in-product, even for other blocklist types.
-      let url = Blocklist.getPluginInfoURL(pluginInfo.pluginTag);
-
-      if (pluginInfo.blocklistState != Ci.nsIBlocklistService.STATE_NOT_BLOCKED) {
-        if (!url) {
-          url = Blocklist.getPluginBlocklistURL(pluginInfo.pluginTag);
-        }
-      } else {
-        url = Services.urlFormatter.formatURLPref("app.support.baseURL") + "clicktoplay";
-      }
-      pluginInfo.detailsLink = url;
-
       pluginData.set(pluginInfo.permissionString, pluginInfo);
     }
 
@@ -306,19 +289,6 @@ var gPluginHandler = {
 
     if (plugins.length == 1) {
       let pluginInfo = plugins[0];
-      // If a block contains an infoURL, we should always prefer that to the default
-      // URL that we construct in-product, even for other blocklist types.
-      let url = Blocklist.getPluginInfoURL(pluginInfo.pluginTag);
-
-      if (pluginInfo.blocklistState != Ci.nsIBlocklistService.STATE_NOT_BLOCKED) {
-        if (!url) {
-          url = Blocklist.getPluginBlocklistURL(pluginInfo.pluginTag);
-        }
-      } else {
-        url = Services.urlFormatter.formatURLPref("app.support.baseURL") + "clicktoplay";
-      }
-      pluginInfo.detailsLink = url;
-
       let chromeWin = window.QueryInterface(Ci.nsIDOMChromeWindow);
       let isWindowPrivate = PrivateBrowsingUtils.isWindowPrivate(chromeWin);
 

browser/base/content/test/performance/browser_startup.js

@@ -74,9 +74,6 @@ const startupPhases = {
       "resource://gre/modules/PlacesUtils.jsm",
       "resource://gre/modules/Promise.jsm", // imported by devtools during _delayedStartup
       "resource://gre/modules/Preferences.jsm",
-      // Bug 1448944 - This should be in a stricter bucket, but we
-      // load it to check content prefs on the initial about:blank
-      "resource://gre/modules/Sqlite.jsm",
     ]),
     services: new Set([
       "@mozilla.org/browser/search-service;1",
@@ -103,6 +100,7 @@ const startupPhases = {
       "resource://gre/modules/FxAccountsStorage.jsm",
       "resource://gre/modules/PlacesBackups.jsm",
       "resource://gre/modules/PlacesSyncUtils.jsm",
+      "resource://gre/modules/Sqlite.jsm",
     ]),
     services: new Set([
       "@mozilla.org/browser/annotation-service;1",

browser/base/content/test/plugins/blocklist_proxy.js

@@ -57,11 +57,8 @@ var BlocklistProxy = {
     return 0; // STATE_NOT_BLOCKED
   },
 
-  getPluginBlocklistURL(aPluginTag) {
-    return "";
-  },
-
-  getPluginInfoURL(aPluginTag) {
+  async getPluginBlockURL(aPluginTag) {
+    await new Promise(r => setTimeout(r, 150));
     return "";
   },
 };

browser/extensions/webcompat/bootstrap.js

@@ -6,7 +6,14 @@ ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 
 const PREF_BRANCH = "extensions.webcompat.";
-const PREF_DEFAULTS = {perform_ua_overrides: true};
+const PREF_DEFAULTS = {
+  perform_injections: true,
+  perform_ua_overrides: true
+};
+
+const INJECTIONS_ENABLE_PREF_NAME = "extensions.webcompat.perform_injections";
+
+const BROWSER_STARTUP_FINISHED_TOPIC = "browser-delayed-startup-finished";
 
 const UA_OVERRIDES_INIT_TOPIC = "useragentoverrides-initialized";
 const UA_ENABLE_PREF_NAME = "extensions.webcompat.perform_ua_overrides";
@@ -15,15 +22,19 @@ ChromeUtils.defineModuleGetter(this, "UAOverrider", "chrome://webcompat/content/
 ChromeUtils.defineModuleGetter(this, "UAOverrides", "chrome://webcompat/content/data/ua_overrides.jsm");
 
 let overrider;
+let webextensionPort;
+
+function InjectionsEnablePrefObserver() {
+  let isEnabled = Services.prefs.getBoolPref(INJECTIONS_ENABLE_PREF_NAME);
+  webextensionPort.postMessage({
+    type: "injection-pref-changed",
+    prefState: isEnabled
+  });
+}
 
 function UAEnablePrefObserver() {
   let isEnabled = Services.prefs.getBoolPref(UA_ENABLE_PREF_NAME);
-  if (isEnabled && !overrider) {
-    overrider = new UAOverrider(UAOverrides);
-    overrider.init();
-  } else if (!isEnabled && overrider) {
-    overrider = null;
-  }
+  overrider.setShouldOverride(isEnabled);
 }
 
 function setDefaultPrefs() {
@@ -57,6 +68,9 @@ this.startup = function({webExtension}) {
   // Intentionally reset the preference on every browser restart to avoid site
   // breakage by accidentally toggled preferences or by leaving it off after
   // debugging a site.
+  Services.prefs.clearUserPref(INJECTIONS_ENABLE_PREF_NAME);
+  Services.prefs.addObserver(INJECTIONS_ENABLE_PREF_NAME, InjectionsEnablePrefObserver);
+
   Services.prefs.clearUserPref(UA_ENABLE_PREF_NAME);
   Services.prefs.addObserver(UA_ENABLE_PREF_NAME, UAEnablePrefObserver);
 
@@ -64,7 +78,7 @@ this.startup = function({webExtension}) {
   // initialize our overrider there. This is done to avoid slowing down the
   // apparent startup proces, since we avoid loading anything before the first
   // window is visible to the user. See bug 1371442 for details.
-  let startupWatcher = {
+  let uaStartupObserver = {
     observe(aSubject, aTopic, aData) {
       if (aTopic !== UA_OVERRIDES_INIT_TOPIC) {
         return;
@@ -75,9 +89,29 @@ this.startup = function({webExtension}) {
       overrider.init();
     }
   };
-  Services.obs.addObserver(startupWatcher, UA_OVERRIDES_INIT_TOPIC);
+  Services.obs.addObserver(uaStartupObserver, UA_OVERRIDES_INIT_TOPIC);
+
+  // Observe browser-delayed-startup-finished and only initialize our embedded
+  // WebExtension after that. Otherwise, we'd try to initialize as soon as the
+  // browser starts up, which adds a heavy startup penalty.
+  let appStartupObserver = {
+    observe(aSubject, aTopic, aData) {
+      webExtension.startup().then((api) => {
+        api.browser.runtime.onConnect.addListener((port) => {
+          webextensionPort = port;
+        });
+
+        return Promise.resolve();
+      }).catch((ex) => {
+        console.error(ex);
+      });
+      Services.obs.removeObserver(this, BROWSER_STARTUP_FINISHED_TOPIC);
+    }
+  };
+  Services.obs.addObserver(appStartupObserver, BROWSER_STARTUP_FINISHED_TOPIC);
 };
 
 this.shutdown = function() {
+  Services.prefs.removeObserver(INJECTIONS_ENABLE_PREF_NAME, InjectionsEnablePrefObserver);
   Services.prefs.removeObserver(UA_ENABLE_PREF_NAME, UAEnablePrefObserver);
 };

browser/extensions/webcompat/content/data/ua_overrides.jsm

@@ -3,41 +3,11 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 /**
- * This is an array of objects that specify user agent overrides. Each object
- * can have three attributes:
+ * For detailed information on our policies, and a documention on this format
+ * and its possibilites, please check the Mozilla-Wiki at
  *
- *   * `baseDomain`, required: The base domain that further checks and user
- *     agents override are applied to. This does not include subdomains.
- *   * `uriMatcher`: Function that gets the requested URI passed in the first
- *     argument and needs to return boolean whether or not the override should
- *     be applied. If not provided, the user agent override will be applied
- *     every time.
- *   * `uaTransformer`, required: Function that gets the original Firefox user
- *     agent passed as its first argument and needs to return a string that
- *     will be used as the the user agent for this URI.
- *
- * Examples:
- *
- * Gets applied for all requests to mozilla.org and subdomains:
- *
- * ```
- *   {
- *     baseDomain: "mozilla.org",
- *     uaTransformer: (originalUA) => `Ohai Mozilla, it's me, ${originalUA}`
- *   }
- * ```
- *
- * Applies to *.example.com/app/*:
- *
- * ```
- *   {
- *     baseDomain: "example.com",
- *     uriMatcher: (uri) => uri.includes("/app/"),
- *     uaTransformer: (originalUA) => originalUA.replace("Firefox", "Otherfox")
- *   }
- * ```
+ * https://wiki.mozilla.org/Compatibility/Go_Faster_Addon/Override_Policies_and_Workflows#User_Agent_overrides
  */
-
 const UAOverrides = [
 
   /*
@@ -49,7 +19,8 @@ const UAOverrides = [
    */
   {
     baseDomain: "schub.io",
-    uriMatcher: (uri) => uri.includes("webcompat-ua-dummy.schub.io"),
+    applications: ["firefox", "fennec"],
+    uriMatcher: (uri) => uri.includes("webcompat-addon-testcases.schub.io"),
     uaTransformer: (originalUA) => {
       let prefix = originalUA.substr(0, originalUA.indexOf(")") + 1);
       return `${prefix} AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36`;

browser/extensions/webcompat/content/lib/ua_overrider.jsm

@@ -4,18 +4,39 @@
 
 ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 
+ChromeUtils.defineModuleGetter(this, "Services", "resource://gre/modules/Services.jsm");
 ChromeUtils.defineModuleGetter(this, "UserAgentOverrides", "resource://gre/modules/UserAgentOverrides.jsm");
 XPCOMUtils.defineLazyServiceGetter(this, "eTLDService", "@mozilla.org/network/effective-tld-service;1", "nsIEffectiveTLDService");
 
 class UAOverrider {
   constructor(overrides) {
     this._overrides = {};
+    this._shouldOverride = true;
 
     this.initOverrides(overrides);
   }
 
   initOverrides(overrides) {
+    // on xpcshell tests, there is no impleentation for nsIXULAppInfo, so this
+    // might fail there. To have all of our test cases running at all times,
+    // assume they are on Desktop for now.
+    let currentApplication = "firefox";
+    try {
+      currentApplication = Services.appinfo.name.toLowerCase();
+    } catch (_) {}
+
     for (let override of overrides) {
+      // Firefox for Desktop is the default application for all overrides.
+      if (!override.applications) {
+        override.applications = ["firefox"];
+      }
+
+      // If the current application is not targeted by the override in question,
+      // we can skip adding the override to our checks entirely.
+      if (!override.applications.includes(currentApplication)) {
+        continue;
+      }
+
       if (!this._overrides[override.baseDomain]) {
         this._overrides[override.baseDomain] = [];
       }
@@ -28,11 +49,26 @@ class UAOverrider {
     }
   }
 
+  /**
+   * Used for disabling overrides when the pref has been flipped to false.
+   *
+   * Since we no longer use our own event handlers, we check this bool in our
+   * override callback and simply return early if we are not supposed to do
+   * anything.
+   */
+  setShouldOverride(newState) {
+    this._shouldOverride = newState;
+  }
+
   init() {
     UserAgentOverrides.addComplexOverride(this.overrideCallback.bind(this));
   }
 
   overrideCallback(channel, defaultUA) {
+    if (!this._shouldOverride) {
+      return false;
+    }
+
     let uaOverride = this.lookupUAOverride(channel.URI, defaultUA);
     if (uaOverride) {
       console.log("The user agent has been overridden for compatibility reasons.");

browser/extensions/webcompat/install.rdf.in

@@ -10,13 +10,13 @@
 
   <Description about="urn:mozilla:install-manifest">
     <em:id>webcompat@mozilla.org</em:id>
-    <em:version>1.1</em:version>
+    <em:version>2.0</em:version>
     <em:type>2</em:type>
     <em:bootstrap>true</em:bootstrap>
     <em:multiprocessCompatible>true</em:multiprocessCompatible>
+    <em:hasEmbeddedWebExtension>true</em:hasEmbeddedWebExtension>
 
-    <!-- Target Application this extension can install into,
-        with minimum and maximum supported versions. -->
+    <!-- Firefox Desktop -->
     <em:targetApplication>
       <Description>
         <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
@@ -25,6 +25,15 @@
       </Description>
     </em:targetApplication>
 
+    <!-- Firefox for Android -->
+    <em:targetApplication>
+      <Description>
+        <em:id>{aa3c5121-dab2-40e2-81ca-7ea25febc110}</em:id>
+        <em:minVersion>@MOZ_APP_VERSION@</em:minVersion>
+        <em:maxVersion>@MOZ_APP_MAXVERSION@</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+
     <!-- Front End MetaData -->
     <em:name>Web Compat</em:name>
     <em:description>Urgent post-release fixes for web compatibility.</em:description>

browser/extensions/webcompat/moz.build

@@ -11,6 +11,19 @@ FINAL_TARGET_FILES.features['webcompat@mozilla.org'] += [
   'bootstrap.js'
 ]
 
+FINAL_TARGET_FILES.features['webcompat@mozilla.org']['webextension'] += [
+  'webextension/background.js',
+  'webextension/manifest.json'
+]
+
+FINAL_TARGET_FILES.features['webcompat@mozilla.org']['webextension']['injections']['css'] += [
+  'webextension/injections/css/bug0000000-dummy-css-injection.css'
+]
+
+FINAL_TARGET_FILES.features['webcompat@mozilla.org']['webextension']['injections']['js'] += [
+  'webextension/injections/js/bug0000000-dummy-js-injection.js'
+]
+
 FINAL_TARGET_PP_FILES.features['webcompat@mozilla.org'] += [
   'install.rdf.in'
 ]

browser/extensions/webcompat/webextension/background.js

@@ -0,0 +1,55 @@
+/**
+ * For detailed information on our policies, and a documention on this format
+ * and its possibilites, please check the Mozilla-Wiki at
+ *
+ * https://wiki.mozilla.org/Compatibility/Go_Faster_Addon/Override_Policies_and_Workflows#User_Agent_overrides
+ */
+const contentScripts = [
+  {
+    matches: ["*://webcompat-addon-testcases.schub.io/*"],
+    css: [{file: "injections/css/bug0000000-dummy-css-injection.css"}],
+    js: [{file: "injections/js/bug0000000-dummy-js-injection.js"}],
+    runAt: "document_start"
+  }
+];
+
+/* globals browser */
+
+let port = browser.runtime.connect();
+let registeredContentScripts = [];
+
+function registerContentScripts() {
+  contentScripts.forEach(async (contentScript) => {
+    try {
+      let handle = await browser.contentScripts.register(contentScript);
+      registeredContentScripts.push(handle);
+    } catch (ex) {
+      console.error("Registering WebCompat GoFaster content scripts failed: ", ex);
+    }
+  });
+}
+
+function unregisterContentScripts() {
+  registeredContentScripts.forEach((contentScript) => {
+    contentScript.unregister();
+  });
+}
+
+port.onMessage.addListener((message) => {
+  switch (message.type) {
+    case "injection-pref-changed":
+      if (message.prefState) {
+        registerContentScripts();
+      } else {
+        unregisterContentScripts();
+      }
+      break;
+  }
+});
+
+/**
+ * Note that we reset all preferences on extension startup, so the injections will
+ * never be disabled when this loads up. Because of that, we can simply register
+ * right away.
+ */
+registerContentScripts();

browser/extensions/webcompat/webextension/injections/css/bug0000000-dummy-css-injection.css

@@ -0,0 +1,3 @@
+#css-injection.red {
+  background-color: #0f0;
+}

browser/extensions/webcompat/webextension/injections/js/bug0000000-dummy-js-injection.js

@@ -0,0 +1,11 @@
+"use strict";
+
+/* globals exportFunction */
+
+Object.defineProperty(window.wrappedJSObject, "isTestFeatureSupported", {
+  get: exportFunction(function() {
+    return true;
+  }, window),
+
+  set: exportFunction(function() {}, window)
+});

browser/extensions/webcompat/webextension/manifest.json

@@ -0,0 +1,23 @@
+{
+  "manifest_version": 2,
+  "name": "Web Compat",
+  "description": "Urgent post-release fixes for web compatibility.",
+  "version": "2.0",
+
+  "applications": {
+    "gecko": {
+      "id": "webcompat@mozilla.org",
+      "strict_min_version": "59.0b5"
+    }
+  },
+
+  "permissions": [
+    "<all_urls>"
+  ],
+
+  "background": {
+    "scripts": [
+      "background.js"
+    ]
+  }
+}

build/mach_bootstrap.py

@@ -54,6 +54,7 @@ MACH_MODULES = [
     'testing/marionette/mach_commands.py',
     'testing/mochitest/mach_commands.py',
     'testing/mozharness/mach_commands.py',
+    'testing/raptor/mach_commands.py',
     'testing/talos/mach_commands.py',
     'testing/web-platform/mach_commands.py',
     'testing/xpcshell/mach_commands.py',

devtools/client/definitions.js

@@ -101,7 +101,14 @@ Tools.webConsole = {
   accesskey: l10n("webConsoleCmd.accesskey"),
   ordinal: 2,
   url: "chrome://devtools/content/webconsole/webconsole.html",
-  browserConsoleURL: "chrome://devtools/content/webconsole/browserconsole.xul",
+  get browserConsoleUsesHTML() {
+    return Services.prefs.getBoolPref("devtools.browserconsole.html");
+  },
+  get browserConsoleURL() {
+    return this.browserConsoleUsesHTML ?
+      "chrome://devtools/content/webconsole/webconsole.html" :
+      "chrome://devtools/content/webconsole/browserconsole.xul";
+  },
   icon: "chrome://devtools/skin/images/tool-webconsole.svg",
   label: l10n("ToolboxTabWebconsole.label"),
   menuLabel: l10n("MenuWebconsole.label"),

devtools/client/preferences/devtools-client.js

@@ -242,6 +242,9 @@ pref("devtools.accessibility.enabled", false);
 // Web Audio Editor Inspector Width should be a preference
 pref("devtools.webaudioeditor.inspectorWidth", 300);
 
+// Experimental UI for the browser console that doesn't use a XUL wrapper doc
+pref("devtools.browserconsole.html", false);
+
 // Web console filters
 pref("devtools.webconsole.filter.error", true);
 pref("devtools.webconsole.filter.warn", true);

devtools/client/webconsole/hudservice.js

@@ -183,21 +183,27 @@ HUD_SERVICE.prototype =
     }
 
     async function openWindow(t) {
-      let browserConsoleURL = Tools.webConsole.browserConsoleURL;
-      let win = Services.ww.openWindow(null, browserConsoleURL, "_blank",
-                                       BC_WINDOW_FEATURES, null);
+      let win = Services.ww.openWindow(null, Tools.webConsole.browserConsoleURL,
+                                       "_blank", BC_WINDOW_FEATURES, null);
+      let iframeWindow = win;
+
       await new Promise(resolve => {
         win.addEventListener("DOMContentLoaded", resolve, {once: true});
       });
 
       win.document.title = l10n.getStr("browserConsole.title");
 
-      let iframe = win.document.querySelector("iframe");
-      await new Promise(resolve => {
-        iframe.addEventListener("DOMContentLoaded", resolve, {once: true});
-      });
+      // With a XUL wrapper doc, we host webconsole.html in an iframe.
+      // Wait for that to be ready before resolving:
+      if (!Tools.webConsole.browserConsoleUsesHTML) {
+        let iframe = win.document.querySelector("iframe");
+        await new Promise(resolve => {
+          iframe.addEventListener("DOMContentLoaded", resolve, {once: true});
+        });
+        iframeWindow = iframe.contentWindow;
+      }
 
-      return {iframeWindow: iframe.contentWindow, chromeWindow: win};
+      return {iframeWindow, chromeWindow: win};
     }
 
     // Temporarily cache the async startup sequence so that if toggleBrowserConsole

devtools/client/webconsole/test/mochitest/browser.ini

@@ -332,6 +332,7 @@ subsuite = clipboard
 [browser_webconsole_show_subresource_security_errors.js]
 [browser_webconsole_shows_reqs_from_netmonitor.js]
 [browser_webconsole_shows_reqs_in_netmonitor.js]
+[browser_webconsole_sidebar_object_expand_when_message_pruned.js]
 [browser_webconsole_sourcemap_css.js]
 [browser_webconsole_sourcemap_error.js]
 [browser_webconsole_sourcemap_invalid.js]

devtools/client/webconsole/test/mochitest/browser_webconsole_sidebar_object_expand_when_message_pruned.js

@@ -0,0 +1,80 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Test that an object in the sidebar can still be expanded after the message where it was
+// logged is pruned.
+
+"use strict";
+
+const TEST_URI = "data:text/html;charset=utf8," +
+  "<script>console.log({a:1,b:2,c:[3,4,5]});</script>";
+
+add_task(async function() {
+  // Should be removed when sidebar work is complete (Bug 1447235)
+  await pushPref("devtools.webconsole.sidebarToggle", true);
+  // Set the loglimit to 1 so message gets pruned as soon as another message is displayed.
+  await pushPref("devtools.hud.loglimit", 1);
+
+  let hud = await openNewTabAndConsole(TEST_URI);
+
+  let message = await waitFor(() => findMessage(hud, "Object"));
+  let object = message.querySelector(".object-inspector .objectBox-object");
+
+  const sidebar = await showSidebarWithContextMenu(hud, object, true);
+
+  let oi = sidebar.querySelector(".object-inspector");
+  let oiNodes = oi.querySelectorAll(".node");
+  if (oiNodes.length === 1) {
+    // If this is the case, we wait for the properties to be fetched and displayed.
+    await waitFor(() => oi.querySelectorAll(".node").length > 1);
+    oiNodes = oi.querySelectorAll(".node");
+  }
+
+  info("Log a message so the original one gets pruned");
+  const messageText = "hello world";
+  const onMessage = waitForMessage(hud, messageText);
+  ContentTask.spawn(gBrowser.selectedBrowser, messageText, async function(str) {
+    content.console.log(str);
+  });
+  await onMessage;
+
+  ok(!findMessage(hud, "Object"), "Message with object was pruned");
+
+  info("Expand the 'c' node in the sidebar");
+  // Here's what the object in the sidebar looks like:
+  // ▼ {…}
+  //     a: 1
+  //     b: 2
+  //   ▶︎ c: (3) […]
+  //   ▶︎ <prototype>: {…}
+  const cNode = oiNodes[3];
+  const onNodeExpanded = waitFor(() => oi.querySelectorAll(".node").length > 5);
+  cNode.click();
+  await onNodeExpanded;
+
+  // Here's what the object in the sidebar should look like:
+  // ▼ {…}
+  //     a: 1
+  //     b: 2
+  //   ▼ c: (3) […]
+  //       0: 3
+  //       1: 4
+  //       2: 5
+  //       length: 3
+  //     ▶︎ <prototype>: []
+  //   ▶︎ <prototype>: {…}
+  is(oi.querySelectorAll(".node").length, 10, "The 'c' property was expanded");
+});
+
+async function showSidebarWithContextMenu(hud, node) {
+  let wrapper = hud.ui.document.querySelector(".webconsole-output-wrapper");
+  let onSidebarShown = waitFor(() => wrapper.querySelector(".sidebar"));
+
+  let contextMenu = await openContextMenu(hud, node);
+  let openInSidebar = contextMenu.querySelector("#console-menu-open-sidebar");
+  openInSidebar.click();
+  await onSidebarShown;
+  await hideContextMenu(hud);
+  return onSidebarShown;
+}

dom/base/DirectionalityUtils.cpp

@@ -241,7 +241,7 @@ DoesNotParticipateInAutoDirection(const Element* aElement)
           nodeInfo->Equals(nsGkAtoms::script) ||
           nodeInfo->Equals(nsGkAtoms::style) ||
           nodeInfo->Equals(nsGkAtoms::textarea) ||
-          aElement->IsInAnonymousSubtree());
+          (aElement->IsInAnonymousSubtree() && !aElement->HasDirAuto()));
 }
 
 /**
@@ -254,7 +254,8 @@ DoesNotAffectDirectionOfAncestors(const Element* aElement)
 {
   return (DoesNotParticipateInAutoDirection(aElement) ||
           aElement->IsHTMLElement(nsGkAtoms::bdi) ||
-          aElement->HasFixedDir());
+          aElement->HasFixedDir() ||
+          aElement->IsInAnonymousSubtree());
 }
 
 /**
@@ -280,10 +281,14 @@ inline static bool
 NodeAffectsDirAutoAncestor(nsINode* aTextNode)
 {
   Element* parent = aTextNode->GetParentElement();
+  // In the anonymous content, we limit our implementation to only
+  // allow the children text node of the direct dir=auto parent in
+  // the same anonymous subtree to affact the direction.
   return (parent &&
           !DoesNotParticipateInAutoDirection(parent) &&
           parent->NodeOrAncestorHasDirAuto() &&
-          !aTextNode->IsInAnonymousSubtree());
+          (!aTextNode->IsInAnonymousSubtree() ||
+            parent->HasDirAuto()));
 }
 
 Directionality
@@ -920,14 +925,19 @@ SetDirectionFromNewTextNode(nsTextNode* aTextNode)
 void
 ResetDirectionSetByTextNode(nsTextNode* aTextNode)
 {
-  if (!NodeAffectsDirAutoAncestor(aTextNode)) {
-    nsTextNodeDirectionalityMap::EnsureMapIsClearFor(aTextNode);
+  // We used to check NodeAffectsDirAutoAncestor() in this function, but
+  // stopped doing that since calling IsInAnonymousSubtree()
+  // too late (during nsTextNode::UnbindFromTree) is impossible and this
+  // function was no-op when there's no directionality map.
+  if (!aTextNode->HasTextNodeDirectionalityMap()) {
     return;
   }
 
   Directionality dir = GetDirectionFromText(aTextNode->GetText());
-  if (dir != eDir_NotSet && aTextNode->HasTextNodeDirectionalityMap()) {
+  if (dir != eDir_NotSet) {
     nsTextNodeDirectionalityMap::ResetTextNodeDirection(aTextNode, aTextNode);
+  } else {
+    nsTextNodeDirectionalityMap::EnsureMapIsClearFor(aTextNode);
   }
 }
 

gfx/layers/AnimationHelper.cpp

@@ -156,6 +156,14 @@ AnimationHelper::SampleAnimationForEachNode(
   MOZ_ASSERT(!aAnimations.IsEmpty(), "Should be called with animations");
 
   bool hasInEffectAnimations = false;
+#ifdef DEBUG
+  // In cases where this function returns a SampleResult::Skipped, we actually
+  // do populate aAnimationValue in debug mode, so that we can MOZ_ASSERT at the
+  // call site that the value that would have been computed matches the stored
+  // value that we end up using. This flag is used to ensure we populate
+  // aAnimationValue in this scenario.
+  bool shouldBeSkipped = false;
+#endif
   // Process in order, since later aAnimations override earlier ones.
   for (size_t i = 0, iEnd = aAnimations.Length(); i < iEnd; ++i) {
     Animation& animation = aAnimations[i];
@@ -204,7 +212,11 @@ AnimationHelper::SampleAnimationForEachNode(
           iterCompositeOperation,
           animData.mProgressOnLastCompose,
           animData.mCurrentIterationOnLastCompose)) {
+#ifdef DEBUG
+      shouldBeSkipped = true;
+#else
       return SampleResult::Skipped;
+#endif
     }
 
     uint32_t segmentIndex = 0;
@@ -236,7 +248,11 @@ AnimationHelper::SampleAnimationForEachNode(
         animData.mSegmentIndexOnLastCompose == segmentIndex &&
         !animData.mPortionInSegmentOnLastCompose.IsNull() &&
         animData.mPortionInSegmentOnLastCompose.Value() == portion) {
+#ifdef DEBUG
+      shouldBeSkipped = true;
+#else
       return SampleResult::Skipped;
+#endif
     }
 
     AnimationPropertySegment animSegment;
@@ -260,6 +276,13 @@ AnimationHelper::SampleAnimationForEachNode(
         iterCompositeOperation,
         portion,
         computedTiming.mCurrentIteration).Consume();
+
+#ifdef DEBUG
+    if (shouldBeSkipped) {
+      return SampleResult::Skipped;
+    }
+#endif
+
     hasInEffectAnimations = true;
     animData.mProgressOnLastCompose = computedTiming.mProgress;
     animData.mCurrentIterationOnLastCompose = computedTiming.mCurrentIteration;

gfx/layers/composite/AsyncCompositionManager.cpp

@@ -575,6 +575,51 @@ AsyncCompositionManager::AlignFixedAndStickyLayers(Layer* aTransformedSubtreeRoo
   }
 }
 
+static Matrix4x4
+ServoAnimationValueToMatrix4x4(const RefPtr<RawServoAnimationValue>& aValue,
+                               const TransformData& aTransformData)
+{
+  // FIXME: Bug 1457033: We should convert servo's animation value to matrix
+  // directly without nsCSSValueSharedList.
+  RefPtr<nsCSSValueSharedList> list;
+  Servo_AnimationValue_GetTransform(aValue, &list);
+  // we expect all our transform data to arrive in device pixels
+  Point3D transformOrigin = aTransformData.transformOrigin();
+  nsDisplayTransform::FrameTransformProperties props(Move(list),
+                                                     transformOrigin);
+
+  return nsDisplayTransform::GetResultingTransformMatrix(
+    props, aTransformData.origin(),
+    aTransformData.appUnitsPerDevPixel(),
+    0, &aTransformData.bounds());
+}
+
+
+static Matrix4x4
+FrameTransformToTransformInDevice(const Matrix4x4& aFrameTransform,
+                                  Layer* aLayer,
+                                  const TransformData& aTransformData)
+{
+  Matrix4x4 transformInDevice = aFrameTransform;
+  // If our parent layer is a perspective layer, then the offset into reference
+  // frame coordinates is already on that layer. If not, then we need to ask
+  // for it to be added here.
+  if (!aLayer->GetParent() ||
+      !aLayer->GetParent()->GetTransformIsPerspective()) {
+    nsLayoutUtils::PostTranslate(transformInDevice, aTransformData.origin(),
+      aTransformData.appUnitsPerDevPixel(),
+      true);
+  }
+
+  if (ContainerLayer* c = aLayer->AsContainerLayer()) {
+    transformInDevice.PostScale(c->GetInheritedXScale(),
+                                c->GetInheritedYScale(),
+                                1);
+  }
+
+  return transformInDevice;
+}
+
 static void
 ApplyAnimatedValue(Layer* aLayer,
                    CompositorAnimationStorage* aStorage,
@@ -600,34 +645,15 @@ ApplyAnimatedValue(Layer* aLayer,
       break;
     }
     case eCSSProperty_transform: {
-      RefPtr<nsCSSValueSharedList> list;
-      Servo_AnimationValue_GetTransform(aValue, &list);
       const TransformData& transformData = aAnimationData.get_TransformData();
-      nsPoint origin = transformData.origin();
-      // we expect all our transform data to arrive in device pixels
-      Point3D transformOrigin = transformData.transformOrigin();
-      nsDisplayTransform::FrameTransformProperties props(Move(list),
-                                                         transformOrigin);
+
+      Matrix4x4 frameTransform =
+        ServoAnimationValueToMatrix4x4(aValue, transformData);
 
       Matrix4x4 transform =
-        nsDisplayTransform::GetResultingTransformMatrix(props, origin,
-                                                        transformData.appUnitsPerDevPixel(),
-                                                        0, &transformData.bounds());
-      Matrix4x4 frameTransform = transform;
-
-      // If our parent layer is a perspective layer, then the offset into reference
-      // frame coordinates is already on that layer. If not, then we need to ask
-      // for it to be added here.
-      if (!aLayer->GetParent() ||
-          !aLayer->GetParent()->GetTransformIsPerspective()) {
-        nsLayoutUtils::PostTranslate(transform, origin,
-                                     transformData.appUnitsPerDevPixel(),
-                                     true);
-      }
-
-      if (ContainerLayer* c = aLayer->AsContainerLayer()) {
-        transform.PostScale(c->GetInheritedXScale(), c->GetInheritedYScale(), 1);
-      }
+        FrameTransformToTransformInDevice(frameTransform,
+                                          aLayer,
+                                          transformData);
 
       layerCompositor->SetShadowBaseTransform(transform);
       layerCompositor->SetShadowTransformSetByAnimation(true);
@@ -684,16 +710,30 @@ SampleAnimations(Layer* aLayer,
             // Sanity check that the animation value is surely unchanged.
             switch (animations[0].property()) {
               case eCSSProperty_opacity:
+                MOZ_ASSERT(
+                  layer->AsHostLayer()->GetShadowOpacitySetByAnimation());
                 MOZ_ASSERT(FuzzyEqualsMultiplicative(
-                  layer->AsHostLayer()->GetShadowOpacity(),
+                  Servo_AnimationValue_GetOpacity(animationValue),
                   *(aStorage->GetAnimationOpacity(layer->GetCompositorAnimationsId()))));
                 break;
               case eCSSProperty_transform: {
+                MOZ_ASSERT(
+                  layer->AsHostLayer()->GetShadowTransformSetByAnimation());
+
                 AnimatedValue* transform =
                   aStorage->GetAnimatedValue(layer->GetCompositorAnimationsId());
+
+                const TransformData& transformData =
+                  animations[0].data().get_TransformData();
+                Matrix4x4 frameTransform =
+                  ServoAnimationValueToMatrix4x4(animationValue, transformData);
+                Matrix4x4 transformInDevice =
+                  FrameTransformToTransformInDevice(frameTransform,
+                                                    layer,
+                                                    transformData);
                 MOZ_ASSERT(
                   transform->mTransform.mTransformInDevSpace.FuzzyEqualsMultiplicative(
-                    (layer->AsHostLayer()->GetShadowBaseTransform())));
+                  transformInDevice));
                 break;
               }
               default:

gfx/webrender_bindings/RenderThread.cpp

@@ -192,7 +192,7 @@ RenderThread::NewFrameReady(wr::WindowId aWindowId)
   }
 
   UpdateAndRender(aWindowId);
-  DecPendingFrameCount(aWindowId);
+  FrameRenderingComplete(aWindowId);
 }
 
 void
@@ -383,6 +383,25 @@ RenderThread::IncPendingFrameCount(wr::WindowId aWindowId)
   mWindowInfos.Put(AsUint64(aWindowId), info);
 }
 
+void
+RenderThread::DecPendingFrameCount(wr::WindowId aWindowId)
+{
+  MutexAutoLock lock(mFrameCountMapLock);
+  // Get the old count.
+  WindowInfo info;
+  if (!mWindowInfos.Get(AsUint64(aWindowId), &info)) {
+    MOZ_ASSERT(false);
+    return;
+  }
+  MOZ_ASSERT(info.mPendingCount > 0);
+  if (info.mPendingCount <= 0) {
+    return;
+  }
+  // Update pending frame count.
+  info.mPendingCount = info.mPendingCount - 1;
+  mWindowInfos.Put(AsUint64(aWindowId), info);
+}
+
 void
 RenderThread::IncRenderingFrameCount(wr::WindowId aWindowId)
 {
@@ -399,7 +418,7 @@ RenderThread::IncRenderingFrameCount(wr::WindowId aWindowId)
 }
 
 void
-RenderThread::DecPendingFrameCount(wr::WindowId aWindowId)
+RenderThread::FrameRenderingComplete(wr::WindowId aWindowId)
 {
   MutexAutoLock lock(mFrameCountMapLock);
   // Get the old count.
@@ -523,12 +542,12 @@ extern "C" {
 static void NewFrameReady(mozilla::wr::WrWindowId aWindowId)
 {
   mozilla::wr::RenderThread::Get()->IncRenderingFrameCount(aWindowId);
-  mozilla::wr::RenderThread::Get()->NewFrameReady(mozilla::wr::WindowId(aWindowId));
+  mozilla::wr::RenderThread::Get()->NewFrameReady(aWindowId);
 }
 
 void wr_notifier_wake_up(mozilla::wr::WrWindowId aWindowId)
 {
-  mozilla::wr::RenderThread::Get()->WakeUp(mozilla::wr::WindowId(aWindowId));
+  mozilla::wr::RenderThread::Get()->WakeUp(aWindowId);
 }
 
 void wr_notifier_new_frame_ready(mozilla::wr::WrWindowId aWindowId)
@@ -536,15 +555,9 @@ void wr_notifier_new_frame_ready(mozilla::wr::WrWindowId aWindowId)
   NewFrameReady(aWindowId);
 }
 
-void wr_notifier_new_scroll_frame_ready(mozilla::wr::WrWindowId aWindowId, bool aCompositeNeeded)
+void wr_notifier_nop_frame_done(mozilla::wr::WrWindowId aWindowId)
 {
-  // If we sent a transaction that contained both scrolling updates and a
-  // GenerateFrame, we can get this function called with aCompositeNeeded=true
-  // instead of wr_notifier_new_frame_ready. In that case we want to update the
-  // rendering.
-  if (aCompositeNeeded) {
-    NewFrameReady(aWindowId);
-  }
+  mozilla::wr::RenderThread::Get()->DecPendingFrameCount(aWindowId);
 }
 
 void wr_notifier_external_event(mozilla::wr::WrWindowId aWindowId, size_t aRawEvent)

gfx/webrender_bindings/RenderThread.h

@@ -153,9 +153,11 @@ public:
   /// Can be called from any thread.
   void IncPendingFrameCount(wr::WindowId aWindowId);
   /// Can be called from any thread.
+  void DecPendingFrameCount(wr::WindowId aWindowId);
+  /// Can be called from any thread.
   void IncRenderingFrameCount(wr::WindowId aWindowId);
   /// Can be called from any thread.
-  void DecPendingFrameCount(wr::WindowId aWindowId);
+  void FrameRenderingComplete(wr::WindowId aWindowId);
 
   /// Can be called from any thread.
   WebRenderThreadPool& ThreadPool() { return mThreadPool; }

gfx/webrender_bindings/src/bindings.rs

@@ -495,8 +495,7 @@ unsafe impl Send for CppNotifier {}
 extern "C" {
     fn wr_notifier_wake_up(window_id: WrWindowId);
     fn wr_notifier_new_frame_ready(window_id: WrWindowId);
-    fn wr_notifier_new_scroll_frame_ready(window_id: WrWindowId,
-                                          composite_needed: bool);
+    fn wr_notifier_nop_frame_done(window_id: WrWindowId);
     fn wr_notifier_external_event(window_id: WrWindowId,
                                   raw_event: usize);
 }
@@ -516,13 +515,13 @@ impl RenderNotifier for CppNotifier {
 
     fn new_frame_ready(&self,
                        _: DocumentId,
-                       scrolled: bool,
+                       _scrolled: bool,
                        composite_needed: bool) {
         unsafe {
-            if scrolled {
-                wr_notifier_new_scroll_frame_ready(self.window_id, composite_needed);
-            } else if composite_needed {
+            if composite_needed {
                 wr_notifier_new_frame_ready(self.window_id);
+            } else {
+                wr_notifier_nop_frame_done(self.window_id);
             }
         }
     }

gfx/webrender_bindings/webrender_ffi_generated.h

@@ -1368,8 +1368,7 @@ extern void wr_notifier_external_event(WrWindowId aWindowId,
 
 extern void wr_notifier_new_frame_ready(WrWindowId aWindowId);
 
-extern void wr_notifier_new_scroll_frame_ready(WrWindowId aWindowId,
-                                               bool aCompositeNeeded);
+extern void wr_notifier_nop_frame_done(WrWindowId aWindowId);
 
 extern void wr_notifier_wake_up(WrWindowId aWindowId);
 

layout/forms/nsFileControlFrame.cpp

@@ -87,6 +87,8 @@ MakeAnonButton(nsIDocument* aDoc, const char* labelKey,
   button->SetIsNativeAnonymousRoot();
   button->SetAttr(kNameSpaceID_None, nsGkAtoms::type,
                   NS_LITERAL_STRING("button"), false);
+  button->SetAttr(kNameSpaceID_None, nsGkAtoms::dir,
+                  NS_LITERAL_STRING("auto"), false);
 
   // Set the file picking button text depending on the current locale.
   nsAutoString buttonTxt;

layout/reftests/forms/input/file/rtl-ref.xul

@@ -10,7 +10,7 @@
     </html:style>
     <html:div dir='rtl'>
       <html:div class='file' dir='rtl'>
-        <html:button>Browse&#8230;</html:button><label value="No file selected."/>
+        <html:button dir='ltr'>Browse&#8230;</html:button><label value="No file selected."/>
       </html:div>
     </html:div>
   </vbox>

layout/reftests/xul/reftest.list

@@ -87,7 +87,7 @@ test-pref(svg.context-properties.content.enabled,true) == treetwisty-svg-context
 != resizer-bottomend-rtl.xul blank-window.xul
 # fuzzy for comparing SVG image flipped by CSS with a flipped SVG image.
 # See bug 1450017 comment 79.
-fuzzy(42,98) == resizer-bottomend-rtl.xul resizer-bottomend-flipped.xul
+fuzzy(43,98) == resizer-bottomend-rtl.xul resizer-bottomend-flipped.xul
 
 != resizer-bottomstart.xul blank-window.xul
 == resizer-bottomstart.xul resizer-bottomleft.xul
@@ -96,4 +96,4 @@ fuzzy(42,98) == resizer-bottomend-rtl.xul resizer-bottomend-flipped.xul
 != resizer-bottomstart-rtl.xul blank-window.xul
 # fuzzy for comparing SVG image flipped by CSS to a flipped SVG image.
 # See bug 1450017 comment 79.
-fuzzy(42,98) == resizer-bottomstart-rtl.xul resizer-bottomend.xul
+fuzzy(43,98) == resizer-bottomstart-rtl.xul resizer-bottomend.xul

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/BaseSessionTest.kt

@@ -100,4 +100,16 @@ open class BaseSessionTest(noErrorCollector: Boolean = false) {
 
     fun GeckoSession.synthesizeTap(x: Int, y: Int) =
             sessionRule.synthesizeTap(this, x, y)
+
+    fun GeckoSession.evaluateJS(js: String) =
+            sessionRule.evaluateJS(this, js)
+
+    infix fun Any?.dot(prop: Any): Any? =
+            if (prop is Int) this.asJSList<Any>()[prop] else this.asJSMap<Any>()[prop]
+
+    @Suppress("UNCHECKED_CAST")
+    fun <T> Any?.asJSMap(): Map<String, T> = this as Map<String, T>
+
+    @Suppress("UNCHECKED_CAST")
+    fun <T> Any?.asJSList(): List<T> = this as List<T>
 }

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/GeckoSessionTestRuleTest.kt

@@ -8,9 +8,11 @@ import org.mozilla.geckoview.GeckoSession
 import org.mozilla.geckoview.GeckoSessionSettings
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.AssertCalled
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.ClosedSessionAtStart
+import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.NullDelegate
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.Setting
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.TimeoutException
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.TimeoutMillis
+import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.WithDevToolsAPI
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.WithDisplay
 import org.mozilla.geckoview.test.util.Callbacks
 
@@ -76,6 +78,38 @@ class GeckoSessionTestRuleTest : BaseSessionTest(noErrorCollector = true) {
         }
     }
 
+    @NullDelegate.List(NullDelegate(GeckoSession.ContentDelegate::class),
+                       NullDelegate(Callbacks.NavigationDelegate::class))
+    @NullDelegate(Callbacks.ScrollDelegate::class)
+    @Test fun nullDelegate() {
+        assertThat("Content delegate should be null",
+                   sessionRule.session.contentDelegate, nullValue())
+        assertThat("Navigation delegate should be null",
+                   sessionRule.session.navigationDelegate, nullValue())
+        assertThat("Scroll delegate should be null",
+                   sessionRule.session.scrollDelegate, nullValue())
+
+        assertThat("Progress delegate should not be null",
+                   sessionRule.session.progressDelegate, notNullValue())
+    }
+
+    @NullDelegate(GeckoSession.ProgressDelegate::class)
+    @ClosedSessionAtStart
+    @Test fun nullDelegate_closed() {
+        assertThat("Progress delegate should be null",
+                   sessionRule.session.progressDelegate, nullValue())
+    }
+
+    @Test(expected = AssertionError::class)
+    @NullDelegate(GeckoSession.ProgressDelegate::class)
+    @ClosedSessionAtStart
+    fun nullDelegate_requireProgressOnOpen() {
+        assertThat("Progress delegate should be null",
+                   sessionRule.session.progressDelegate, nullValue())
+
+        sessionRule.session.open()
+    }
+
     @Test fun waitForPageStop() {
         sessionRule.session.loadTestPath(HELLO_HTML_PATH)
         sessionRule.waitForPageStop()
@@ -114,6 +148,15 @@ class GeckoSessionTestRuleTest : BaseSessionTest(noErrorCollector = true) {
         assertThat("Callback count should be correct", counter, equalTo(2))
     }
 
+    @Test(expected = AssertionError::class)
+    @NullDelegate(GeckoSession.ProgressDelegate::class)
+    @ClosedSessionAtStart
+    fun waitForPageStops_throwOnNullDelegate() {
+        sessionRule.session.open(sessionRule.runtime) // Avoid waiting for initial load
+        sessionRule.session.reload()
+        sessionRule.session.waitForPageStops(2)
+    }
+
     @Test fun waitUntilCalled_anyInterfaceMethod() {
         sessionRule.session.loadTestPath(HELLO_HTML_PATH)
         sessionRule.waitUntilCalled(GeckoSession.ProgressDelegate::class)
@@ -169,6 +212,19 @@ class GeckoSessionTestRuleTest : BaseSessionTest(noErrorCollector = true) {
         sessionRule.waitUntilCalled(Callbacks.ProgressDelegate::class)
     }
 
+    @Test(expected = AssertionError::class)
+    @NullDelegate(GeckoSession.ScrollDelegate::class)
+    fun waitUntilCalled_throwOnNullDelegateInterface() {
+        sessionRule.session.reload()
+        sessionRule.session.waitUntilCalled(Callbacks.All::class)
+    }
+
+    @NullDelegate(GeckoSession.ScrollDelegate::class)
+    @Test fun waitUntilCalled_notThrowOnNonNullDelegateMethod() {
+        sessionRule.session.reload()
+        sessionRule.session.waitUntilCalled(Callbacks.All::class, "onPageStop")
+    }
+
     @Test fun waitUntilCalled_anyObjectMethod() {
         sessionRule.session.loadTestPath(HELLO_HTML_PATH)
 
@@ -212,6 +268,27 @@ class GeckoSessionTestRuleTest : BaseSessionTest(noErrorCollector = true) {
         assertThat("Callback count should be correct", counter, equalTo(2))
     }
 
+    @Test(expected = AssertionError::class)
+    @NullDelegate(GeckoSession.ScrollDelegate::class)
+    fun waitUntilCalled_throwOnNullDelegateObject() {
+        sessionRule.session.reload()
+        sessionRule.session.waitUntilCalled(object : Callbacks.All {
+            @AssertCalled
+            override fun onScrollChanged(session: GeckoSession, scrollX: Int, scrollY: Int) {
+            }
+        })
+    }
+
+    @NullDelegate(GeckoSession.ScrollDelegate::class)
+    @Test fun waitUntilCalled_notThrowOnNonNullDelegateObject() {
+        sessionRule.session.reload()
+        sessionRule.session.waitUntilCalled(object : Callbacks.All {
+            @AssertCalled
+            override fun onPageStop(session: GeckoSession, success: Boolean) {
+            }
+        })
+    }
+
     @Test fun waitUntilCalled_multipleCount() {
         sessionRule.session.loadTestPath(HELLO_HTML_PATH)
         sessionRule.session.reload()
@@ -544,6 +621,40 @@ class GeckoSessionTestRuleTest : BaseSessionTest(noErrorCollector = true) {
         })
     }
 
+    @Test(expected = AssertionError::class)
+    @NullDelegate(GeckoSession.ScrollDelegate::class)
+    fun forCallbacksDuringWait_throwOnAnyNullDelegate() {
+        sessionRule.session.reload()
+        sessionRule.session.waitForPageStop()
+
+        sessionRule.session.forCallbacksDuringWait(object : Callbacks.All {})
+    }
+
+    @Test(expected = AssertionError::class)
+    @NullDelegate(GeckoSession.ScrollDelegate::class)
+    fun forCallbacksDuringWait_throwOnSpecificNullDelegate() {
+        sessionRule.session.reload()
+        sessionRule.session.waitForPageStop()
+
+        sessionRule.session.forCallbacksDuringWait(object : Callbacks.All {
+            @AssertCalled
+            override fun onScrollChanged(session: GeckoSession, scrollX: Int, scrollY: Int) {
+            }
+        })
+    }
+
+    @NullDelegate(GeckoSession.ScrollDelegate::class)
+    @Test fun forCallbacksDuringWait_notThrowOnNonNullDelegate() {
+        sessionRule.session.reload()
+        sessionRule.session.waitForPageStop()
+
+        sessionRule.session.forCallbacksDuringWait(object : Callbacks.All {
+            @AssertCalled
+            override fun onPageStop(session: GeckoSession, success: Boolean) {
+            }
+        })
+    }
+
     @Test(expected = AssertionError::class)
     fun getCurrentCall_throwOnNoCurrentCall() {
         sessionRule.currentCall
@@ -734,6 +845,15 @@ class GeckoSessionTestRuleTest : BaseSessionTest(noErrorCollector = true) {
         sessionRule.waitForPageStop()
     }
 
+    @Test(expected = AssertionError::class)
+    @NullDelegate(GeckoSession.NavigationDelegate::class)
+    fun delegateDuringNextWait_throwOnNullDelegate() {
+        sessionRule.session.delegateDuringNextWait(object : Callbacks.NavigationDelegate {
+            override fun onLocationChange(session: GeckoSession, url: String) {
+            }
+        })
+    }
+
     @Test fun wrapSession() {
         val session = sessionRule.wrapSession(
           GeckoSession(sessionRule.session.settings))
@@ -1056,4 +1176,96 @@ class GeckoSessionTestRuleTest : BaseSessionTest(noErrorCollector = true) {
         sessionRule.session.synthesizeTap(5, 5)
         sessionRule.session.waitForPageStop()
     }
+
+    @WithDevToolsAPI
+    @Test fun evaluateJS() {
+        assertThat("JS string result should be correct",
+                   sessionRule.session.evaluateJS("'foo'") as String, equalTo("foo"))
+
+        assertThat("JS number result should be correct",
+                   sessionRule.session.evaluateJS("1+1") as Double, equalTo(2.0))
+
+        assertThat("JS boolean result should be correct",
+                   sessionRule.session.evaluateJS("!0") as Boolean, equalTo(true))
+
+        assertThat("JS object result should be correct",
+                   sessionRule.session.evaluateJS("({foo:'bar',bar:42,baz:true})").asJSMap(),
+                   equalTo(mapOf("foo" to "bar", "bar" to 42.0, "baz" to true)))
+
+        assertThat("JS array result should be correct",
+                   sessionRule.session.evaluateJS("[1,2,3]").asJSList(),
+                   equalTo(listOf(1.0, 2.0, 3.0)))
+
+        assertThat("JS DOM object result should be correct",
+                   sessionRule.session.evaluateJS("document.body").asJSMap(),
+                   hasEntry("tagName", "BODY"))
+
+        assertThat("JS DOM array result should be correct",
+                   sessionRule.session.evaluateJS("document.childNodes").asJSList<Any>(),
+                   not(empty()))
+    }
+
+    @WithDevToolsAPI
+    @Test fun evaluateJS_windowObject() {
+        sessionRule.session.loadTestPath(HELLO_HTML_PATH)
+        sessionRule.session.waitForPageStop()
+
+        // Make sure we can access large objects like "window", which can strain our RDP connection.
+        // Also make sure we can dynamically access sub-objects like Location.
+        assertThat("JS DOM window result should be correct",
+                   (sessionRule.session.evaluateJS("window")
+                           dot "location"
+                           dot "pathname") as String,
+                   equalTo(HELLO_HTML_PATH))
+    }
+
+    @WithDevToolsAPI
+    @Test fun evaluateJS_multipleSessions() {
+        val newSession = sessionRule.createOpenSession()
+
+        sessionRule.session.evaluateJS("this.foo = 42")
+        assertThat("Variable should be set",
+                   sessionRule.session.evaluateJS("this.foo") as Double, equalTo(42.0))
+
+        assertThat("New session should have separate JS context",
+                   newSession.evaluateJS("this.foo"), nullValue())
+    }
+
+    @WithDevToolsAPI
+    @Test fun evaluateJS_jsToString() {
+        val obj = sessionRule.session.evaluateJS("({foo:'bar'})")
+        assertThat("JS object toString should follow lazy evaluation",
+                   obj.toString(), equalTo("[Object]"))
+
+        assertThat("JS object should be correct",
+                   (obj dot "foo") as String, equalTo("bar"))
+        assertThat("JS object toString should be expanded after evaluation",
+                   obj.toString(), equalTo("[Object]{foo=bar}"))
+
+        val array = sessionRule.session.evaluateJS("['foo','bar']")
+        assertThat("JS array toString should follow lazy evaluation",
+                   array.toString(), equalTo("[Array(2)]"))
+
+        assertThat("JS array should be correct",
+                   (array dot 0) as String, equalTo("foo"))
+        assertThat("JS array toString should be expanded after evaluation",
+                   array.toString(), equalTo("[Array(2)][foo, bar]"))
+    }
+
+    @Test(expected = AssertionError::class)
+    fun evaluateJS_throwOnNotWithDevTools() {
+        sessionRule.session.evaluateJS("0")
+    }
+
+    @WithDevToolsAPI
+    @Test(expected = RuntimeException::class)
+    fun evaluateJS_throwOnJSException() {
+        sessionRule.session.evaluateJS("throw Error()")
+    }
+
+    @WithDevToolsAPI
+    @Test(expected = RuntimeException::class)
+    fun evaluateJS_throwOnSyntaxError() {
+        sessionRule.session.evaluateJS("<{[")
+    }
 }

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/NavigationDelegateTest.kt

@@ -4,10 +4,10 @@
 
 package org.mozilla.geckoview.test
 
-import android.support.test.InstrumentationRegistry
 import org.mozilla.geckoview.GeckoResponse
 import org.mozilla.geckoview.GeckoSession
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.AssertCalled
+import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.NullDelegate
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.WithDisplay
 import org.mozilla.geckoview.test.util.Callbacks
 
@@ -84,6 +84,16 @@ class NavigationDelegateTest : BaseSessionTest() {
         })
     }
 
+    @NullDelegate(GeckoSession.NavigationDelegate::class)
+    @Test fun load_withoutNavigationDelegate() {
+        // Test that when navigation delegate is disabled, we can still perform loads.
+        sessionRule.session.loadTestPath(HELLO_HTML_PATH)
+        sessionRule.session.waitForPageStop()
+
+        sessionRule.session.reload()
+        sessionRule.session.waitForPageStop()
+    }
+
     @Test fun loadString() {
         val dataString = "<html><head><title>TheTitle</title></head><body>TheBody</body></html>"
         val mimeType = "text/html"

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/SessionLifecycleTest.kt

@@ -6,6 +6,9 @@ package org.mozilla.geckoview.test
 
 import org.mozilla.geckoview.GeckoSession
 import org.mozilla.geckoview.GeckoSessionSettings
+import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.ClosedSessionAtStart
+import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.NullDelegate
+import org.mozilla.geckoview.test.util.Callbacks
 
 import android.support.test.filters.MediumTest
 import android.support.test.runner.AndroidJUnit4
@@ -190,6 +193,36 @@ class SessionLifecycleTest : BaseSessionTest() {
         session3.waitForPageStop()
     }
 
+    @NullDelegate(GeckoSession.NavigationDelegate::class)
+    @ClosedSessionAtStart
+    @Test fun readFromParcel_moduleUpdated() {
+        val session = sessionRule.createOpenSession()
+
+        // Disable navigation notifications on the old, open session.
+        assertThat("Old session navigation delegate should be null",
+                   session.navigationDelegate, nullValue())
+
+        // Enable navigation notifications on the new, closed session.
+        var onLocationCount = 0
+        sessionRule.session.navigationDelegate = object : Callbacks.NavigationDelegate {
+            override fun onLocationChange(session: GeckoSession, url: String) {
+                onLocationCount++
+            }
+        }
+
+        // Transferring the old session to the new session should
+        // automatically re-enable navigation notifications.
+        session.toParcel { parcel ->
+            sessionRule.session.readFromParcel(parcel)
+        }
+
+        sessionRule.session.reload()
+        sessionRule.session.waitForPageStop()
+
+        assertThat("New session should receive navigation notifications",
+                   onLocationCount, equalTo(1))
+    }
+
     @Test fun createFromParcel() {
         val session = sessionRule.createOpenSession()
 

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rdp/Actor.java

@@ -0,0 +1,83 @@
+/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.geckoview.test.rdp;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * Base class for actors in the remote debugging protocol. Provides basic methods such as
+ * {@link #sendPacket}. The actor is automatically registered with the connection on
+ * creation, and its {@link onPacket} method is called whenever a packet is received with
+ * the actor as the target.
+ */
+public class Actor {
+    public final RDPConnection connection;
+    public final String name;
+    protected JSONObject mReply;
+
+    protected Actor(final RDPConnection connection, final JSONObject packet) {
+        this(connection, packet.optString("actor", null));
+    }
+
+    protected Actor(final RDPConnection connection, final String name) {
+        if (name == null) {
+            throw new IllegalArgumentException();
+        }
+        this.connection = connection;
+        this.name = name;
+        connection.addActor(name, this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return (o instanceof Actor) && name.equals(((Actor) o).name);
+    }
+
+    @Override
+    public int hashCode() {
+        return name.hashCode();
+    }
+
+    protected void release() {
+        connection.removeActor(name);
+    }
+
+    protected JSONObject sendPacket(final String packet, final String replyProp) {
+        if (packet.charAt(0) != '{') {
+            throw new IllegalArgumentException();
+        }
+        connection.sendRawPacket("{\"to\":" + JSONObject.quote(name) + ',' + packet.substring(1));
+        return getReply(replyProp);
+    }
+
+    protected JSONObject sendPacket(final JSONObject packet, final String replyProp) {
+        try {
+            packet.put("to", name);
+            connection.sendRawPacket(packet);
+            return getReply(replyProp);
+        } catch (final JSONException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected void onPacket(final JSONObject packet) {
+        mReply = packet;
+    }
+
+    protected JSONObject getReply(final String replyProp) {
+        mReply = null;
+        do {
+            connection.dispatchInputPacket();
+
+            if (mReply != null && replyProp != null && !mReply.has(replyProp)) {
+                // Out-of-band notifications not supported currently.
+                mReply = null;
+            }
+        } while (mReply == null);
+        return mReply;
+    }
+}

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rdp/Console.java

@@ -0,0 +1,39 @@
+/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.geckoview.test.rdp;
+
+import org.json.JSONObject;
+
+/**
+ * Provide access to the webconsole API.
+ */
+public final class Console extends Actor {
+    /* package */ Console(final RDPConnection connection, final String name) {
+        super(connection, name);
+    }
+
+    /**
+     * Evaluate a JavaScript expression within the scope of this actor, and return its
+     * result. Null and undefined are converted to null. Boolean and string results are
+     * converted to their Java counterparts. Number results are converted to Double.
+     * Array-like object results, including Array, arguments, and NodeList, are converted
+     * to {@code List<Object>}. Other object results, including DOM nodes, are converted
+     * to {@code Map<String, Object>}.
+     *
+     * @param js JavaScript expression.
+     * @return Result of the evaluation.
+     */
+    public Object evaluateJS(final String js) {
+        final JSONObject reply = sendPacket("{\"type\":\"evaluateJS\",\"text\":" +
+                                                    JSONObject.quote(js) + '}',
+                                            "result");
+        if (reply.has("exception") && !reply.isNull("exception")) {
+            throw new RuntimeException("JS exception: " + reply.optString("exceptionMessage",
+                                                                          null));
+        }
+        return Grip.unpack(connection, reply.opt("result"));
+    }
+}

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rdp/Grip.java

@@ -0,0 +1,292 @@
+/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.geckoview.test.rdp;
+
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+
+import org.json.JSONObject;
+
+import java.util.AbstractList;
+import java.util.AbstractMap;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Provide methods for interacting with grips, including unpacking grips into Java
+ * objects.
+ */
+/* package */ final class Grip extends Actor {
+
+    private static final class Cache extends HashMap<String, Object> {
+    }
+
+    private static final class LazyObject extends AbstractMap<String, Object> {
+        private final Cache mCache;
+        private final String mType;
+        private Grip mGrip;
+        private Map<String, Object> mRealObject;
+
+        public LazyObject(final @NonNull Cache cache,
+                          final @NonNull String type,
+                          final @NonNull Grip grip) {
+            mCache = cache;
+            mType = type;
+            mGrip = grip;
+
+            cache.put(mGrip.name, this);
+        }
+
+        private Map<String, Object> ensureRealObject() {
+            if (mRealObject == null) {
+                mRealObject = mGrip.unpackAsObject(mCache);
+                mGrip.release();
+                mGrip = null;
+            }
+            return mRealObject;
+        }
+
+        @Override
+        public boolean equals(final Object object) {
+            if (object instanceof LazyObject) {
+                final LazyObject other = (LazyObject) object;
+                if (mGrip != null && other.mGrip != null) {
+                    return mGrip.equals(other.mGrip);
+                }
+                return ensureRealObject().equals(other.ensureRealObject());
+            }
+            return ensureRealObject().equals(object);
+        }
+
+        @Override
+        public String toString() {
+            return "[" + mType + ']' + (mRealObject != null ? mRealObject : "");
+        }
+
+        @Override
+        public Set<Entry<String, Object>> entrySet() {
+            return ensureRealObject().entrySet();
+        }
+
+        @Override
+        public boolean containsKey(final Object key) {
+            return ensureRealObject().containsKey(key);
+        }
+
+        @Override
+        public Object get(final Object key) {
+            return ensureRealObject().get(key);
+        }
+
+        @Override
+        public Set<String> keySet() {
+            return ensureRealObject().keySet();
+        }
+    }
+
+    private static final class LazyArray extends AbstractList<Object> {
+        private final Cache mCache;
+        private final String mType;
+        private final int mLength;
+        private Grip mGrip;
+        private List<Object> mRealObject;
+
+        public LazyArray(final @NonNull Cache cache,
+                         final @NonNull String type,
+                         final int length,
+                         final @NonNull Grip grip) {
+            mCache = cache;
+            mType = type;
+            mLength = length;
+            mGrip = grip;
+
+            cache.put(mGrip.name, this);
+        }
+
+        private List<Object> ensureRealObject() {
+            if (mRealObject == null) {
+                mRealObject = mGrip.unpackAsArray(mCache);
+                mGrip.release();
+                mGrip = null;
+            }
+            return mRealObject;
+        }
+
+        @Override
+        public boolean equals(Object object) {
+            if (object instanceof LazyArray) {
+                final LazyArray other = (LazyArray) object;
+                if (mGrip != null && other.mGrip != null) {
+                    return mGrip.equals(other.mGrip);
+                }
+                return ensureRealObject().equals(other.ensureRealObject());
+            }
+            return ensureRealObject().equals(object);
+        }
+
+        @Override
+        public String toString() {
+            final String length = (mRealObject != null) ? ("(" + mRealObject.size() + ')') :
+                                  (mLength >= 0)        ? ("(" + mLength + ')') : "";
+            return "[" + mType + length + ']' + (mRealObject != null ? mRealObject : "");
+        }
+
+        @Override
+        public Object get(int i) {
+            return ensureRealObject().get(i);
+        }
+
+        @Override
+        public int size() {
+            return ensureRealObject().size();
+        }
+    }
+
+    private static final class Function {
+        @Override
+        public String toString() {
+            return "[Function]";
+        }
+    }
+
+    /**
+     * Unpack a received grip value into a Java object. The grip can be either a primitive
+     * value, or a JSONObject that represents a live object on the server.
+     *
+     * @param connection Connection associated with this grip.
+     * @param value Grip value received from the server.
+     */
+    public static Object unpack(final RDPConnection connection,
+                                final Object value) {
+        return unpackGrip(new Cache(), connection, value);
+    }
+
+    private static Object unpackGrip(final Cache cache,
+                                     final RDPConnection connection,
+                                     final Object value) {
+        if (value == null || value instanceof String || value instanceof Boolean) {
+            return value;
+        } else if (value instanceof Number) {
+            return ((Number) value).doubleValue();
+        }
+
+        final JSONObject obj = (JSONObject) value;
+        switch (obj.optString("type")) {
+            case "null":
+            case "undefined":
+                return null;
+            case "Infinity":
+                return Double.POSITIVE_INFINITY;
+            case "-Infinity":
+                return Double.NEGATIVE_INFINITY;
+            case "NaN":
+                return Double.NaN;
+            case "-0":
+                return -0.0;
+            case "object":
+                break;
+            default:
+                throw new IllegalArgumentException();
+        }
+
+        final String actor = obj.optString("actor", null);
+        final Object cached = cache.get(actor);
+        if (cached != null) {
+            return cached;
+        }
+
+        final String cls = obj.optString("class", null);
+        if ("Function".equals(cls)) {
+            return new Function();
+        }
+
+        final JSONObject preview = obj.optJSONObject("preview");
+        final boolean isArray;
+        if ("Array".equals(cls)) {
+            isArray = true;
+        } else if (preview != null) {
+            isArray = "ArrayLike".equals(preview.optString("kind"));
+        } else {
+            isArray = false;
+        }
+
+        final Grip grip = new Grip(connection, obj);
+        final Object output;
+        if (isArray) {
+            final int length = (preview != null) ? preview.optInt("length", -1) : -1;
+            output = new LazyArray(cache, cls, length, grip);
+        } else {
+            output = new LazyObject(cache, cls, grip);
+        }
+        return output;
+    }
+
+    private Grip(final RDPConnection connection, final JSONObject grip) {
+        super(connection, grip);
+    }
+
+    @Override
+    protected void release() {
+        sendPacket("{\"type\":\"release\"}", null);
+        super.release();
+    }
+
+    /* package */ List<Object> unpackAsArray(final @NonNull Cache cache) {
+        final JSONObject reply = sendPacket("{\"type\":\"prototypeAndProperties\"}",
+                                            "ownProperties");
+        final JSONObject props = reply.optJSONObject("ownProperties");
+        final JSONObject getterValues = reply.optJSONObject("safeGetterValues");
+
+        JSONObject prop = props.optJSONObject("length");
+        String valueKey = "value";
+        if (prop == null) {
+            prop = getterValues.optJSONObject("length");
+            valueKey = "getterValue";
+        }
+
+        final int len = prop.optInt(valueKey);
+        final Object[] output = new Object[len];
+        for (int i = 0; i < len; i++) {
+            prop = props.optJSONObject(String.valueOf(i));
+            valueKey = "value";
+            if (prop == null) {
+                prop = getterValues.optJSONObject(String.valueOf(i));
+                valueKey = "getterValue";
+            }
+            output[i] = unpackGrip(cache, connection, prop.opt(valueKey));
+        }
+        return Arrays.asList(output);
+    }
+
+    /* package */ Map<String, Object> unpackAsObject(final @NonNull Cache cache) {
+        final JSONObject reply = sendPacket("{\"type\":\"prototypeAndProperties\"}",
+                                            "ownProperties");
+        final Map<String, Object> output = new HashMap<>();
+
+        fillProperties(cache, output, reply.optJSONObject("ownProperties"), "value");
+        fillProperties(cache, output, reply.optJSONObject("safeGetterValues"), "getterValue");
+        return output;
+    }
+
+    private void fillProperties(final @NonNull Cache cache,
+                                final @NonNull Map<String, Object> output,
+                                final @Nullable JSONObject props,
+                                final @NonNull String valueKey) {
+        if (props == null) {
+            return;
+        }
+        for (final Iterator<String> it = props.keys(); it.hasNext();) {
+            final String key = it.next();
+            final JSONObject prop = props.optJSONObject(key);
+            final Object value = prop.opt(valueKey);
+            output.put(key, unpackGrip(cache, connection, value));
+        }
+    }
+}

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rdp/RDPConnection.java

@@ -0,0 +1,206 @@
+/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.geckoview.test.rdp;
+
+import android.net.LocalSocket;
+import android.net.LocalSocketAddress;
+import android.util.Log;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.BufferedInputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+
+/**
+ * Class for connecting to a remote debugging protocol server, and retrieving various
+ * actors after connection. After establishing a connection, use {@link #getMostRecentTab}
+ * to get the actor for the most recent tab, which allows further interactions with the
+ * tab.
+ */
+public final class RDPConnection implements Closeable {
+    private static final String LOGTAG = "GeckoRDPConnection";
+
+    private final LocalSocket mSocket = new LocalSocket();
+    private final InputStream mInput;
+    private final OutputStream mOutput;
+    private final HashMap<String, Actor> mActors = new HashMap<>();
+    private final Actor mRoot = new Actor(this, "root");
+    private final JSONObject mRuntimeInfo;
+
+    {
+        mActors.put(mRoot.name, mRoot);
+    }
+
+    /**
+     * Create a connection to a server.
+     *
+     * @param address Address to the remote debugging protocol socket; can be an address
+     * in either the filesystem or the abstract namespace.
+     */
+    public RDPConnection(final LocalSocketAddress address) {
+        try {
+            mSocket.connect(address);
+            mInput = new BufferedInputStream(mSocket.getInputStream());
+            mOutput = mSocket.getOutputStream();
+            mRuntimeInfo = mRoot.getReply(null);
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Get the socket timeout.
+     *
+     * @return Socket timeout in milliseconds.
+     */
+    public int getTimeout() {
+        try {
+            return mSocket.getSoTimeout();
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Set the socket timeout. IOException is thrown if the timeout expires while waiting
+     * for a socket operation.
+     */
+    public void setTimeout(final int timeout) {
+        try {
+            mSocket.setSoTimeout(timeout);
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Close the server connection.
+     */
+    @Override
+    public void close() {
+        try {
+            mOutput.close();
+            mSocket.shutdownOutput();
+            mInput.close();
+            mSocket.shutdownInput();
+            mSocket.close();
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /* package */ void addActor(final String name, final Actor actor) {
+        mActors.put(name, actor);
+    }
+
+    /* package */ void removeActor(final String name) {
+        mActors.remove(name);
+    }
+
+    /* package */ Actor getActor(final JSONObject packet) {
+        return mActors.get(packet.optString("actor", null));
+    }
+
+    /* package */ Actor getActor(final String name) {
+        return mActors.get(name);
+    }
+
+    /* package */ void sendRawPacket(final JSONObject packet) {
+        sendRawPacket(packet.toString());
+    }
+
+    /* package */ void sendRawPacket(final String packet) {
+        try {
+            final byte[] buffer = packet.getBytes("utf-8");
+            final byte[] header = (String.valueOf(buffer.length) + ':').getBytes("utf-8");
+            mOutput.write(header);
+            mOutput.write(buffer);
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /* package */ void dispatchInputPacket() {
+        try {
+            byte[] buffer = new byte[128];
+            int len = 0;
+            for (int c = mInput.read(); c != ':'; c = mInput.read()) {
+                if (c == -1) {
+                    throw new IllegalStateException("EOF reached");
+                }
+                buffer[len++] = (byte) c;
+            }
+
+            final String header = new String(buffer, 0, len, "utf-8");
+            final int length;
+            try {
+                length = Integer.valueOf(header.substring(header.lastIndexOf(' ') + 1));
+            } catch (final NumberFormatException e) {
+                throw new RuntimeException("Invalid packet header: " + header);
+            }
+
+            if (header.startsWith("bulk ")) {
+                // Bulk packet not supported; skip the data.
+                mInput.skip(length);
+                return;
+            }
+
+            // JSON packet.
+            if (length > buffer.length) {
+                buffer = new byte[length];
+            }
+            int cursor = 0;
+            do {
+                final int read = mInput.read(buffer, cursor, length - cursor);
+                if (read <= 0) {
+                    throw new IllegalStateException("EOF reached");
+                }
+                cursor += read;
+            } while (cursor < length);
+
+            final String str = new String(buffer, 0, length, "utf-8");
+            final JSONObject json;
+            try {
+                json = new JSONObject(str);
+            } catch (final JSONException e) {
+                throw new RuntimeException(e);
+            }
+
+            final String error = json.optString("error", null);
+            if (error != null) {
+                throw new UnsupportedOperationException("Request failed: " + error);
+            }
+
+            final String from = json.optString("from", "none");
+            final Actor actor = mActors.get(from);
+            if (actor != null) {
+                actor.onPacket(json);
+            } else {
+                Log.w(LOGTAG, "Packet from unknown actor " + from);
+            }
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Get the actor for the most recent tab. For GeckoView, this tab represents the most
+     * recent GeckoSession.
+     *
+     * @return Tab actor.
+     */
+    public Tab getMostRecentTab() {
+        final JSONObject reply = mRoot.sendPacket("{\"type\":\"getTab\"}", "tab")
+                                      .optJSONObject("tab");
+        final Actor actor = getActor(reply);
+        return (actor != null) ? (Tab) actor : new Tab(this, reply);
+    }
+}

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rdp/Tab.java

@@ -0,0 +1,51 @@
+/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.geckoview.test.rdp;
+
+import org.json.JSONObject;
+
+/**
+ * Provide access to the tab API.
+ */
+public final class Tab extends Actor {
+    public final String title;
+    public final String url;
+    public final long outerWindowID;
+    private final JSONObject mTab;
+
+    /* package */ Tab(final RDPConnection connection, final JSONObject tab) {
+        super(connection, tab);
+        title = tab.optString("title", null);
+        url = tab.optString("url", null);
+        outerWindowID = tab.optLong("outerWindowID", -1);
+        mTab = tab;
+    }
+
+    /**
+     * Attach to the server tab.
+     */
+    public void attach() {
+        sendPacket("{\"type\":\"attach\"}", "type");
+    }
+
+    /**
+     * Detach from the server tab.
+     */
+    public void detach() {
+        sendPacket("{\"type\":\"detach\"}", "type");
+    }
+
+    /**
+     * Get the console object for access to the webconsole API.
+     *
+     * @return Console object.
+     */
+    public Console getConsole() {
+        final String name = mTab.optString("consoleActor", null);
+        final Actor console = connection.getActor(name);
+        return (console != null) ? (Console) console : new Console(connection, name);
+    }
+}

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java

@@ -11,10 +11,13 @@ import org.mozilla.geckoview.GeckoRuntime;
 import org.mozilla.geckoview.GeckoRuntimeSettings;
 import org.mozilla.geckoview.GeckoSession;
 import org.mozilla.geckoview.GeckoSessionSettings;
+import org.mozilla.geckoview.test.rdp.RDPConnection;
+import org.mozilla.geckoview.test.rdp.Tab;
 import org.mozilla.geckoview.test.util.Callbacks;
 
 import static org.hamcrest.Matchers.*;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
 
 import org.hamcrest.Matcher;
 
@@ -25,6 +28,7 @@ import org.junit.runners.model.Statement;
 import android.app.Instrumentation;
 import android.graphics.Point;
 import android.graphics.SurfaceTexture;
+import android.net.LocalSocketAddress;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Debug;
@@ -129,6 +133,32 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
         boolean value() default true;
     }
 
+    /**
+     * Specify that the test will set a delegate to null when creating a session, rather
+     * than setting the delegate to a proxy. The test cannot wait on any delegates that
+     * are set to null.
+     */
+    @Target({ElementType.METHOD, ElementType.TYPE})
+    @Retention(RetentionPolicy.RUNTIME)
+    public @interface NullDelegate {
+        Class<?> value();
+
+        @Target({ElementType.METHOD, ElementType.TYPE})
+        @Retention(RetentionPolicy.RUNTIME)
+        @interface List {
+            NullDelegate[] value();
+        }
+    }
+
+    /**
+     * Specify that the test uses DevTools-enabled APIs, such as {@link #evaluateJS}.
+     */
+    @Target({ElementType.METHOD, ElementType.TYPE})
+    @Retention(RetentionPolicy.RUNTIME)
+    public @interface WithDevToolsAPI {
+        boolean value() default true;
+    }
+
     /**
      * Specify a list of GeckoSession settings to be applied to the GeckoSession object
      * under test. Can be used on classes or methods. Note that the settings values must
@@ -453,6 +483,9 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
                 if (!ifce.isInstance(callback)) {
                     continue;
                 }
+                assertThat("Cannot delegate null-delegate callbacks",
+                           ifce, not(isIn(mNullDelegates)));
+
                 for (final Method method : ifce.getMethods()) {
                     final Method callbackMethod;
                     try {
@@ -538,7 +571,9 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
     }
 
     private static final List<Class<?>> CALLBACK_CLASSES = Arrays.asList(getCallbackClasses());
+
     private static GeckoRuntime sRuntime;
+    private static RDPConnection sRDPConnection;
     private static long sLongestWait;
 
     public final Environment env = new Environment();
@@ -551,6 +586,7 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
     protected ErrorCollector mErrorCollector;
     protected GeckoSession mMainSession;
     protected Object mCallbackProxy;
+    protected Set<Class<?>> mNullDelegates;
     protected List<CallRecord> mCallRecords;
     protected CallRecordHandler mCallRecordHandler;
     protected CallbackDelegates mWaitScopeDelegates;
@@ -564,6 +600,8 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
     protected Surface mDisplaySurface;
     protected GeckoDisplay mDisplay;
     protected boolean mClosedSession;
+    protected boolean mWithDevTools;
+    protected Map<GeckoSession, Tab> mRDPTabs;
 
     public GeckoSessionTestRule() {
         mDefaultSettings = new GeckoSessionSettings();
@@ -664,6 +702,18 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
         return GeckoSession.class.getMethod("get" + cls.getSimpleName());
     }
 
+    private void addNullDelegate(final Class<?> delegate) {
+        if (!Callbacks.class.equals(delegate.getDeclaringClass())) {
+            assertThat("Null-delegate must be valid interface class",
+                       delegate, isIn(CALLBACK_CLASSES));
+            mNullDelegates.add(delegate);
+            return;
+        }
+        for (final Class<?> ifce : delegate.getInterfaces()) {
+            addNullDelegate(ifce);
+        }
+    }
+
     protected void applyAnnotations(final Collection<Annotation> annotations,
                                     final GeckoSessionSettings settings) {
         for (final Annotation annotation : annotations) {
@@ -678,11 +728,19 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
                 for (final Setting setting : ((Setting.List) annotation).value()) {
                     setting.key().set(settings, setting.value());
                 }
+            } else if (NullDelegate.class.equals(annotation.annotationType())) {
+                addNullDelegate(((NullDelegate) annotation).value());
+            } else if (NullDelegate.List.class.equals(annotation.annotationType())) {
+                for (final NullDelegate nullDelegate : ((NullDelegate.List) annotation).value()) {
+                    addNullDelegate(nullDelegate.value());
+                }
             } else if (WithDisplay.class.equals(annotation.annotationType())) {
                 final WithDisplay displaySize = (WithDisplay)annotation;
                 mDisplaySize = new Point(displaySize.width(), displaySize.height());
             } else if (ClosedSessionAtStart.class.equals(annotation.annotationType())) {
                 mClosedSession = ((ClosedSessionAtStart) annotation).value();
+            } else if (WithDevToolsAPI.class.equals(annotation.annotationType())) {
+                mWithDevTools = ((WithDevToolsAPI) annotation).value();
             }
         }
     }
@@ -711,7 +769,9 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
         final GeckoSessionSettings settings = new GeckoSessionSettings(mDefaultSettings);
         mTimeoutMillis = env.isDebugging() ? DEFAULT_IDE_DEBUG_TIMEOUT_MILLIS
                                            : getDefaultTimeoutMillis();
+        mNullDelegates = new HashSet<>();
         mClosedSession = false;
+        mWithDevTools = false;
 
         applyAnnotations(Arrays.asList(description.getTestClass().getAnnotations()), settings);
         applyAnnotations(description.getAnnotations(), settings);
@@ -724,6 +784,9 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
         mTestScopeDelegates = testDelegates;
         mLastWaitStart = 0;
         mLastWaitEnd = 0;
+        if (mWithDevTools) {
+            mRDPTabs = new HashMap<>();
+        }
 
         final InvocationHandler recorder = new InvocationHandler() {
             @Override
@@ -781,6 +844,8 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
                 runtimeSettingsBuilder.build());
         }
 
+        sRuntime.getSettings().setRemoteDebuggingEnabled(mWithDevTools);
+
         mMainSession = new GeckoSession(settings);
         prepareSession(mMainSession);
 
@@ -798,7 +863,7 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
 
     protected void prepareSession(final GeckoSession session) throws Throwable {
         for (final Class<?> cls : CALLBACK_CLASSES) {
-            if (cls != null) {
+            if (!mNullDelegates.contains(cls)) {
                 getCallbackSetter(cls).invoke(session, mCallbackProxy);
             }
         }
@@ -813,6 +878,22 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
     public void openSession(final GeckoSession session) {
         session.open(sRuntime);
         waitForInitialLoad(session);
+
+        if (mWithDevTools) {
+            if (sRDPConnection == null) {
+                final String dataDir = InstrumentationRegistry.getTargetContext()
+                                                              .getApplicationInfo().dataDir;
+                final LocalSocketAddress address = new LocalSocketAddress(
+                        dataDir + "/firefox-debugger-socket",
+                        LocalSocketAddress.Namespace.FILESYSTEM);
+                sRDPConnection = new RDPConnection(address);
+                sRDPConnection.setTimeout((int) Math.min(DEFAULT_TIMEOUT_MILLIS,
+                                                         Integer.MAX_VALUE));
+            }
+            final Tab tab = sRDPConnection.getMostRecentTab();
+            tab.attach();
+            mRDPTabs.put(session, tab);
+        }
     }
 
     private void waitForInitialLoad(final GeckoSession session) {
@@ -822,13 +903,22 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
         // and ignore everything in-between from that session.
 
         try {
+            // We cannot detect initial page load without progress delegate.
+            assertThat("ProgressDelegate cannot be null-delegate when opening session",
+                       GeckoSession.ProgressDelegate.class, not(isIn(mNullDelegates)));
+
+            // If navigation delegate is a null-delegate, instead of looking for
+            // onLocationChange(), start with the first call that targets this session.
+            final boolean nullNavigation = mNullDelegates.contains(
+                    GeckoSession.NavigationDelegate.class);
+
             mCallRecordHandler = new CallRecordHandler() {
                 private boolean mFoundStart = false;
 
                 @Override
                 public boolean handleCall(final Method method, final Object[] args) {
-                    if (!mFoundStart && sOnLocationChange.equals(method) &&
-                            session.equals(args[0]) && "about:blank".equals(args[1])) {
+                    if (!mFoundStart && session.equals(args[0]) && (nullNavigation ||
+                            (sOnLocationChange.equals(method) && "about:blank".equals(args[1])))) {
                         mFoundStart = true;
                         return true;
                     } else if (mFoundStart && session.equals(args[0])) {
@@ -859,6 +949,11 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
     }
 
     protected void cleanupSession(final GeckoSession session) {
+        final Tab tab = (mRDPTabs != null) ? mRDPTabs.get(session) : null;
+        if (tab != null) {
+            tab.detach();
+            mRDPTabs.remove(session);
+        }
         if (session.isOpen()) {
             session.close();
         }
@@ -882,12 +977,14 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
 
         mMainSession = null;
         mCallbackProxy = null;
+        mNullDelegates = null;
         mCallRecords = null;
         mWaitScopeDelegates = null;
         mTestScopeDelegates = null;
         mLastWaitStart = 0;
         mLastWaitEnd = 0;
         mTimeoutMillis = 0;
+        mRDPTabs = null;
     }
 
     @Override
@@ -1159,7 +1256,7 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
                 }
                 final AssertCalled ac = getAssertCalled(callbackMethod, callback);
                 if (ac != null && ac.value()) {
-                    methodCalls.add(new MethodCall(session, callbackMethod,
+                    methodCalls.add(new MethodCall(session, method,
                                                    ac, /* target */ null));
                 }
             }
@@ -1188,11 +1285,29 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
                     InvocationTargetException e) {
                 throw unwrapRuntimeException(e);
             }
+            if (mNullDelegates.contains(ifce)) {
+                // Null-delegates are initially null but are allowed to be any value.
+                continue;
+            }
             assertThat(ifce.getSimpleName() + " callbacks should be " +
                        "accessed through GeckoSessionTestRule delegate methods",
                        callback, sameInstance(mCallbackProxy));
         }
 
+        if (methodCalls.isEmpty()) {
+            // Waiting for any call on `delegate`; make sure it doesn't contain any null-delegates.
+            for (final Class<?> ifce : mNullDelegates) {
+                assertThat("Cannot wait on null-delegate callbacks",
+                           delegate, not(typeCompatibleWith(ifce)));
+            }
+        } else {
+            // Waiting for particular calls; make sure those calls aren't from a null-delegate.
+            for (final MethodCall call : methodCalls) {
+                assertThat("Cannot wait on null-delegate callbacks",
+                           call.method.getDeclaringClass(), not(isIn(mNullDelegates)));
+            }
+        }
+
         boolean calledAny = false;
         int index = mLastWaitStart = mLastWaitEnd;
 
@@ -1250,10 +1365,16 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
                                        final @NonNull Object callback) {
         final Method[] declaredMethods = callback.getClass().getDeclaredMethods();
         final List<MethodCall> methodCalls = new ArrayList<>(declaredMethods.length);
+        boolean assertingAnyCall = true;
+        Class<?> foundNullDelegate = null;
+
         for (final Class<?> ifce : CALLBACK_CLASSES) {
             if (!ifce.isInstance(callback)) {
                 continue;
             }
+            if (mNullDelegates.contains(ifce)) {
+                foundNullDelegate = ifce;
+            }
             for (final Method method : ifce.getMethods()) {
                 final Method callbackMethod;
                 try {
@@ -1262,12 +1383,24 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
                 } catch (final NoSuchMethodException e) {
                     throw new RuntimeException(e);
                 }
-                methodCalls.add(new MethodCall(
+                final MethodCall call = new MethodCall(
                         session, callbackMethod, getAssertCalled(callbackMethod, callback),
-                        /* target */ null));
+                        /* target */ null);
+                methodCalls.add(call);
+
+                if (call.requirement != null) {
+                    if (foundNullDelegate == ifce) {
+                        fail("Cannot assert on null-delegate " + ifce.getSimpleName());
+                    }
+                    assertingAnyCall = false;
+                }
             }
         }
 
+        if (assertingAnyCall && foundNullDelegate != null) {
+            fail("Cannot assert on null-delegate " + foundNullDelegate.getSimpleName());
+        }
+
         int order = 0;
         boolean calledAny = false;
 
@@ -1477,4 +1610,22 @@ public class GeckoSessionTestRule extends UiThreadTestRule {
         assertThat("Should be in a method call", mCurrentMethodCall, notNullValue());
         return values[Math.min(mCurrentMethodCall.getCurrentCount(), values.length) - 1];
     }
+
+    /**
+     * Evaluate a JavaScript expression in the context of the target page and return the result.
+     * RDP must be enabled first using the {@link WithDevToolsAPI} annotation. String, number, and
+     * boolean results are converted to Java values. Undefined and null results are returned as
+     * null. Objects are returned as Map instances. Arrays are returned as Object[] instances.
+     *
+     * @param session Session containing the target page.
+     * @param js JavaScript expression.
+     * @return Result of evaluating the expression.
+     */
+    public Object evaluateJS(final @NonNull GeckoSession session, final @NonNull String js) {
+        assertThat("Must enable RDP using @WithDevToolsAPI", mRDPTabs, notNullValue());
+
+        final Tab tab = mRDPTabs.get(session);
+        assertThat("Session should have tab object", tab, notNullValue());
+        return tab.getConsole().evaluateJS(js);
+    }
 }

modules/libpref/init/all.js

@@ -241,7 +241,7 @@ pref("dom.keyboardevent.keypress.dispatch_non_printable_keys_only_system_group_i
 // if you need to limit under a directory, the path should end with "/" like
 // "example.com/foo/".  Note that this cannot limit port number for now.
 pref("dom.keyboardevent.keypress.hack.dispatch_non_printable_keys",
-     "docs.google.com,mail.google.com,hangouts.google.com,keep.google.com,inbox.google.com,*.etherpad.org/p/,etherpad.wikimedia.org/p/,board.net/p/,pad.riseup.net/p/,*.sandstorm.io,factor.cc/pad/,*.etherpad.fr/p/,piratenpad.de/p/,notes.typo3.org/p/,etherpad.net/p/,*.framapad.org/p/,pad.ouvaton.coop/,pad.systemli.org/p/,pad.lqdn.fr/p/,public.etherpad-mozilla.org/p/,*.cloudron.me/p/,pad.aquilenet.fr/p/,free.primarypad.com/p/,pad.ondesk.work/p/,demo.maadix.org/etherpad/pads/");
+     "docs.google.com,mail.google.com,hangouts.google.com,keep.google.com,inbox.google.com,*.etherpad.org/p/,etherpad.wikimedia.org/p/,board.net/p/,pad.riseup.net/p/,*.sandstorm.io,factor.cc/pad/,*.etherpad.fr/p/,piratenpad.de/p/,notes.typo3.org/p/,etherpad.net/p/,*.framapad.org/p/,pad.ouvaton.coop/,pad.systemli.org/p/,pad.lqdn.fr/p/,public.etherpad-mozilla.org/p/,*.cloudron.me/p/,pad.aquilenet.fr/p/,free.primarypad.com/p/,pad.ondesk.work/p/,demo.maadix.org/etherpad/pads/,www.rememberthemilk.com");
 #else
 pref("dom.keyboardevent.keypress.dispatch_non_printable_keys_only_system_group_in_content", false);
 #endif

python/mozbuild/mozbuild/backend/tup.py

@@ -524,7 +524,7 @@ class TupBackend(CommonBackend):
             fh.write('topsrcdir = $(MOZ_OBJ_ROOT)/%s\n' % (
                 os.path.relpath(self.environment.topsrcdir, self.environment.topobjdir)
             ))
-            fh.write('PYTHON = PYTHONDONTWRITEBYTECODE=1 $(MOZ_OBJ_ROOT)/_virtualenv/bin/python\n')
+            fh.write('PYTHON = PYTHONDONTWRITEBYTECODE=1 %s\n' % self.environment.substs['PYTHON'])
             fh.write('PYTHON_PATH = $(PYTHON) $(topsrcdir)/config/pythonpath.py\n')
             fh.write('PLY_INCLUDE = -I$(topsrcdir)/other-licenses/ply\n')
             fh.write('IDL_PARSER_DIR = $(topsrcdir)/xpcom/idl-parser\n')

services/common/kinto-offline-client.js

@@ -33,7 +33,7 @@ const global = this;
 var EXPORTED_SYMBOLS = ["Kinto"];
 
 /*
- * Version 11.1.0 - 91f9229
+ * Version 11.1.2 - 2476e07
  */
 
 (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Kinto = f()}})(function(){var define,module,exports;return (function(){function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s}return e})()({1:[function(require,module,exports){
@@ -906,13 +906,24 @@ class SyncResultObject {
     if (!Array.isArray(this[type])) {
       return;
     }
+    if (!Array.isArray(entries)) {
+      entries = [entries];
+    }
     // Deduplicate entries by id. If the values don't have `id` attribute, just
     // keep all.
-    const deduplicated = this[type].concat(entries).reduce((acc, cur) => {
-      const existing = acc.filter(r => cur.id && r.id ? cur.id != r.id : true);
-      return existing.concat(cur);
-    }, []);
-    this[type] = deduplicated;
+    const recordsWithoutId = new Set();
+    const recordsById = new Map();
+    function addOneRecord(record) {
+      if (!record.id) {
+        recordsWithoutId.add(record);
+      } else {
+        recordsById.set(record.id, record);
+      }
+    }
+    this[type].forEach(addOneRecord);
+    entries.forEach(addOneRecord);
+
+    this[type] = Array.from(recordsById.values()).concat(Array.from(recordsWithoutId));
     this.ok = this.errors.length + this.conflicts.length === 0;
     return this;
   }

servo/components/style/properties/shorthand/outline.mako.rs

@@ -5,7 +5,7 @@
 <%namespace name="helpers" file="/helpers.mako.rs" />
 
 <%helpers:shorthand name="outline"
-                    sub_properties="outline-width outline-style outline-color"
+                    sub_properties="outline-color outline-style outline-width"
                     derive_serialize="True"
                     spec="https://drafts.csswg.org/css-ui/#propdef-outline">
     use properties::longhands::{outline_color, outline_width, outline_style};

testing/mozbase/mozinstall/mozinstall/mozinstall.py

@@ -103,7 +103,6 @@ def install(src, dest):
     :param dest: Path to install to (to ensure we do not overwrite any existent
                  files the folder should not exist yet)
     """
-
     if not is_installer(src):
         msg = "{} is not a valid installer file".format(src)
         if '://' in src:
@@ -311,7 +310,7 @@ def _install_dmg(src, dest):
 
     finally:
         if appDir:
-            subprocess.call('hdiutil detach %s -quiet' % appDir,
+            subprocess.call('hdiutil detach "%s" -quiet' % appDir,
                             shell=True)
 
     return dest

testing/mozbase/mozinstall/tests/conftest.py

@@ -0,0 +1,16 @@
+from __future__ import absolute_import
+
+import pytest
+
+
+@pytest.fixture
+def get_installer(request):
+    def _get_installer(extension):
+        """Get path to the installer for the specified extension."""
+        stub_dir = request.node.fspath.dirpath('installer_stubs')
+
+        # We had to remove firefox.exe since it is not valid for mozinstall 1.12 and higher
+        # Bug 1157352 - We should grab a firefox.exe from the build process or download it
+        return stub_dir.join('firefox.{}'.format(extension)).strpath
+
+    return _get_installer

testing/mozbase/mozinstall/tests/manifest.ini

@@ -1,3 +1,7 @@
 [DEFAULT]
 subsuite = mozbase, os == "linux"
-[test.py]
+
+[test_binary.py]
+[test_install.py]
+[test_is_installer.py]
+[test_uninstall.py]

testing/mozbase/mozinstall/tests/test.py

@@ -1,178 +0,0 @@
-#!/usr/bin/env python
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function
-
-import mozinfo
-import mozinstall
-import mozfile
-import os
-import tempfile
-import unittest
-
-import mozunit
-
-# Store file location at load time
-here = os.path.dirname(os.path.abspath(__file__))
-
-
-class TestMozInstall(unittest.TestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        """ Setting up stub installers """
-        cls.dmg = os.path.join(here, 'Installer-Stubs', 'firefox.dmg')
-        # XXX: We have removed firefox.exe since it is not valid for mozinstall 1.12 and higher
-        # Bug 1157352 - We should grab a firefox.exe from the build process or download it
-        cls.exe = os.path.join(here, 'Installer-Stubs', 'firefox.exe')
-        cls.zipfile = os.path.join(here, 'Installer-Stubs', 'firefox.zip')
-        cls.bz2 = os.path.join(here, 'Installer-Stubs', 'firefox.tar.bz2')
-
-    def setUp(self):
-        self.tempdir = tempfile.mkdtemp()
-
-    def tearDown(self):
-        mozfile.rmtree(self.tempdir)
-
-    @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe "
-                     "for mozinstall 1.12 and higher.")
-    def test_get_binary(self):
-        """ Test mozinstall's get_binary method """
-
-        if mozinfo.isLinux:
-            installdir = mozinstall.install(self.bz2, self.tempdir)
-            binary = os.path.join(installdir, 'firefox')
-            self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox'))
-
-        elif mozinfo.isWin:
-            installdir_exe = mozinstall.install(self.exe,
-                                                os.path.join(self.tempdir, 'exe'))
-            binary_exe = os.path.join(installdir_exe, 'core', 'firefox.exe')
-            self.assertEqual(binary_exe, mozinstall.get_binary(installdir_exe,
-                                                               'firefox'))
-
-            installdir_zip = mozinstall.install(self.zipfile,
-                                                os.path.join(self.tempdir, 'zip'))
-            binary_zip = os.path.join(installdir_zip, 'firefox.exe')
-            self.assertEqual(binary_zip, mozinstall.get_binary(installdir_zip,
-                                                               'firefox'))
-
-        elif mozinfo.isMac:
-            installdir = mozinstall.install(self.dmg, self.tempdir)
-            binary = os.path.join(installdir, 'Contents', 'MacOS', 'firefox')
-            self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox'))
-
-    def test_get_binary_error(self):
-        """ Test an InvalidBinary error is raised """
-
-        tempdir_empty = tempfile.mkdtemp()
-        self.assertRaises(mozinstall.InvalidBinary, mozinstall.get_binary,
-                          tempdir_empty, 'firefox')
-        mozfile.rmtree(tempdir_empty)
-
-    @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe "
-                     "for mozinstall 1.12 and higher.")
-    def test_is_installer(self):
-        """ Test we can identify a correct installer """
-
-        if mozinfo.isLinux:
-            self.assertTrue(mozinstall.is_installer(self.bz2))
-
-        if mozinfo.isWin:
-            # test zip installer
-            self.assertTrue(mozinstall.is_installer(self.zipfile))
-
-            # test exe installer
-            self.assertTrue(mozinstall.is_installer(self.exe))
-
-            try:
-                # test stub browser file
-                # without pefile on the system this test will fail
-                import pefile  # noqa
-                stub_exe = os.path.join(here, 'build_stub', 'firefox.exe')
-                self.assertFalse(mozinstall.is_installer(stub_exe))
-            except ImportError:
-                pass
-
-        if mozinfo.isMac:
-            self.assertTrue(mozinstall.is_installer(self.dmg))
-
-    def test_invalid_source_error(self):
-        """ Test InvalidSource error is raised with an incorrect installer """
-
-        if mozinfo.isLinux:
-            self.assertRaises(mozinstall.InvalidSource, mozinstall.install,
-                              self.dmg, 'firefox')
-
-        elif mozinfo.isWin:
-            self.assertRaises(mozinstall.InvalidSource, mozinstall.install,
-                              self.bz2, 'firefox')
-
-        elif mozinfo.isMac:
-            self.assertRaises(mozinstall.InvalidSource, mozinstall.install,
-                              self.bz2, 'firefox')
-
-        # Test an invalid url handler
-        self.assertRaises(mozinstall.InvalidSource, mozinstall.install,
-                          'file://foo.bar', 'firefox')
-
-    @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe "
-                     "for mozinstall 1.12 and higher.")
-    def test_install(self):
-        """ Test mozinstall's install capability """
-
-        if mozinfo.isLinux:
-            installdir = mozinstall.install(self.bz2, self.tempdir)
-            self.assertEqual(os.path.join(self.tempdir, 'firefox'), installdir)
-
-        elif mozinfo.isWin:
-            installdir_exe = mozinstall.install(self.exe,
-                                                os.path.join(self.tempdir, 'exe'))
-            self.assertEqual(os.path.join(self.tempdir, 'exe', 'firefox'),
-                             installdir_exe)
-
-            installdir_zip = mozinstall.install(self.zipfile,
-                                                os.path.join(self.tempdir, 'zip'))
-            self.assertEqual(os.path.join(self.tempdir, 'zip', 'firefox'),
-                             installdir_zip)
-
-        elif mozinfo.isMac:
-            installdir = mozinstall.install(self.dmg, self.tempdir)
-            self.assertEqual(os.path.join(os.path.realpath(self.tempdir),
-                                          'FirefoxStub.app'), installdir)
-
-    @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe "
-                     "for mozinstall 1.12 and higher.")
-    def test_uninstall(self):
-        """ Test mozinstall's uninstall capabilites """
-        # Uninstall after installing
-
-        if mozinfo.isLinux:
-            installdir = mozinstall.install(self.bz2, self.tempdir)
-            mozinstall.uninstall(installdir)
-            self.assertFalse(os.path.exists(installdir))
-
-        elif mozinfo.isWin:
-            # Exe installer for Windows
-            installdir_exe = mozinstall.install(self.exe,
-                                                os.path.join(self.tempdir, 'exe'))
-            mozinstall.uninstall(installdir_exe)
-            self.assertFalse(os.path.exists(installdir_exe))
-
-            # Zip installer for Windows
-            installdir_zip = mozinstall.install(self.zipfile,
-                                                os.path.join(self.tempdir, 'zip'))
-            mozinstall.uninstall(installdir_zip)
-            self.assertFalse(os.path.exists(installdir_zip))
-
-        elif mozinfo.isMac:
-            installdir = mozinstall.install(self.dmg, self.tempdir)
-            mozinstall.uninstall(installdir)
-            self.assertFalse(os.path.exists(installdir))
-
-
-if __name__ == '__main__':
-    mozunit.main()

testing/mozbase/mozinstall/tests/test_binary.py

@@ -0,0 +1,46 @@
+from __future__ import absolute_import
+
+import os
+
+import mozinfo
+import mozinstall
+import mozunit
+import pytest
+
+
+@pytest.mark.skipif(
+    mozinfo.isWin, reason='Bug 1157352 - New firefox.exe needed for mozinstall 1.12 and higher.')
+def test_get_binary(tmpdir, get_installer):
+    """Test to retrieve binary from install path."""
+    if mozinfo.isLinux:
+        installdir = mozinstall.install(get_installer('tar.bz2'), tmpdir.strpath)
+        binary = os.path.join(installdir, 'firefox')
+
+        assert mozinstall.get_binary(installdir, 'firefox') == binary
+
+    elif mozinfo.isWin:
+        installdir_exe = mozinstall.install(get_installer('exe'), tmpdir.join('exe').strpath)
+        binary_exe = os.path.join(installdir_exe, 'core', 'firefox.exe')
+
+        assert mozinstall.get_binary(installdir_exe, 'firefox') == binary_exe
+
+        installdir_zip = mozinstall.install(get_installer('zip'), tmpdir.join('zip').strpath)
+        binary_zip = os.path.join(installdir_zip, 'firefox.exe')
+
+        assert mozinstall.get_binary(installdir_zip, 'firefox') == binary_zip
+
+    elif mozinfo.isMac:
+        installdir = mozinstall.install(get_installer('dmg'), tmpdir.strpath)
+        binary = os.path.join(installdir, 'Contents', 'MacOS', 'firefox')
+
+        assert mozinstall.get_binary(installdir, 'firefox') == binary
+
+
+def test_get_binary_error(tmpdir):
+    """Test that an InvalidBinary error is raised."""
+    with pytest.raises(mozinstall.InvalidBinary):
+        mozinstall.get_binary(tmpdir.strpath, 'firefox')
+
+
+if __name__ == '__main__':
+    mozunit.main()

testing/mozbase/mozinstall/tests/test_install.py

@@ -0,0 +1,81 @@
+from __future__ import absolute_import
+
+import subprocess
+
+import mozinfo
+import mozinstall
+import mozunit
+import pytest
+
+
+@pytest.mark.skipif(
+    mozinfo.isWin, reason='Bug 1157352 - New firefox.exe needed for mozinstall 1.12 and higher.')
+def test_is_installer(request, get_installer):
+    """Test that we can identify a correct installer."""
+    if mozinfo.isLinux:
+        assert mozinstall.is_installer(get_installer('tar.bz2'))
+
+    if mozinfo.isWin:
+        # test zip installer
+        assert mozinstall.is_installer(get_installer('zip'))
+
+        # test exe installer
+        assert mozinstall.is_installer(get_installer('exe'))
+
+        try:
+            # test stub browser file
+            # without pefile on the system this test will fail
+            import pefile  # noqa
+            stub_exe = request.node.fspath.dirpath('build_stub').join('firefox.exe').strpath
+            assert not mozinstall.is_installer(stub_exe)
+        except ImportError:
+            pass
+
+    if mozinfo.isMac:
+        assert mozinstall.is_installer(get_installer('dmg'))
+
+
+def test_invalid_source_error(get_installer):
+    """Test that InvalidSource error is raised with an incorrect installer."""
+    if mozinfo.isLinux:
+        with pytest.raises(mozinstall.InvalidSource):
+            mozinstall.install(get_installer('dmg'), 'firefox')
+
+    elif mozinfo.isWin:
+        with pytest.raises(mozinstall.InvalidSource):
+            mozinstall.install(get_installer('tar.bz2'), 'firefox')
+
+    elif mozinfo.isMac:
+        with pytest.raises(mozinstall.InvalidSource):
+            mozinstall.install(get_installer('tar.bz2'), 'firefox')
+
+    # Test an invalid url handler
+    with pytest.raises(mozinstall.InvalidSource):
+        mozinstall.install('file://foo.bar', 'firefox')
+
+
+@pytest.mark.skipif(
+    mozinfo.isWin, reason='Bug 1157352 - New firefox.exe needed for mozinstall 1.12 and higher.')
+def test_install(tmpdir, get_installer):
+    """Test to install an installer."""
+    if mozinfo.isLinux:
+        installdir = mozinstall.install(get_installer('tar.bz2'), tmpdir.strpath)
+        assert installdir == tmpdir.join('firefox').strpath
+
+    elif mozinfo.isWin:
+        installdir_exe = mozinstall.install(get_installer('exe'), tmpdir.join('exe').strpath)
+        assert installdir_exe == tmpdir.join('exe', 'firefox').strpath
+
+        installdir_zip = mozinstall.install(get_installer('zip'), tmpdir.join('zip').strpath)
+        assert installdir_zip == tmpdir.join('zip', 'firefox').strpath
+
+    elif mozinfo.isMac:
+        installdir = mozinstall.install(get_installer('dmg'), tmpdir.strpath)
+        assert installdir == tmpdir.realpath().join('Firefox Stub.app').strpath
+
+        mounted_images = subprocess.check_output(['hdiutil', 'info'])
+        assert get_installer('dmg') not in mounted_images
+
+
+if __name__ == '__main__':
+    mozunit.main()

testing/mozbase/mozinstall/tests/test_is_installer.py

@@ -0,0 +1,37 @@
+from __future__ import absolute_import
+
+import mozinfo
+import mozinstall
+import mozunit
+import pytest
+
+
+@pytest.mark.skipif(
+    mozinfo.isWin, reason='Bug 1157352 - New firefox.exe needed for mozinstall 1.12 and higher.')
+def test_is_installer(request, get_installer):
+    """Test that we can identify a correct installer."""
+    if mozinfo.isLinux:
+        assert mozinstall.is_installer(get_installer('tar.bz2'))
+
+    if mozinfo.isWin:
+        # test zip installer
+        assert mozinstall.is_installer(get_installer('zip'))
+
+        # test exe installer
+        assert mozinstall.is_installer(get_installer('exe'))
+
+        try:
+            # test stub browser file
+            # without pefile on the system this test will fail
+            import pefile  # noqa
+            stub_exe = request.node.fspath.dirpath('build_stub').join('firefox.exe').strpath
+            assert not mozinstall.is_installer(stub_exe)
+        except ImportError:
+            pass
+
+    if mozinfo.isMac:
+        assert mozinstall.is_installer(get_installer('dmg'))
+
+
+if __name__ == '__main__':
+    mozunit.main()

testing/mozbase/mozinstall/tests/test_uninstall.py

@@ -0,0 +1,35 @@
+from __future__ import absolute_import
+
+import mozinfo
+import mozinstall
+import mozunit
+import py
+import pytest
+
+
+@pytest.mark.skipif(
+    mozinfo.isWin, reason='Bug 1157352 - New firefox.exe needed for mozinstall 1.12 and higher.')
+def test_uninstall(tmpdir, get_installer):
+    """Test to uninstall an installed binary."""
+    if mozinfo.isLinux:
+        installdir = mozinstall.install(get_installer('tar.bz2'), tmpdir.strpath)
+        mozinstall.uninstall(installdir)
+        assert not py.path.local(installdir).check()
+
+    elif mozinfo.isWin:
+        installdir_exe = mozinstall.install(get_installer('exe'), tmpdir.join('exe').strpath)
+        mozinstall.uninstall(installdir_exe)
+        assert not py.path.local(installdir).check()
+
+        installdir_zip = mozinstall.install(get_installer('zip'), tmpdir.join('zip').strpath)
+        mozinstall.uninstall(installdir_zip)
+        assert not py.path.local(installdir).check()
+
+    elif mozinfo.isMac:
+        installdir = mozinstall.install(get_installer('dmg'), tmpdir.strpath)
+        mozinstall.uninstall(installdir)
+        assert not py.path.local(installdir).check()
+
+
+if __name__ == '__main__':
+    mozunit.main()

testing/mozharness/configs/raptor/linux64_config_taskcluster.py

@@ -0,0 +1,45 @@
+import os
+import sys
+
+PYTHON = sys.executable
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+
+exes = {
+    'python': PYTHON,
+}
+ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
+INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": INSTALLER_PATH,
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "exes": exes,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+    "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+    "tooltool_cache": "/builds/worker/tooltool-cache",
+}

testing/mozharness/configs/raptor/linux_config.py

@@ -0,0 +1,41 @@
+import os
+import platform
+
+VENV_PATH = '%s/build/venv' % os.getcwd()
+if platform.architecture()[0] == '64bit':
+    TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+    MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+else:
+    TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
+    MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "setup-mitmproxy",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+    "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+    "tooltool_cache": "/builds/tooltool_cache",
+}

testing/mozharness/configs/raptor/mac_config.py

@@ -0,0 +1,50 @@
+ENABLE_SCREEN_RESOLUTION_CHECK = True
+
+SCREEN_RESOLUTION_CHECK = {
+    "name": "check_screen_resolution",
+    "cmd": ["bash", "-c", "screenresolution get && screenresolution list && system_profiler SPDisplaysDataType"],
+    "architectures": ["32bit", "64bit"],
+    "halt_on_failure": False,
+    "enabled": ENABLE_SCREEN_RESOLUTION_CHECK
+}
+
+import os
+
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "run_cmd_checks_enabled": True,
+    "preflight_run_cmd_suites": [
+        SCREEN_RESOLUTION_CHECK,
+    ],
+    "postflight_run_cmd_suites": [
+        SCREEN_RESOLUTION_CHECK,
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": "macosx64-minidump_stackwalk",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/macosx64/releng.manifest",
+    "tooltool_cache": "/builds/tooltool_cache",
+}

testing/mozharness/configs/raptor/windows_config.py

@@ -0,0 +1,56 @@
+import os
+import socket
+import sys
+
+PYTHON = sys.executable
+PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
+VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "pip_index": False,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
+    "exes": {
+        'python': PYTHON,
+        'easy_install': ['%s/scripts/python' % VENV_PATH,
+                         '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
+        'mozinstall': ['%s/scripts/python' % VENV_PATH,
+                       '%s/scripts/mozinstall-script.py' % VENV_PATH],
+        'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+        'tooltool.py': [PYTHON, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
+    },
+    "title": socket.gethostname().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
+    "download_minidump_stackwalk": True,
+    "tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
+    "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+    "python3_manifest": {
+        "win32": "python3.manifest",
+        "win64": "python3_x64.manifest",
+    },
+    "env": {
+        # python3 requires C runtime, found in firefox installation; see bug 1361732
+        "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
+    }
+}

testing/mozharness/configs/raptor/windows_vm_config.py

@@ -0,0 +1,55 @@
+import os
+import socket
+import sys
+
+PYTHON = sys.executable
+PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
+VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "pip_index": False,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
+    "exes": {
+        'python': PYTHON,
+        'easy_install': ['%s/scripts/python' % VENV_PATH,
+                         '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
+        'mozinstall': ['%s/scripts/python' % VENV_PATH,
+                       '%s/scripts/mozinstall-script.py' % VENV_PATH],
+        'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+    },
+    "title": socket.gethostname().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
+    "download_minidump_stackwalk": True,
+    "tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
+    "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+    "python3_manifest": {
+        "win32": "python3.manifest",
+        "win64": "python3_x64.manifest",
+    },
+    "env": {
+        # python3 requires C runtime, found in firefox installation; see bug 1361732
+        "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
+    }
+}

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -0,0 +1,401 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import os
+import re
+import sys
+
+import mozharness
+
+from mozharness.base.config import parse_config_file
+from mozharness.base.errors import PythonErrorList
+from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO, WARNING
+from mozharness.base.python import Python3Virtualenv
+from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.mozilla.tooltool import TooltoolMixin
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.testing.codecoverage import (
+    CodeCoverageMixin,
+    code_coverage_config_options
+)
+
+scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
+external_tools_path = os.path.join(scripts_path, 'external_tools')
+
+RaptorErrorList = PythonErrorList + [
+    {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
+    {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
+    {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
+    {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
+    {'substr': r'''raptorError''', 'level': CRITICAL},
+    {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
+    {'substr': r"""No such file or directory: 'browser_output.txt'""",
+     'level': CRITICAL,
+     'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
+]
+
+class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin):
+    """
+    install and run raptor tests
+    """
+    config_options = [
+        [["--test"],
+         {"action": "store",
+          "dest": "test",
+          "help": "Raptor test to run"
+          }],
+        [["--branch-name"],
+         {"action": "store",
+          "dest": "branch",
+          "help": "branch running against"
+          }],
+        [["--add-option"],
+         {"action": "extend",
+          "dest": "raptor_extra_options",
+          "default": None,
+          "help": "extra options to raptor"
+          }],
+    ] + testing_config_options + copy.deepcopy(blobupload_config_options) \
+                               + copy.deepcopy(code_coverage_config_options)
+
+    def __init__(self, **kwargs):
+        kwargs.setdefault('config_options', self.config_options)
+        kwargs.setdefault('all_actions', ['clobber',
+                                          'read-buildbot-config',
+                                          'download-and-extract',
+                                          'populate-webroot',
+                                          'create-virtualenv',
+                                          'install',
+                                          'run-tests',
+                                          ])
+        kwargs.setdefault('default_actions', ['clobber',
+                                              'download-and-extract',
+                                              'populate-webroot',
+                                              'create-virtualenv',
+                                              'install',
+                                              'run-tests',
+                                              ])
+        kwargs.setdefault('config', {})
+        super(Raptor, self).__init__(**kwargs)
+
+        self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
+
+        self.run_local = self.config.get('run_local')
+        self.installer_url = self.config.get("installer_url")
+        self.raptor_json_url = self.config.get("raptor_json_url")
+        self.raptor_json = self.config.get("raptor_json")
+        self.raptor_json_config = self.config.get("raptor_json_config")
+        self.repo_path = self.config.get("repo_path")
+        self.obj_path = self.config.get("obj_path")
+        self.tests = None
+        self.gecko_profile = self.config.get('gecko_profile')
+        self.gecko_profile_interval = self.config.get('gecko_profile_interval')
+        self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary
+        self.mitmproxy_pageset = None # zip file found on tooltool that contains all of the mitmproxy recordings
+        self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set
+        self.mitmdump = None # path to mitmdump tool itself, in py3 venv
+
+    # We accept some configuration options from the try commit message in the format mozharness: <options>
+    # Example try commit message:
+    #   mozharness: --geckoProfile try: <stuff>
+    def query_gecko_profile_options(self):
+        gecko_results = []
+        if self.buildbot_config:
+            # this is inside automation
+            # now let's see if we added GeckoProfile specs in the commit message
+            try:
+                junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:')
+            except IndexError:
+                # when we don't have comments on changes (bug 1255187)
+                opts = None
+
+            if opts:
+                # In the case of a multi-line commit message, only examine
+                # the first line for mozharness options
+                opts = opts.split('\n')[0]
+                opts = re.sub(r'\w+:.*', '', opts).strip().split(' ')
+                if "--geckoProfile" in opts:
+                    # overwrite whatever was set here.
+                    self.gecko_profile = True
+                try:
+                    idx = opts.index('--geckoProfileInterval')
+                    if len(opts) > idx + 1:
+                        self.gecko_profile_interval = opts[idx + 1]
+                except ValueError:
+                    pass
+            else:
+                # no opts, check for '--geckoProfile' in try message text directly
+                if self.try_message_has_flag('geckoProfile'):
+                    self.gecko_profile = True
+
+        # finally, if gecko_profile is set, we add that to the raptor options
+        if self.gecko_profile:
+            gecko_results.append('--geckoProfile')
+            if self.gecko_profile_interval:
+                gecko_results.extend(
+                    ['--geckoProfileInterval', str(self.gecko_profile_interval)]
+                )
+        return gecko_results
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(Raptor, self).query_abs_dirs()
+        abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+        abs_dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def raptor_options(self, args=None, **kw):
+        """return options to raptor"""
+        # binary path
+        binary_path = self.binary_path or self.config.get('binary_path')
+        if not binary_path:
+            self.fatal("Raptor requires a path to the binary.  You can specify binary_path or add download-and-extract to your action list.")
+        # raptor options
+        if binary_path.endswith('.exe'):
+            binary_path = binary_path[:-4]
+        options = []
+        kw_options = {'binary': binary_path}
+        # options overwritten from **kw
+        if 'suite' in self.config:
+            kw_options['suite'] = self.config['suite']
+        if self.config.get('branch'):
+            kw_options['branchName'] = self.config['branch']
+        if self.symbols_path:
+            kw_options['symbolsPath'] = self.symbols_path
+        kw_options.update(kw)
+        # configure profiling options
+        options.extend(self.query_gecko_profile_options())
+        # extra arguments
+        if args is not None:
+            options += args
+        if 'raptor_extra_options' in self.config:
+            options += self.config['raptor_extra_options']
+        if self.config.get('code_coverage', False):
+            options.extend(['--code-coverage'])
+        for key, value in kw_options.items():
+            options.extend(['--%s' % key, value])
+        return options
+
+    def populate_webroot(self):
+        """Populate the production test slaves' webroots"""
+        self.raptor_path = os.path.join(
+            self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
+        )
+
+        if self.config.get('run_local'):
+            # raptor initiated locally, get and verify test from cmd line
+            self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
+            if 'raptor_extra_options' in self.config:
+                if '--test' in self.config['raptor_extra_options']:
+                    # --test specified, get test from cmd line and ensure is valid
+                    test_name_index = self.config['raptor_extra_options'].index('--test') + 1
+                    if test_name_index < len(self.config['raptor_extra_options']):
+                        self.test = self.config['raptor_extra_options'][test_name_index]
+                    else:
+                        self.fatal("Test name not provided")
+        else:
+            # raptor initiated in production via mozharness
+            self.test = self.config['test']
+
+    # Action methods. {{{1
+    # clobber defined in BaseScript
+    # read_buildbot_config defined in BuildbotMixin
+
+    def download_and_extract(self, extract_dirs=None, suite_categories=None):
+        return super(Raptor, self).download_and_extract(
+            suite_categories=['common', 'raptor']
+        )
+
+    def create_virtualenv(self, **kwargs):
+        """VirtualenvMixin.create_virtualenv() assuemes we're using
+        self.config['virtualenv_modules']. Since we are installing
+        raptor from its source, we have to wrap that method here."""
+        # if virtualenv already exists, just add to path and don't re-install, need it
+        # in path so can import jsonschema later when validating output for perfherder
+        _virtualenv_path = self.config.get("virtualenv_path")
+
+        if self.run_local and os.path.exists(_virtualenv_path):
+            self.info("Virtualenv already exists, skipping creation")
+            _python_interp = self.config.get('exes')['python']
+
+            if 'win' in self.platform_name():
+                _path = os.path.join(_virtualenv_path,
+                                     'Lib',
+                                     'site-packages')
+            else:
+                _path = os.path.join(_virtualenv_path,
+                                     'lib',
+                                     os.path.basename(_python_interp),
+                                     'site-packages')
+            sys.path.append(_path)
+            return
+
+        # virtualenv doesn't already exist so create it
+        # install mozbase first, so we use in-tree versions
+        if not self.run_local:
+            mozbase_requirements = os.path.join(
+                self.query_abs_dirs()['abs_test_install_dir'],
+                'config',
+                'mozbase_requirements.txt'
+            )
+        else:
+            mozbase_requirements = os.path.join(
+                os.path.dirname(self.raptor_path),
+                'config',
+                'mozbase_source_requirements.txt'
+            )
+        self.register_virtualenv_module(
+            requirements=[mozbase_requirements],
+            two_pass=True,
+            editable=True,
+        )
+        # require pip >= 1.5 so pip will prefer .whl files to install
+        super(Raptor, self).create_virtualenv(
+            modules=['pip>=1.5']
+        )
+        # raptor in harness requires what else is
+        # listed in raptor requirements.txt file.
+        self.install_module(
+            requirements=[os.path.join(self.raptor_path,
+                                       'requirements.txt')]
+        )
+
+    def _validate_treeherder_data(self, parser):
+        # late import is required, because install is done in create_virtualenv
+        import jsonschema
+
+        if len(parser.found_perf_data) != 1:
+            self.critical("PERFHERDER_DATA was seen %d times, expected 1."
+                          % len(parser.found_perf_data))
+            return
+
+        schema_path = os.path.join(external_tools_path,
+                                   'performance-artifact-schema.json')
+        self.info("Validating PERFHERDER_DATA against %s" % schema_path)
+        try:
+            with open(schema_path) as f:
+                schema = json.load(f)
+            data = json.loads(parser.found_perf_data[0])
+            jsonschema.validate(data, schema)
+        except:
+            self.exception("Error while validating PERFHERDER_DATA")
+
+    def _artifact_perf_data(self, dest):
+        src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'local.json')
+        try:
+            shutil.copyfile(src, dest)
+        except:
+            self.critical("Error copying results %s to upload dir %s" % (src, dest))
+
+    def run_tests(self, args=None, **kw):
+        """run raptor tests"""
+
+        # get raptor options
+        options = self.raptor_options(args=args, **kw)
+
+        # python version check
+        python = self.query_python_path()
+        self.run_command([python, "--version"])
+        parser = RaptorOutputParser(config=self.config, log_obj=self.log_obj,
+                                   error_list=RaptorErrorList)
+        env = {}
+        env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
+        if not self.run_local:
+            env['MINIDUMP_STACKWALK'] = self.query_minidump_stackwalk()
+        env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
+        env['RUST_BACKTRACE'] = 'full'
+        if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
+            self.mkdir_p(env['MOZ_UPLOAD_DIR'])
+        env = self.query_env(partial_env=env, log_level=INFO)
+        # adjust PYTHONPATH to be able to use raptor as a python package
+        if 'PYTHONPATH' in env:
+            env['PYTHONPATH'] = self.raptor_path + os.pathsep + env['PYTHONPATH']
+        else:
+            env['PYTHONPATH'] = self.raptor_path
+
+        # mitmproxy needs path to mozharness when installing the cert
+        env['SCRIPTSPATH'] = scripts_path
+
+        if self.repo_path is not None:
+            env['MOZ_DEVELOPER_REPO_DIR'] = self.repo_path
+        if self.obj_path is not None:
+            env['MOZ_DEVELOPER_OBJ_DIR'] = self.obj_path
+
+        # sets a timeout for how long raptor should run without output
+        output_timeout = self.config.get('raptor_output_timeout', 3600)
+        # run raptor tests
+        run_tests = os.path.join(self.raptor_path, 'raptor', 'raptor.py')
+
+        mozlog_opts = ['--log-tbpl-level=debug']
+        if not self.run_local and 'suite' in self.config:
+            fname_pattern = '%s_%%s.log' % self.config['test']
+            mozlog_opts.append('--log-errorsummary=%s'
+                               % os.path.join(env['MOZ_UPLOAD_DIR'],
+                                              fname_pattern % 'errorsummary'))
+            mozlog_opts.append('--log-raw=%s'
+                               % os.path.join(env['MOZ_UPLOAD_DIR'],
+                                              fname_pattern % 'raw'))
+
+        def launch_in_debug_mode(cmdline):
+            cmdline = set(cmdline)
+            debug_opts = {'--debug', '--debugger', '--debugger_args'}
+
+            return bool(debug_opts.intersection(cmdline))
+
+        command = [python, run_tests] + options + mozlog_opts
+        if launch_in_debug_mode(command):
+            raptor_process = subprocess.Popen(command, cwd=self.workdir, env=env)
+            raptor_process.wait()
+        else:
+            self.return_code = self.run_command(command, cwd=self.workdir,
+                                            output_timeout=output_timeout,
+                                            output_parser=parser,
+                                            env=env)
+        if parser.minidump_output:
+            self.info("Looking at the minidump files for debugging purposes...")
+            for item in parser.minidump_output:
+                self.run_command(["ls", "-l", item])
+
+        if self.return_code not in [0]:
+            # update the worst log level
+            log_level = ERROR
+            if self.return_code == 1:
+                log_level = WARNING
+            if self.return_code == 4:
+                log_level = WARNING
+
+        elif '--no-upload-results' not in options:
+            if not self.gecko_profile:
+                self._validate_treeherder_data(parser)
+                if not self.run_local:
+                    # copy results to upload dir so they are included as an artifact
+                    dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
+                    self._artifact_perf_data(dest)
+
+
+class RaptorOutputParser(OutputParser):
+    minidump_regex = re.compile(r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''')
+    RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
+
+    def __init__(self, **kwargs):
+        super(RaptorOutputParser, self).__init__(**kwargs)
+        self.minidump_output = None
+        self.found_perf_data = []
+
+    def parse_single_line(self, line):
+        m = self.minidump_regex.search(line)
+        if m:
+            self.minidump_output = (m.group(1), m.group(2), m.group(3))
+
+        m = self.RE_PERF_DATA.match(line)
+        if m:
+            self.found_perf_data.append(m.group(1))
+        super(RaptorOutputParser, self).parse_single_line(line)
+

testing/mozharness/scripts/raptor_script.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""raptor
+
+"""
+
+import os
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.mozilla.testing.raptor import Raptor
+
+if __name__ == '__main__':
+    raptor = Raptor()
+    raptor.run_and_exit()

testing/raptor/mach_commands.py

@@ -0,0 +1,118 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Originally taken from /talos/mach_commands.py
+
+# Integrates raptor mozharness with mach
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import sys
+import json
+import socket
+
+from mozbuild.base import MozbuildObject, MachCommandBase
+from mach.decorators import CommandProvider, Command
+
+HERE = os.path.dirname(os.path.realpath(__file__))
+
+
+class RaptorRunner(MozbuildObject):
+    def run_test(self, raptor_args):
+        """
+        We want to do couple of things before running raptor
+        1. Clone mozharness
+        2. Make config for raptor mozharness
+        3. Run mozharness
+        """
+
+        self.init_variables(raptor_args)
+        self.make_config()
+        self.write_config()
+        self.make_args()
+        return self.run_mozharness()
+
+    def init_variables(self, raptor_args):
+        self.raptor_dir = os.path.join(self.topsrcdir, 'testing', 'raptor')
+        self.mozharness_dir = os.path.join(self.topsrcdir, 'testing',
+                                           'mozharness')
+        self.config_file_path = os.path.join(self._topobjdir, 'testing',
+                                             'raptor-in_tree_conf.json')
+        self.binary_path = self.get_binary_path()
+        self.virtualenv_script = os.path.join(self.topsrcdir, 'third_party', 'python',
+                                              'virtualenv', 'virtualenv.py')
+        self.virtualenv_path = os.path.join(self._topobjdir, 'testing',
+                                            'raptor-venv')
+        self.python_interp = sys.executable
+        self.raptor_args = raptor_args
+
+    def make_config(self):
+        default_actions = ['populate-webroot', 'create-virtualenv', 'run-tests']
+        self.config = {
+            'run_local': True,
+            'binary_path': self.binary_path,
+            'repo_path': self.topsrcdir,
+            'raptor_path': self.raptor_dir,
+            'obj_path': self.topobjdir,
+            'log_name': 'raptor',
+            'virtualenv_path': self.virtualenv_path,
+            'pypi_url': 'http://pypi.python.org/simple',
+            'base_work_dir': self.mozharness_dir,
+            'exes': {
+                'python': self.python_interp,
+                'virtualenv': [self.python_interp, self.virtualenv_script],
+            },
+            'title': socket.gethostname(),
+            'default_actions': default_actions,
+            'raptor_extra_options': self.raptor_args,
+            'python3_manifest': {
+                'win32': 'python3.manifest',
+                'win64': 'python3_x64.manifest',
+            }
+        }
+
+    def make_args(self):
+        self.args = {
+            'config': {},
+            'initial_config_file': self.config_file_path,
+        }
+
+    def write_config(self):
+        try:
+            config_file = open(self.config_file_path, 'wb')
+            config_file.write(json.dumps(self.config))
+            config_file.close()
+        except IOError as e:
+            err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
+            print(err_str.format(self.config_file_path, str(e)))
+            raise e
+
+    def run_mozharness(self):
+        sys.path.insert(0, self.mozharness_dir)
+        from mozharness.mozilla.testing.raptor import Raptor
+        raptor_mh = Raptor(config=self.args['config'],
+                           initial_config_file=self.args['initial_config_file'])
+        return raptor_mh.run()
+
+
+def create_parser():
+    sys.path.insert(0, HERE)  # allow to import the raptor package
+    from raptor.cmdline import create_parser
+    return create_parser(mach_interface=True)
+
+
+@CommandProvider
+class MachRaptor(MachCommandBase):
+    @Command('raptor-test', category='testing',
+             description='Run raptor performance tests.',
+             parser=create_parser)
+    def run_raptor_test(self, **kwargs):
+        raptor = self._spawn(RaptorRunner)
+
+        try:
+            return raptor.run_test(sys.argv[2:])
+        except Exception as e:
+            print(str(e))
+            return 1

testing/raptor/raptor/cmdline.py

@@ -13,13 +13,17 @@ def create_parser(mach_interface=False):
     parser = argparse.ArgumentParser()
     add_arg = parser.add_argument
 
-    add_arg('-t', '--test', default=None, dest="test",
+    if not mach_interface:
+        add_arg('--app', default='firefox', dest='app',
+                help="name of the application we are testing (default: firefox)",
+                choices=['firefox', 'chrome'])
+        add_arg('-b', '--binary', required=True, dest='binary',
+                help="path to the browser executable that we are testing")
+
+    # remaining arg is test name
+    add_arg("test",
+            nargs="*",
             help="name of raptor test to run")
-    add_arg('--app', default='firefox', dest='app',
-            help="name of the application we are testing (default: firefox)",
-            choices=['firefox', 'chrome'])
-    add_arg('-b', '--binary', required=True,
-            help="path to the browser executable that we are testing")
 
     add_logging_group(parser)
     return parser

testing/raptor/raptor/control_server.py

@@ -2,13 +2,14 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# simple local server on port 8000, to demonstrate
-# receiving hero element timing results from a web extension
+# control server for raptor performance framework
+# communicates with the raptor browser webextension
 from __future__ import absolute_import
 
 import BaseHTTPServer
 import json
 import os
+import socket
 import threading
 
 from mozlog import get_proxy_logger
@@ -69,11 +70,18 @@ class RaptorControlServer():
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.server = None
         self._server_thread = None
+        self.port = None
 
     def start(self):
         config_dir = os.path.join(here, 'tests')
         os.chdir(config_dir)
-        server_address = ('', 8000)
+
+        # pick a free port
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.bind(('', 0))
+        self.port = sock.getsockname()[1]
+        sock.close()
+        server_address = ('', self.port)
 
         server_class = BaseHTTPServer.HTTPServer
         handler_class = MyHandler
@@ -83,7 +91,7 @@ class RaptorControlServer():
         self._server_thread = threading.Thread(target=httpd.serve_forever)
         self._server_thread.setDaemon(True)  # don't hang on exit
         self._server_thread.start()
-        LOG.info("raptor control server running on port 8000...")
+        LOG.info("raptor control server running on port %d..." % self.port)
         self.server = httpd
 
     def stop(self):

testing/raptor/raptor/gen_test_config.py

@@ -13,15 +13,15 @@ webext_dir = os.path.join(os.path.dirname(here), 'webext', 'raptor')
 LOG = get_proxy_logger(component="gen_test_url")
 
 
-def gen_test_config(browser, test):
+def gen_test_config(browser, test, cs_port):
     LOG.info("writing test settings url background js, so webext can get it")
 
     data = """// this file is auto-generated by raptor, do not edit directly
 function getTestConfig() {
-    return {"browser": "%s", "test_settings_url": "http://localhost:8000/%s.json"};
+    return {"browser": "%s", "test_settings_url": "http://localhost:%d/%s.json"};
 }
 
-""" % (browser, test)
+""" % (browser, cs_port, test)
 
     webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
 

testing/raptor/raptor/manifest.py

@@ -66,7 +66,8 @@ def get_raptor_test_list(args):
     # get a list of available raptor tests, for the browser we're testing on
     available_tests = get_browser_test_list(args.app)
     tests_to_run = []
-
+    # currently only support one test name on cmd line
+    args.test = args.test[0]
     # if test name not provided on command line, run all available raptor tests for this browser;
     # if test name provided on command line, make sure it exists, and then only include that one
     if args.test is not None:

testing/raptor/raptor/raptor.py

@@ -16,15 +16,17 @@ from mozlog import commandline, get_default_logger
 from mozprofile import create_profile
 from mozrunner import runners
 
-from raptor.cmdline import parse_args
-from raptor.control_server import RaptorControlServer
-from raptor.gen_test_config import gen_test_config
-from raptor.outputhandler import OutputHandler
-from raptor.playback import get_playback
-from raptor.manifest import get_raptor_test_list
-
+# need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext')
+sys.path.insert(0, here)
+
+from cmdline import parse_args
+from control_server import RaptorControlServer
+from gen_test_config import gen_test_config
+from outputhandler import OutputHandler
+from playback import get_playback
+from manifest import get_raptor_test_list
 
 
 class Raptor(object):
@@ -79,7 +81,7 @@ class Raptor(object):
 
     def run_test(self, test, timeout=None):
         self.log.info("starting raptor test: %s" % test['name'])
-        gen_test_config(self.config['app'], test['name'])
+        gen_test_config(self.config['app'], test['name'], self.control_server.port)
 
         self.profile.addons.install(os.path.join(webext_dir, 'raptor'))
 

testing/raptor/raptor/tests/raptor-chrome-tp7.ini

@@ -1,22 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor tp7 chrome
-
-[DEFAULT]
-apps = chrome
-type =  pageload
-playback = mitmproxy
-release_bin_mac = mitmproxy-2.0.2-osx.tar.gz
-page_cycles = 25
-
-[raptor-chrome-tp7]
-test_url = http://localhost:8081/heroes
-measure =
-  fcp
-  hero
-hero =
-  mugshot
-  title
-  anime

testing/raptor/raptor/tests/raptor-firefox-tp6.ini

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# raptor tp6 firefox
+# raptor tp6 on firefox
 
 [DEFAULT]
 apps = firefox

testing/raptor/raptor/tests/raptor-speedometer.ini

@@ -1,14 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor speedometer
-
-[raptor-speedometer]
-apps =
-  firefox
-  chrome
-type = benchmark
-test_url = http://localhost:8081/Speedometer/index.html?raptor
-page_cycles = 1
-page_timeout = 120000

testing/raptor/test/test_raptor.py

@@ -10,7 +10,6 @@ import pytest
 from mozprofile import BaseProfile
 from mozrunner.errors import RunnerNotStartedError
 
-from raptor.control_server import RaptorControlServer
 from raptor.raptor import Raptor
 
 
@@ -36,12 +35,16 @@ def test_create_profile(options, app, get_prefs):
 
 
 def test_start_and_stop_server(raptor):
+    print("*RW* control server is now:")
+    print(str(raptor.control_server))
     assert raptor.control_server is None
 
     raptor.start_control_server()
-    assert isinstance(raptor.control_server, RaptorControlServer)
 
     assert raptor.control_server._server_thread.is_alive()
+    assert raptor.control_server.port is not None
+    assert raptor.control_server.server is not None
+
     raptor.clean_up()
     assert not raptor.control_server._server_thread.is_alive()
 

testing/web-platform/meta/MANIFEST.json

@@ -394477,7 +394477,9 @@
    "webdriver/tests/close_window/user_prompts.py": [
     [
      "/webdriver/tests/close_window/user_prompts.py",
-     {}
+     {
+      "timeout": "long"
+     }
     ]
    ],
    "webdriver/tests/contexts/json_serialize_windowproxy.py": [

testing/web-platform/meta/css/cssom/shorthand-values.html.ini

@@ -26,9 +26,6 @@
   [The serialization of border: solid; border-style: dotted should be canonical.]
     expected: FAIL
 
-  [The serialization of outline-width: 2px; outline-style: dotted; outline-color: blue; should be canonical.]
-    expected: FAIL
-
   [The serialization of list-style-type: circle; list-style-position: inside; list-style-image: initial; should be canonical.]
     expected: FAIL
 

testing/web-platform/meta/webdriver/tests/cookies/delete_cookie.py.ini

@@ -1,9 +1,3 @@
 [delete_cookie.py]
-  disabled:
-    if debug and stylo and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86") and (bits == 32): wpt-sync Bug None
   [test_handle_prompt_accept]
     expected: FAIL
-
-  [test_unknown_cookie]
-    expected: FAIL
-

testing/web-platform/meta/webdriver/tests/element_click/bubbling.py.ini

@@ -1,3 +0,0 @@
-[bubbling.py]
-  disabled:
-    if not debug and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86") and (bits == 32): wpt-sync Bug 1439945

testing/web-platform/meta/webdriver/tests/execute_script/cyclic.py.ini

@@ -1,3 +0,0 @@
-[cyclic.py]
-  disabled:
-    if not debug and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86") and (bits == 32): wpt-sync Bug 1439951

testing/web-platform/tests/webdriver/tests/close_window/user_prompts.py

@@ -1,3 +1,5 @@
+# META: timeout=long
+
 from tests.support.asserts import assert_error, assert_dialog_handled
 from tests.support.fixtures import create_dialog, create_window
 from tests.support.inline import inline

testing/web-platform/tests/webdriver/tests/cookies/delete_cookie.py

@@ -1,23 +1,19 @@
-from tests.support.asserts import assert_error, assert_dialog_handled, assert_success
+from tests.support.asserts import assert_dialog_handled, assert_error, assert_success
 from tests.support.fixtures import create_dialog
 from tests.support.inline import inline
 
 
 def delete_cookie(session, name):
-    return session.transport.send("DELETE", "/session/%s/cookie/%s" % (session.session_id, name))
-
-
-# 16.4 Delete Cookie
+    return session.transport.send(
+        "DELETE", "/session/{session_id}/cookie/{name}".format(
+            session_id=session.session_id,
+            name=name))
 
 
 def test_no_browsing_context(session, create_window):
-    """
-    1. If the current top-level browsing context is no longer open,
-    return error with error code no such window.
-
-    """
     session.window_handle = create_window()
     session.close()
+
     response = delete_cookie(session, "foo")
     assert_error(response, "no such window")
 
@@ -35,70 +31,30 @@ def test_handle_prompt_ignore():
 
 
 def test_handle_prompt_accept(new_session, add_browser_capabilites):
-    """
-    2. Handle any user prompts and return its value if it is an error.
-
-    [...]
-
-    In order to handle any user prompts a remote end must take the
-    following steps:
-
-      [...]
-
-      2. Perform the following substeps based on the current session's
-      user prompt handler:
-
-        [...]
-
-        - accept state
-           Accept the current user prompt.
-
-    """
     _, session = new_session({"capabilities": {"alwaysMatch": add_browser_capabilites({"unhandledPromptBehavior": "accept"})}})
     session.url = inline("<title>WD doc title</title>")
 
     create_dialog(session)("alert", text="dismiss #1", result_var="dismiss1")
     response = delete_cookie(session, "foo")
-    assert response.status == 200
+    assert_success(response)
     assert_dialog_handled(session, "dismiss #1")
 
     create_dialog(session)("confirm", text="dismiss #2", result_var="dismiss2")
     response = delete_cookie(session, "foo")
-    assert response.status == 200
+    assert_success(response)
     assert_dialog_handled(session, "dismiss #2")
 
     create_dialog(session)("prompt", text="dismiss #3", result_var="dismiss3")
     response = delete_cookie(session, "foo")
-    assert response.status == 200
+    assert_success(response)
     assert_dialog_handled(session, "dismiss #3")
 
 
 def test_handle_prompt_missing_value(session, create_dialog):
-    """
-    2. Handle any user prompts and return its value if it is an error.
-
-    [...]
-
-    In order to handle any user prompts a remote end must take the
-    following steps:
-
-      [...]
-
-      2. Perform the following substeps based on the current session's
-      user prompt handler:
-
-        [...]
-
-        - missing value default state
-           1. Dismiss the current user prompt.
-           2. Return error with error code unexpected alert open.
-
-    """
     session.url = inline("<title>WD doc title</title>")
     create_dialog("alert", text="dismiss #1", result_var="dismiss1")
 
     response = delete_cookie(session, "foo")
-
     assert_error(response, "unexpected alert open")
     assert_dialog_handled(session, "dismiss #1")
 
@@ -117,6 +73,4 @@ def test_handle_prompt_missing_value(session, create_dialog):
 
 def test_unknown_cookie(session):
     response = delete_cookie(session, "stilton")
-    assert response.status == 200
-    assert "value" in response.body
-    assert response.body["value"] is None
+    assert_success(response)

testing/web-platform/tests/webdriver/tests/element_click/bubbling.py

@@ -1,6 +1,7 @@
 from tests.support.asserts import assert_success
 from tests.support.inline import inline
 
+
 def click(session, element):
     return session.transport.send(
         "POST", "/session/{session_id}/element/{element_id}/click".format(
@@ -135,7 +136,8 @@ def test_element_disappears_during_click(session):
 
         function logEvent({type, target, currentTarget}) {
           log.innerHTML += "<p></p>";
-          log.lastElementChild.textContent = `${type} in ${target.id} (handled by ${currentTarget.id})`;
+          log.lastElementChild.textContent =
+              `${type} in ${target.id} (handled by ${currentTarget.id})`;
         }
 
         for (let ev of ["click", "mousedown", "mouseup"]) {
@@ -144,7 +146,9 @@ def test_element_disappears_during_click(session):
           body.addEventListener(ev, logEvent);
         }
 
-        over.addEventListener("mousedown", () => over.style.display = "none");
+        over.addEventListener("mousedown", function(mousedownEvent) {
+          over.style.display = "none";
+        });
         </script>
         """)
     over = session.find.css("#over", all=False)

toolkit/components/extensions/Extension.jsm

@@ -506,6 +506,10 @@ class ExtensionData {
     };
   }
 
+  canUseExperiment(manifest) {
+    return this.experimentsAllowed && manifest.experiment_apis;
+  }
+
   async parseManifest() {
     let [manifest] = await Promise.all([
       this.readJSON("manifest.json"),
@@ -645,7 +649,7 @@ class ExtensionData {
         return manager.initModuleJSON([modules]);
       };
 
-      if (manifest.experiment_apis) {
+      if (this.canUseExperiment(manifest)) {
         let parentModules = {};
         let childModules = {};
 
@@ -1404,6 +1408,8 @@ class Extension extends ExtensionData {
 
   get isPrivileged() {
     return (this.addonData.signedState === AddonManager.SIGNEDSTATE_PRIVILEGED ||
+            this.addonData.signedState === AddonManager.SIGNEDSTATE_SYSTEM ||
+            this.addonData.builtIn ||
             (AppConstants.MOZ_ALLOW_LEGACY_EXTENSIONS &&
              this.addonData.temporarilyInstalled));
   }

toolkit/components/extensions/ExtensionTestCommon.jsm

@@ -373,12 +373,19 @@ var ExtensionTestCommon = class ExtensionTestCommon {
       id = uuidGen.generateUUID().number;
     }
 
+    let signedState = AddonManager.SIGNEDSTATE_SIGNED;
+    if (data.isPrivileged) {
+      signedState = AddonManager.SIGNEDSTATE_PRIVILEGED;
+    }
+    if (data.isSystem) {
+      signedState = AddonManager.SIGNEDSTATE_SYSTEM;
+    }
+
     return new Extension({
       id,
       resourceURI: jarURI,
       cleanupFile: file,
-      signedState: data.isPrivileged ? AddonManager.SIGNEDSTATE_PRIVILEGED
-                                     : AddonManager.SIGNEDSTATE_SIGNED,
+      signedState,
       temporarilyInstalled: !!data.temporarilyInstalled,
     });
   }

toolkit/components/extensions/test/xpcshell/test_ext_experiments.js

@@ -107,36 +107,61 @@ async function testFooExperiment() {
                         "await foo.parent()");
 }
 
+async function testFooFailExperiment() {
+  browser.test.assertEq("object", typeof browser.experiments,
+                        "typeof browser.experiments");
+
+  browser.test.assertEq("undefined", typeof browser.experiments.foo,
+                        "typeof browser.experiments.foo");
+}
+
 add_task(async function test_bundled_experiments() {
-  async function background() {
-    await testFooExperiment();
+  let testCases = [
+    {isSystem: true, temporarilyInstalled: true, shouldHaveExperiments: true},
+    {isSystem: true, temporarilyInstalled: false, shouldHaveExperiments: true},
+    {isPrivileged: true, temporarilyInstalled: true, shouldHaveExperiments: true},
+    {isPrivileged: true, temporarilyInstalled: false, shouldHaveExperiments: true},
+    {isPrivileged: false, temporarilyInstalled: true, shouldHaveExperiments: true},
+    {isPrivileged: false, temporarilyInstalled: false, shouldHaveExperiments: false},
+  ];
+
+  async function background(shouldHaveExperiments) {
+    if (shouldHaveExperiments) {
+      await testFooExperiment();
+    } else {
+      await testFooFailExperiment();
+    }
 
     browser.test.notifyPass("background.experiments.foo");
   }
 
-  let extension = ExtensionTestUtils.loadExtension({
-    isPrivileged: true,
+  for (let testCase of testCases) {
+    let extension = ExtensionTestUtils.loadExtension({
+      isPrivileged: testCase.isPrivileged,
+      isSystem: testCase.isSystem,
+      temporarilyInstalled: testCase.temporarilyInstalled,
 
-    manifest: {
-      experiment_apis: fooExperimentAPIs,
-    },
+      manifest: {
+        experiment_apis: fooExperimentAPIs,
+      },
 
-    background: `
-      ${testFooExperiment}
-      (${background})();
-    `,
+      background: `
+        ${testFooExperiment}
+        ${testFooFailExperiment}
+        (${background})(${testCase.shouldHaveExperiments});
+      `,
 
-    files: fooExperimentFiles,
-  });
+      files: fooExperimentFiles,
+    });
 
-  await extension.startup();
+    await extension.startup();
 
-  await extension.awaitFinish("background.experiments.foo");
+    await extension.awaitFinish("background.experiments.foo");
 
-  await extension.unload();
+    await extension.unload();
+  }
 });
 
-
 add_task(async function test_unbundled_experiments() {
   async function background() {
     await testFooExperiment();

toolkit/components/extensions/test/xpcshell/test_ext_schemas_interactive.js

@@ -91,6 +91,7 @@ function setHandlingUserInput(extension) {
 // proxied api implementations.
 add_task(async function test_proxy() {
   let extension = ExtensionTestUtils.loadExtension({
+    isPrivileged: true,
     background() {
       browser.test.onMessage.addListener(async () => {
         try {
@@ -128,6 +129,7 @@ add_task(async function test_proxy() {
 // non-proxied api implementations.
 add_task(async function test_local() {
   let extension = ExtensionTestUtils.loadExtension({
+    isPrivileged: true,
     background() {
       browser.test.onMessage.addListener(async () => {
         try {

toolkit/components/normandy/test/browser/browser_Normandy.js

@@ -152,6 +152,11 @@ decorate_task(
       [[{"test.rollout-pref": 1}]],
       "finishInit should record original values of the study prefs",
     );
+
+    // cleanup
+    defaultBranch.deleteBranch(experimentPref1);
+    defaultBranch.deleteBranch(experimentPref2);
+    defaultBranch.deleteBranch(experimentPref3);
   },
 );
 

toolkit/components/places/PlacesDBUtils.jsm

@@ -696,16 +696,23 @@ var PlacesDBUtils = {
       // S.3 set missing added and last modified dates.
       { query:
         `UPDATE moz_bookmarks
-        SET dateAdded = COALESCE(dateAdded, lastModified, (
+        SET dateAdded = COALESCE(NULLIF(dateAdded, 0), NULLIF(lastModified, 0), NULLIF((
               SELECT MIN(visit_date) FROM moz_historyvisits
               WHERE place_id = fk
-            ), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000),
-            lastModified = COALESCE(lastModified, dateAdded, (
+            ), 0), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000),
+            lastModified = COALESCE(NULLIF(lastModified, 0), NULLIF(dateAdded, 0), NULLIF((
               SELECT MAX(visit_date) FROM moz_historyvisits
               WHERE place_id = fk
-            ), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000)
-        WHERE dateAdded IS NULL OR
-              lastModified IS NULL`,
+            ), 0), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000)
+        WHERE NULLIF(dateAdded, 0) IS NULL OR
+              NULLIF(lastModified, 0) IS NULL`,
+      },
+
+      // S.4 reset added dates that are ahead of last modified dates.
+      { query:
+        `UPDATE moz_bookmarks
+         SET dateAdded = lastModified
+         WHERE dateAdded > lastModified`,
       },
     ];
 

toolkit/components/places/SyncedBookmarksMirror.jsm

@@ -1287,7 +1287,10 @@ class SyncedBookmarksMirror {
    */
   async updateLocalItemsInPlaces(mergedRoot, localDeletions, remoteDeletions) {
     MirrorLog.trace("Setting up merge states table");
-    let mergeStatesParams = Array.from(mergedRoot.mergeStatesParams());
+    let mergeStatesParams = [];
+    for await (let param of yieldingIterator(mergedRoot.mergeStatesParams())) {
+      mergeStatesParams.push(param);
+    }
     if (mergeStatesParams.length) {
       await this.db.execute(`
         INSERT INTO mergeStates(localGuid, mergedGuid, parentGuid, level,
@@ -1699,7 +1702,7 @@ class SyncedBookmarksMirror {
       SELECT parentId, guid FROM structureToUpload
       ORDER BY parentId, position`);
 
-    for (let row of childGuidRows) {
+    for await (let row of yieldingIterator(childGuidRows)) {
       let localParentId = row.getResultByName("parentId");
       let childRecordId = PlacesSyncUtils.bookmarks.guidToRecordId(
         row.getResultByName("guid"));
@@ -3054,15 +3057,6 @@ class BookmarkNode {
     return this.age < otherNode.age;
   }
 
-  * descendants() {
-    for (let node of this.children) {
-      yield node;
-      if (node.isFolder()) {
-        yield* node.descendants();
-      }
-    }
-  }
-
   /**
    * Checks if remoteNode has a kind that's compatible with this *local* node.
    * - Nodes with the same kind are always compatible.
@@ -3497,7 +3491,7 @@ class BookmarkMerger {
   }
 
   async subsumes(tree) {
-    for await (let guid of Async.yieldingIterator(tree.syncableGuids())) {
+    for await (let guid of yieldingIterator(tree.syncableGuids())) {
       if (!this.mentions(guid)) {
         return false;
       }

toolkit/components/places/tests/unit/test_preventive_maintenance.js

@@ -1485,12 +1485,16 @@ tests.push({
 
   async setup() {
     let placeIdWithVisits = addPlace();
+    let placeIdWithZeroVisit = addPlace();
     this._placeVisits.push({
       placeId: placeIdWithVisits,
       visitDate: PlacesUtils.toPRTime(new Date(2017, 9, 4)),
     }, {
       placeId: placeIdWithVisits,
       visitDate: PlacesUtils.toPRTime(new Date(2017, 9, 8)),
+    }, {
+      placeId: placeIdWithZeroVisit,
+      visitDate: 0,
     });
 
     this._bookmarksWithDates.push({
@@ -1523,33 +1527,47 @@ tests.push({
       parentId: bs.unfiledBookmarksFolder,
       dateAdded: PlacesUtils.toPRTime(new Date(2017, 9, 3)),
       lastModified: PlacesUtils.toPRTime(new Date(2017, 9, 6)),
+    }, {
+      guid: "bookmarkFFFF",
+      placeId: placeIdWithZeroVisit,
+      parentId: bs.unfiledBookmarksFolder,
+      dateAdded: 0,
+      lastModified: 0,
     });
 
     await PlacesUtils.withConnectionWrapper(
-      "Insert bookmarks and visits with dates",
+      "S.3: Insert bookmarks and visits",
       db => db.executeTransaction(async () => {
-        await db.executeCached(`
+        await db.execute(`
           INSERT INTO moz_historyvisits(place_id, visit_date)
           VALUES(:placeId, :visitDate)`,
           this._placeVisits);
 
-        await db.executeCached(`
+        await db.execute(`
           INSERT INTO moz_bookmarks(fk, type, parent, guid, dateAdded,
                                     lastModified)
           VALUES(:placeId, 1, :parentId, :guid, :dateAdded,
                  :lastModified)`,
           this._bookmarksWithDates);
+
+        await db.execute(`
+          UPDATE moz_bookmarks SET
+            dateAdded = 0,
+            lastModified = NULL
+          WHERE id = :toolbarFolderId`,
+          { toolbarFolderId: bs.toolbarFolder });
       })
     );
   },
 
   async check() {
     let db = await PlacesUtils.promiseDBConnection();
-    let updatedRows = await db.executeCached(`
+    let updatedRows = await db.execute(`
       SELECT guid, dateAdded, lastModified
       FROM moz_bookmarks
-      WHERE guid IN (?, ?, ?, ?, ?)`,
-      this._bookmarksWithDates.map(info => info.guid));
+      WHERE guid = :guid`,
+      [{ guid: bs.toolbarGuid },
+       ...this._bookmarksWithDates.map(({ guid }) => ({ guid }))]);
 
     for (let row of updatedRows) {
       let guid = row.getResultByName("guid");
@@ -1577,10 +1595,14 @@ tests.push({
           break;
         }
 
-        // Neither date added nor last modified exists, and no visits, so we
-        // should fall back to the current time for both.
-        case "bookmarkCCCC": {
+        // C has no visits, date added, or last modified time, F has zeros for
+        // all, and the toolbar has a zero date added and no last modified time.
+        // In all cases, we should fall back to the current time.
+        case "bookmarkCCCC":
+        case "bookmarkFFFF":
+        case bs.toolbarGuid: {
           let nowAsPRTime = PlacesUtils.toPRTime(new Date());
+          Assert.greater(dateAdded, 0);
           Assert.equal(dateAdded, lastModified);
           Assert.ok(dateAdded <= nowAsPRTime);
           break;
@@ -1605,6 +1627,63 @@ tests.push({
           Assert.equal(lastModified, expectedInfo.lastModified);
           break;
         }
+
+        default:
+          throw new Error(`Unexpected row for bookmark ${guid}`);
+      }
+    }
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "S.4",
+  desc: "reset added dates that are ahead of last modified dates",
+  _bookmarksWithDates: [],
+
+  async setup() {
+    this._bookmarksWithDates.push({
+      guid: "bookmarkGGGG",
+      parentId: bs.unfiledBookmarksFolder,
+      dateAdded: PlacesUtils.toPRTime(new Date(2017, 9, 6)),
+      lastModified: PlacesUtils.toPRTime(new Date(2017, 9, 3)),
+    });
+
+    await PlacesUtils.withConnectionWrapper(
+      "S.4: Insert bookmarks and visits",
+      db => db.executeTransaction(async () => {
+        await db.execute(`
+          INSERT INTO moz_bookmarks(type, parent, guid, dateAdded,
+                                    lastModified)
+          VALUES(1, :parentId, :guid, :dateAdded, :lastModified)`,
+          this._bookmarksWithDates);
+      })
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let updatedRows = await db.execute(`
+      SELECT guid, dateAdded, lastModified
+      FROM moz_bookmarks
+      WHERE guid = :guid`,
+      this._bookmarksWithDates.map(({ guid }) => ({ guid })));
+
+    for (let row of updatedRows) {
+      let guid = row.getResultByName("guid");
+      let dateAdded = row.getResultByName("dateAdded");
+      let lastModified = row.getResultByName("lastModified");
+      switch (guid) {
+        case "bookmarkGGGG": {
+          let expectedInfo = this._bookmarksWithDates[0];
+          Assert.equal(dateAdded, expectedInfo.lastModified);
+          Assert.equal(lastModified, expectedInfo.lastModified);
+          break;
+        }
+
+        default:
+          throw new Error(`Unexpected row for bookmark ${guid}`);
       }
     }
   },

toolkit/components/telemetry/docs/collection/events.rst

@@ -138,6 +138,11 @@ Record a registered event.
 Throws if the combination of ``category``, ``method`` and ``object`` is unknown.
 Recording an expired event will not throw, but print a warning into the browser console.
 
+.. note::
+
+  Each ``recordEvent`` of a known non-expired combination of ``category``, ``method``, and
+  ``object``, will be :ref:`summarized <events.event-summary>`.
+
 .. warning::
 
   Event Telemetry recording is designed to be cheap, not free. If you wish to record events in a performance-sensitive piece of code, store the events locally and record them only after the performance-sensitive piece ("hot path") has completed.
@@ -173,6 +178,11 @@ Example:
   Services.telemetry.setEventRecordingEnabled("ui", false);
   // ... now "ui" events will not be recorded anymore.
 
+.. note::
+
+  Even if your event category isn't enabled, counts of events that attempted to be recorded will
+  be :ref:`summarized <events.event-summary>`.
+
 ``registerEvents()``
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -223,6 +233,48 @@ Internal API
 
 These functions are only supposed to be used by Telemetry internally or in tests.
 
+.. _events.event-summary:
+
+Event Summary
+=============
+
+Calling ``recordEvent`` on any non-expired registered event will accumulate to a
+:doc:`Scalar <scalars>` for ease of analysing uptake and usage patterns. Even if the event category
+isn't enabled.
+
+The scalar is ``telemetry.event_counts`` for statically-registered events (the ones in
+``Events.yaml``) and ``telemetry.dynamic_event_counts`` for dynamically-registered events (the ones
+registered via ``registerEvents``). These are :ref:`keyed scalars <scalars.keyed-scalars>` where
+the keys are of the form ``category#method#object`` and the values are counts of the number of
+times ``recordEvent`` was called with that combination of ``category``, ``method``, and ``object``.
+
+These two scalars have a default maximum key limit of 500 per process. This limit is configurable
+via the ``toolkit.telemetry.maxEventSummaryKeys`` preference.
+
+Example:
+
+.. code-block:: js
+
+  // telemetry.event_counts summarizes in the same process the events were recorded
+
+  // Let us suppose in the parent process this happens:
+  Services.telemetry.recordEvent("interaction", "click", "document", "xuldoc");
+  Services.telemetry.recordEvent("interaction", "click", "document", "xuldoc-neighbour");
+
+  // And in each of child proccesses 1 through 4, this happens:
+  Services.telemetry.recordEvent("interaction", "click", "document", "htmldoc");
+
+In the case that ``interaction.click.document`` is statically-registered, this will result in the
+parent-process scalar ``telemetry.event_counts`` having a key ``interaction#click#document`` with
+value ``2`` and the content-process scalar ``telemetry.event_counts`` having a key
+``interaction#click#document`` with the value ``4``.
+
+All dynamically-registered events end up in the dynamic-process ``telemetry.dynamic_event_counts``
+(notice the different name) regardless of in which process the events were recorded. From the
+example above, if ``interaction.click.document`` was registered with ``registerEvents`` then
+the dynamic-process scalar ``telemetry.dynamic_event_counts`` would have a key
+``interaction#click#document`` with the value ``6``.
+
 Version History
 ===============
 
@@ -234,4 +286,7 @@ Version History
 
    - Ignore re-registering existing events for a category instead of failing (`bug 1408975 <https://bugzilla.mozilla.org/show_bug.cgi?id=1408975>`_).
    - Removed support for the ``expiry_date`` property, as it was unused (`bug 1414638 <https://bugzilla.mozilla.org/show_bug.cgi?id=1414638>`_).
-- Firefox 61: Enabled support for adding events in artifact builds and build-faster workflows (`bug 1448945 <https://bugzilla.mozilla.org/show_bug.cgi?id=1448945>`_).
+- Firefox 61:
+
+   - Enabled support for adding events in artifact builds and build-faster workflows (`bug 1448945 <https://bugzilla.mozilla.org/show_bug.cgi?id=1448945>`_).
+   - Added summarization of events (`bug 1440673 <https://bugzilla.mozilla.org/show_bug.cgi?id=1440673>`_).

toolkit/components/telemetry/docs/collection/scalars.rst

@@ -172,6 +172,8 @@ String type restrictions
 To prevent abuses, the content of a string scalar is limited to 50 characters in length. Trying
 to set a longer string will result in an error and no string being set.
 
+.. _scalars.keyed-scalars:
+
 Keyed Scalars
 -------------
 Keyed scalars are collections of one of the available scalar types, indexed by a string key that can contain UTF8 characters and cannot be longer than 72 characters. Keyed scalars can contain up to 100 keys. This scalar type is for example useful when you want to break down certain counts by a name, like how often searches happen with which search engine.

toolkit/components/telemetry/docs/internals/preferences.rst

@@ -137,6 +137,11 @@ Preferences
 
   Enable the :doc:`../data/update-ping` on browser updates.
 
+``toolkit.telemetry.maxEventSummaryKeys``
+
+  Set the maximum number of keys per process of the :ref:`Event Summary <events.event-summary>`
+  :ref:`keyed scalars <scalars.keyed-scalars>`. Default is 500. Change requires restart.
+
 Data-choices notification
 -------------------------
 

toolkit/content/aboutTelemetry.js

@@ -4,6 +4,7 @@
 
 "use strict";
 
+ChromeUtils.import("resource://gre/modules/BrowserUtils.jsm");
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://gre/modules/TelemetryTimestamps.jsm");
 ChromeUtils.import("resource://gre/modules/TelemetryController.jsm");
@@ -219,20 +220,20 @@ var Settings = {
     let uploadEnabled = this.getStatusStringForSetting(this.SETTINGS[0]);
     let extendedEnabled = Services.telemetry.canRecordExtended;
     let collectedData = bundle.GetStringFromName(extendedEnabled ? "prereleaseData" : "releaseData");
+    let explanation = bundle.GetStringFromName("settingsExplanation");
 
-    let parameters = [
-      collectedData,
-      this.convertStringToLink(uploadEnabled),
-    ];
-    let explanation = bundle.formatStringFromName("settingsExplanation", parameters, 2);
+    let fragment = BrowserUtils.getLocalizedFragment(document, explanation, collectedData, this.convertStringToLink(uploadEnabled));
+    settingsExplanation.appendChild(fragment);
 
-    // eslint-disable-next-line no-unsanitized/property
-    settingsExplanation.innerHTML = explanation;
     this.attachObservers();
   },
 
   convertStringToLink(string) {
-    return "<a href=\"#\" class=\"change-data-choices-link\">" + string + "</a>";
+    let link = document.createElement("a");
+    link.setAttribute("href", "#");
+    link.setAttribute("class", "change-data-choices-link");
+    link.textContent = string;
+    return link;
   },
 };
 
@@ -302,14 +303,18 @@ var PingPicker = {
 
   render() {
     let pings = bundle.GetStringFromName("pingExplanationLink");
-    let pingLink = "<a href=\"https://firefox-source-docs.mozilla.org/toolkit/components/telemetry/telemetry/concepts/pings.html\">" + pings + "</a>";
+    let pingLink = document.createElement("a");
+    pingLink.setAttribute("href", "https://firefox-source-docs.mozilla.org/toolkit/components/telemetry/telemetry/concepts/pings.html");
+    pingLink.textContent = pings;
     let pingName = this._getSelectedPingName();
+    let pingNameSpan = document.createElement("span");
+    pingNameSpan.setAttribute("class", "change-ping");
 
     // Display the type and controls if the ping is not current
     let pingDate = document.getElementById("ping-date");
     let pingType = document.getElementById("ping-type");
     let controls = document.getElementById("controls");
-    let explanation;
+    let fragment;
     if (!this.viewCurrentPingData) {
       // Change sidebar heading text.
       pingDate.textContent = pingName;
@@ -320,23 +325,22 @@ var PingPicker = {
 
       // Change home page text.
       pingName = bundle.formatStringFromName("namedPing", [pingName, pingTypeText], 2);
-      let pingNameHtml = "<span class=\"change-ping\">" + pingName + "</span>";
-      let parameters = [pingLink, pingNameHtml, pingTypeText];
-      explanation = bundle.formatStringFromName("pingDetails", parameters, 3);
+      pingNameSpan.textContent = pingName;
+      let explanation = bundle.GetStringFromName("pingDetails");
+      fragment = BrowserUtils.getLocalizedFragment(document, explanation, pingLink, pingNameSpan, pingTypeText);
     } else {
       // Change sidebar heading text.
       controls.classList.add("hidden");
       pingType.textContent = bundle.GetStringFromName("currentPingSidebar");
 
       // Change home page text.
-      let pingNameHtml = "<span class=\"change-ping\">" + pingName + "</span>";
-      explanation = bundle.formatStringFromName("pingDetailsCurrent", [pingLink, pingNameHtml], 2);
+      pingNameSpan.textContent = pingName;
+      let explanation = bundle.GetStringFromName("pingDetailsCurrent");
+      fragment = BrowserUtils.getLocalizedFragment(document, explanation, pingLink, pingNameSpan);
     }
 
     let pingExplanation = document.getElementById("ping-explanation");
-
-    // eslint-disable-next-line no-unsanitized/property
-    pingExplanation.innerHTML = explanation;
+    pingExplanation.appendChild(fragment);
     pingExplanation.querySelector(".change-ping").addEventListener("click", (ev) => {
       document.getElementById("ping-picker").classList.remove("hidden");
       ev.stopPropagation();

toolkit/content/widgets/videocontrols.xml

@@ -1656,6 +1656,7 @@
 
         ttBtn.classList.add("textTrackItem");
         ttBtn.setAttribute("index", tt.index);
+        ttBtn.setAttribute("dir", "auto");
         ttBtn.appendChild(ttText);
 
         this.textTrackList.appendChild(ttBtn);

toolkit/modules/E10SUtils.jsm

@@ -237,11 +237,15 @@ var E10SUtils = {
         this.getRemoteTypeForURIObject(aURI, true, remoteType, webNav.currentURI);
     }
 
-    if (sessionHistory.count == 1 && webNav.currentURI.spec == "about:newtab") {
+    if (!aHasPostData &&
+        Services.appinfo.remoteType == WEB_REMOTE_TYPE &&
+        sessionHistory.count == 1 &&
+        webNav.currentURI.spec == "about:newtab") {
       // This is possibly a preloaded browser and we're about to navigate away for
       // the first time. On the child side there is no way to tell for sure if that
       // is the case, so let's redirect this request to the parent to decide if a new
-      // process is needed.
+      // process is needed. But we don't currently properly handle POST data in
+      // redirects (bug 1457520), so if there is POST data, don't return false here.
       return false;
     }
 

toolkit/mozapps/extensions/Blocklist.jsm

@@ -509,9 +509,7 @@ var Blocklist = {
 
   _createBlocklistURL(id) {
     let url = Services.urlFormatter.formatURLPref(PREF_BLOCKLIST_ITEM_URL);
-    url = url.replace(/%blockID%/g, id);
-
-    return url;
+    return url.replace(/%blockID%/g, id);
   },
 
   notify(aTimer) {
@@ -1262,38 +1260,19 @@ var Blocklist = {
     return Ci.nsIBlocklistService.STATE_SOFTBLOCKED;
   },
 
-  /* See nsIBlocklistService */
-  getPluginBlocklistURL(plugin) {
-    if (!this.isLoaded)
-      this._loadBlocklist();
+  async getPluginBlockURL(plugin) {
+    await this.loadBlocklistAsync();
 
     let r = this._getPluginBlocklistEntry(plugin, this._pluginEntries);
     if (!r) {
       return null;
     }
-    let {entry: blockEntry} = r;
+    let blockEntry = r.entry;
     if (!blockEntry.blockID) {
       return null;
     }
 
-    return this._createBlocklistURL(blockEntry.blockID);
-  },
-
-  /* See nsIBlocklistService */
-  getPluginInfoURL(plugin) {
-    if (!this.isLoaded)
-      this._loadBlocklist();
-
-    let r = this._getPluginBlocklistEntry(plugin, this._pluginEntries);
-    if (!r) {
-      return null;
-    }
-    let {entry: blockEntry} = r;
-    if (!blockEntry.blockID) {
-      return null;
-    }
-
-    return blockEntry.infoURL;
+    return blockEntry.infoURL || this._createBlocklistURL(blockEntry.blockID);
   },
 
   _notifyObserversBlocklistGFX() {
@@ -1423,7 +1402,7 @@ var Blocklist = {
             disable: false,
             blocked: state == Ci.nsIBlocklistService.STATE_BLOCKED,
             item: plugin,
-            url: this.getPluginBlocklistURL(plugin),
+            url: await this.getPluginBlockURL(plugin),
           });
         }
       }

toolkit/mozapps/extensions/content/extensions.js

@@ -2841,8 +2841,10 @@ var gDetailView = {
         );
         var errorLink = document.getElementById("detail-error-link");
         errorLink.value = gStrings.ext.GetStringFromName("details.notification.blocked.link");
-        errorLink.href = this._addon.blocklistURL;
-        errorLink.hidden = false;
+        this._addon.getBlocklistURL().then(url => {
+          errorLink.href = url;
+          errorLink.hidden = false;
+        });
       } else if (isDisabledUnsigned(this._addon)) {
         this.node.setAttribute("notification", "error");
         document.getElementById("detail-error").textContent = gStrings.ext.formatStringFromName(
@@ -2877,8 +2879,10 @@ var gDetailView = {
         );
         let warningLink = document.getElementById("detail-warning-link");
         warningLink.value = gStrings.ext.GetStringFromName("details.notification.softblocked.link");
-        warningLink.href = this._addon.blocklistURL;
-        warningLink.hidden = false;
+        this._addon.getBlocklistURL().then(url => {
+          warningLink.href = url;
+          warningLink.hidden = false;
+        });
       } else if (this._addon.blocklistState == Ci.nsIBlocklistService.STATE_OUTDATED) {
         this.node.setAttribute("notification", "warning");
         document.getElementById("detail-warning").textContent = gStrings.ext.formatStringFromName(
@@ -2887,8 +2891,10 @@ var gDetailView = {
         );
         let warningLink = document.getElementById("detail-warning-link");
         warningLink.value = gStrings.ext.GetStringFromName("details.notification.outdated.link");
-        warningLink.href = this._addon.blocklistURL;
-        warningLink.hidden = false;
+        this._addon.getBlocklistURL().then(url => {
+          warningLink.href = url;
+          warningLink.hidden = false;
+        });
       } else if (this._addon.blocklistState == Ci.nsIBlocklistService.STATE_VULNERABLE_UPDATE_AVAILABLE) {
         this.node.setAttribute("notification", "error");
         document.getElementById("detail-error").textContent = gStrings.ext.formatStringFromName(
@@ -2897,8 +2903,10 @@ var gDetailView = {
         );
         let errorLink = document.getElementById("detail-error-link");
         errorLink.value = gStrings.ext.GetStringFromName("details.notification.vulnerableUpdatable.link");
-        errorLink.href = this._addon.blocklistURL;
-        errorLink.hidden = false;
+        this._addon.getBlocklistURL().then(url => {
+          errorLink.href = url;
+          errorLink.hidden = false;
+        });
       } else if (this._addon.blocklistState == Ci.nsIBlocklistService.STATE_VULNERABLE_NO_UPDATE) {
         this.node.setAttribute("notification", "error");
         document.getElementById("detail-error").textContent = gStrings.ext.formatStringFromName(
@@ -2907,8 +2915,10 @@ var gDetailView = {
         );
         let errorLink = document.getElementById("detail-error-link");
         errorLink.value = gStrings.ext.GetStringFromName("details.notification.vulnerableNoUpdate.link");
-        errorLink.href = this._addon.blocklistURL;
-        errorLink.hidden = false;
+        this._addon.getBlocklistURL().then(url => {
+          errorLink.href = url;
+          errorLink.hidden = false;
+        });
       } else if (this._addon.isGMPlugin && !this._addon.isInstalled &&
                  this._addon.isActive) {
         this.node.setAttribute("notification", "warning");