From 3bd56e31c6d103b814828e38ead061bd06d65780 Mon Sep 17 00:00:00 2001
From: "Kearwood \"Kip\" Gilbert" <kgilbert@mozilla.com>
Date: Wed, 22 Nov 2017 15:57:23 -0800
Subject: [PATCH 01/46] Bug 1381645 - Restrict access to WebVR to HTTPS only
 sites r=daoshengmu,smaug

MozReview-Commit-ID: BsqUzSH55nN

--HG--
extra : rebase_source : 054eca4ee6fb28c32534aa46d925beb49670899b
---
 dom/webidl/Navigator.webidl | 4 ++--
 dom/webidl/VRDisplay.webidl | 7 +++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/dom/webidl/Navigator.webidl b/dom/webidl/Navigator.webidl
index d9af6d027115..46b97771c1dc 100644
--- a/dom/webidl/Navigator.webidl
+++ b/dom/webidl/Navigator.webidl
@@ -217,10 +217,10 @@ partial interface Navigator {
 };
 
 partial interface Navigator {
-  [Throws, Pref="dom.vr.enabled"]
+  [SecureContext, Throws, Pref="dom.vr.enabled"]
   Promise<sequence<VRDisplay>> getVRDisplays();
   // TODO: Use FrozenArray once available. (Bug 1236777)
-  [Frozen, Cached, Pure, Pref="dom.vr.enabled"]
+  [SecureContext, Frozen, Cached, Pure, Pref="dom.vr.enabled"]
   readonly attribute sequence<VRDisplay> activeVRDisplays;
   [ChromeOnly, Pref="dom.vr.enabled"]
   readonly attribute boolean isWebVRContentDetected;
diff --git a/dom/webidl/VRDisplay.webidl b/dom/webidl/VRDisplay.webidl
index 43bcd3e00ebc..c75fb9a6de5c 100644
--- a/dom/webidl/VRDisplay.webidl
+++ b/dom/webidl/VRDisplay.webidl
@@ -9,6 +9,7 @@ enum VREye {
 };
 
 [Pref="dom.vr.enabled",
+ SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRFieldOfView {
   readonly attribute double upDegrees;
@@ -49,6 +50,7 @@ dictionary VRLayer {
  * These are expected to be static per-device/per-user.
  */
 [Pref="dom.vr.enabled",
+ SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRDisplayCapabilities {
   /**
@@ -90,6 +92,7 @@ interface VRDisplayCapabilities {
  * that support room-scale experiences.
  */
 [Pref="dom.vr.enabled",
+ SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRStageParameters {
   /**
@@ -115,6 +118,7 @@ interface VRStageParameters {
 };
 
 [Pref="dom.vr.enabled",
+ SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRPose
 {
@@ -136,6 +140,7 @@ interface VRPose
 
 [Constructor,
  Pref="dom.vr.enabled",
+ SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRFrameData {
   readonly attribute DOMHighResTimeStamp timestamp;
@@ -158,6 +163,7 @@ interface VRSubmitFrameResult {
 };
 
 [Pref="dom.vr.enabled",
+ SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VREyeParameters {
   /**
@@ -181,6 +187,7 @@ interface VREyeParameters {
 };
 
 [Pref="dom.vr.enabled",
+ SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRDisplay : EventTarget {
   /**

From 6613a4b93e55b207a984f580ed9d073070392b5b Mon Sep 17 00:00:00 2001
From: "Kearwood \"Kip\" Gilbert" <kgilbert@mozilla.com>
Date: Fri, 15 Dec 2017 10:22:11 -0600
Subject: [PATCH 02/46] Bug 1381645 - Part 2: Move VR tests to secure context
 r=daoshengmu

MozReview-Commit-ID: CTDnXHBuaLU

--HG--
extra : rebase_source : a5be58d0cf10c2b4d7cbda95174793439af3fc5b
---
 dom/vr/test/mochitest/mochitest.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dom/vr/test/mochitest/mochitest.ini b/dom/vr/test/mochitest/mochitest.ini
index c5b82e719e6d..cdd6ee4e02a9 100644
--- a/dom/vr/test/mochitest/mochitest.ini
+++ b/dom/vr/test/mochitest/mochitest.ini
@@ -1,5 +1,6 @@
 # Please confirm there is no other VR display connected. Otherwise, VRPuppetDisplay can't be attached.
 [DEFAULT]
+scheme = https
 support-files =
   VRSimulationDriver.js
   requestPresent.js

From 6ec807f80ddc37ad5c038ed6e5b7115ee1d6d175 Mon Sep 17 00:00:00 2001
From: "Kearwood \"Kip\" Gilbert" <kgilbert@mozilla.com>
Date: Mon, 18 Dec 2017 13:09:52 -0800
Subject: [PATCH 03/46] Bug 1381645 - Part 3: Update test_interfaces.js to
 expect WebVR interfaces to be enabled only in secure contexts r=bz

MozReview-Commit-ID: 3jyxWRi8BIv

--HG--
extra : rebase_source : 87f00b098a52207749ca0400a53d904520a0eb0a
---
 dom/tests/mochitest/general/test_interfaces.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/dom/tests/mochitest/general/test_interfaces.js b/dom/tests/mochitest/general/test_interfaces.js
index f5fa0a3b9f36..6fa4c17816af 100644
--- a/dom/tests/mochitest/general/test_interfaces.js
+++ b/dom/tests/mochitest/general/test_interfaces.js
@@ -1181,21 +1181,21 @@ var interfaceNamesInGlobalScope =
 // IMPORTANT: Do not change this list without review from a DOM peer!
     {name: "VideoStreamTrack", insecureContext: true},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRDisplay", insecureContext: true, releaseNonWindows: false},
+    {name: "VRDisplay", releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRDisplayCapabilities", insecureContext: true, releaseNonWindows: false},
+    {name: "VRDisplayCapabilities", releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRDisplayEvent", insecureContext: true, releaseNonWindows: false},
+    {name: "VRDisplayEvent", releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VREyeParameters", insecureContext: true, releaseNonWindows: false},
+    {name: "VREyeParameters", releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRFieldOfView", insecureContext: true, releaseNonWindows: false},
+    {name: "VRFieldOfView", releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRFrameData", insecureContext: true, releaseNonWindows: false},
+    {name: "VRFrameData", releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRPose", insecureContext: true, releaseNonWindows: false},
+    {name: "VRPose", releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRStageParameters", insecureContext: true, releaseNonWindows: false},
+    {name: "VRStageParameters", releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
     {name: "VTTCue", insecureContext: true},
 // IMPORTANT: Do not change this list without review from a DOM peer!

From c6289726c77732c6310fa01568581c213d51936c Mon Sep 17 00:00:00 2001
From: "Kearwood \"Kip\" Gilbert" <kgilbert@mozilla.com>
Date: Wed, 24 Jan 2018 14:18:47 -0800
Subject: [PATCH 04/46] Bug 1381645 - Part 4: Update web-platform tests
 expectation data for WebVR r=jgraham

The testing/web-platform/tests/webvr/idlharness.html web-platform test is
reporting failures due to the webvr interfaces being available only on secure
contexts.

A PR has been added to the upstream web-platform-tests repo in GitHub to
make webvr/idlharness.html run in a secure context.  Once this change has
landed and been pulled down, we can update the expectation data to expect
these tests to pass once again.

MozReview-Commit-ID: Gar9MHUFEHr

--HG--
extra : rebase_source : 2776b52c02abaa2522f992739d5c8febbe368b5d
---
 .../meta/webvr/idlharness.html.ini            | 204 ++++++++++++++++++
 1 file changed, 204 insertions(+)

diff --git a/testing/web-platform/meta/webvr/idlharness.html.ini b/testing/web-platform/meta/webvr/idlharness.html.ini
index f784e49ab6da..926c1fcf9500 100644
--- a/testing/web-platform/meta/webvr/idlharness.html.ini
+++ b/testing/web-platform/meta/webvr/idlharness.html.ini
@@ -11,3 +11,207 @@
   [HTMLIFrameElement interface: attribute allowvr]
     expected: FAIL
 
+  [Navigator interface: operation getVRDisplays()]
+    expected: FAIL
+
+  [Navigator interface: attribute activeVRDisplays]
+    expected: FAIL
+
+  [VRDisplay interface: existence and properties of interface object]
+    expected: FAIL
+
+  [VRDisplay interface object length]
+    expected: FAIL
+
+  [VRDisplay interface object name]
+    expected: FAIL
+
+  [VRDisplay interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [VRDisplay interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [VRDisplay interface: attribute isPresenting]
+    expected: FAIL
+
+  [VRDisplay interface: attribute capabilities]
+    expected: FAIL
+
+  [VRDisplay interface: attribute stageParameters]
+    expected: FAIL
+
+  [VRDisplay interface: operation getEyeParameters(VREye)]
+    expected: FAIL
+
+  [VRDisplay interface: attribute displayId]
+    expected: FAIL
+
+  [VRDisplay interface: attribute displayName]
+    expected: FAIL
+
+  [VRDisplay interface: operation getFrameData(VRFrameData)]
+    expected: FAIL
+
+  [VRDisplay interface: attribute depthNear]
+    expected: FAIL
+
+  [VRDisplay interface: attribute depthFar]
+    expected: FAIL
+
+  [VRDisplay interface: operation requestAnimationFrame(FrameRequestCallback)]
+    expected: FAIL
+
+  [VRDisplay interface: operation cancelAnimationFrame(long)]
+    expected: FAIL
+
+  [VRDisplay interface: operation requestPresent([object Object\])]
+    expected: FAIL
+
+  [VRDisplay interface: operation exitPresent()]
+    expected: FAIL
+
+  [VRDisplay interface: operation getLayers()]
+    expected: FAIL
+
+  [VRDisplay interface: operation submitFrame()]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface: existence and properties of interface object]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface object length]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface object name]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface: attribute hasPosition]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface: attribute hasExternalDisplay]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface: attribute canPresent]
+    expected: FAIL
+
+  [VRDisplayCapabilities interface: attribute maxLayers]
+    expected: FAIL
+
+  [VRPose interface: existence and properties of interface object]
+    expected: FAIL
+
+  [VRPose interface object length]
+    expected: FAIL
+
+  [VRPose interface object name]
+    expected: FAIL
+
+  [VRPose interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [VRPose interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [VRPose interface: attribute position]
+    expected: FAIL
+
+  [VRPose interface: attribute linearVelocity]
+    expected: FAIL
+
+  [VRPose interface: attribute linearAcceleration]
+    expected: FAIL
+
+  [VRPose interface: attribute orientation]
+    expected: FAIL
+
+  [VRPose interface: attribute angularVelocity]
+    expected: FAIL
+
+  [VRPose interface: attribute angularAcceleration]
+    expected: FAIL
+
+  [VRFrameData interface: existence and properties of interface object]
+    expected: FAIL
+
+  [VRFrameData interface object length]
+    expected: FAIL
+
+  [VRFrameData interface object name]
+    expected: FAIL
+
+  [VRFrameData interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [VRFrameData interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [VRFrameData interface: attribute leftProjectionMatrix]
+    expected: FAIL
+
+  [VRFrameData interface: attribute leftViewMatrix]
+    expected: FAIL
+
+  [VRFrameData interface: attribute rightProjectionMatrix]
+    expected: FAIL
+
+  [VRFrameData interface: attribute rightViewMatrix]
+    expected: FAIL
+
+  [VRFrameData interface: attribute pose]
+    expected: FAIL
+
+  [VREyeParameters interface: existence and properties of interface object]
+    expected: FAIL
+
+  [VREyeParameters interface object length]
+    expected: FAIL
+
+  [VREyeParameters interface object name]
+    expected: FAIL
+
+  [VREyeParameters interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [VREyeParameters interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [VREyeParameters interface: attribute offset]
+    expected: FAIL
+
+  [VREyeParameters interface: attribute renderWidth]
+    expected: FAIL
+
+  [VREyeParameters interface: attribute renderHeight]
+    expected: FAIL
+
+  [VRStageParameters interface: existence and properties of interface object]
+    expected: FAIL
+
+  [VRStageParameters interface object length]
+    expected: FAIL
+
+  [VRStageParameters interface object name]
+    expected: FAIL
+
+  [VRStageParameters interface: existence and properties of interface prototype object]
+    expected: FAIL
+
+  [VRStageParameters interface: existence and properties of interface prototype object's "constructor" property]
+    expected: FAIL
+
+  [VRStageParameters interface: attribute sittingToStandingTransform]
+    expected: FAIL
+
+  [VRStageParameters interface: attribute sizeX]
+    expected: FAIL
+
+  [VRStageParameters interface: attribute sizeZ]
+    expected: FAIL
+

From 6321345295810d4d9c1bd8de0de6d87d722d54b8 Mon Sep 17 00:00:00 2001
From: Thom Chiovoloni <tchiovoloni@mozilla.com>
Date: Tue, 9 Jan 2018 19:22:33 -0500
Subject: [PATCH 05/46] Bug 1429250 - Avoid repeated traversals of toFetch in
 _processIncoming. r=eoger,kitcambridge

MozReview-Commit-ID: N1Zr8iT7da

--HG--
extra : rebase_source : ddaa51704c93985219d15b31aec7ff7e19b2832f
---
 services/sync/modules-testing/rotaryengine.js |  4 +-
 services/sync/modules/bookmark_repair.js      |  8 ++-
 services/sync/modules/engines.js              | 43 +++++++++-----
 services/sync/modules/util.js                 | 52 ++++++++++++++++-
 .../sync/tests/unit/test_bookmark_engine.js   |  2 +-
 .../sync/tests/unit/test_history_engine.js    | 13 +++--
 services/sync/tests/unit/test_syncengine.js   | 56 +++++++++++--------
 .../sync/tests/unit/test_syncengine_sync.js   | 56 ++++++++++---------
 services/sync/tests/unit/test_telemetry.js    |  2 +-
 tools/lint/eslint/modules.json                |  2 +-
 10 files changed, 158 insertions(+), 80 deletions(-)

diff --git a/services/sync/modules-testing/rotaryengine.js b/services/sync/modules-testing/rotaryengine.js
index 9faf4243bd97..77fd60a74ca7 100644
--- a/services/sync/modules-testing/rotaryengine.js
+++ b/services/sync/modules-testing/rotaryengine.js
@@ -100,8 +100,8 @@ RotaryTracker.prototype = {
 this.RotaryEngine = function RotaryEngine(service) {
   SyncEngine.call(this, "Rotary", service);
   // Ensure that the engine starts with a clean slate.
-  this.toFetch        = [];
-  this.previousFailed = [];
+  this.toFetch        = new SerializableSet();
+  this.previousFailed = new SerializableSet();
 };
 RotaryEngine.prototype = {
   __proto__: SyncEngine.prototype,
diff --git a/services/sync/modules/bookmark_repair.js b/services/sync/modules/bookmark_repair.js
index 0d79d99e09b7..7a03315abf47 100644
--- a/services/sync/modules/bookmark_repair.js
+++ b/services/sync/modules/bookmark_repair.js
@@ -194,9 +194,11 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
     for (let id of validationInfo.problems.serverMissing) {
       engine.addForWeakUpload(id);
     }
-    let toFetch = engine.toFetch.concat(validationInfo.problems.clientMissing,
-                                        validationInfo.problems.serverDeleted);
-    engine.toFetch = Array.from(new Set(toFetch));
+    engine.toFetch = Utils.setAddAll(
+      Utils.setAddAll(engine.toFetch,
+                      validationInfo.problems.clientMissing),
+      validationInfo.problems.serverDeleted
+    );
     return true;
   }
 
diff --git a/services/sync/modules/engines.js b/services/sync/modules/engines.js
index 447f07b1ce3c..390f04955ab1 100644
--- a/services/sync/modules/engines.js
+++ b/services/sync/modules/engines.js
@@ -842,6 +842,9 @@ SyncEngine.prototype = {
     if (!json.ids) {
       json.ids = [];
     }
+    // The set serializes the same way as an array, but offers more efficient
+    // methods of manipulation.
+    json.ids = new SerializableSet(json.ids);
     return json;
   },
 
@@ -911,6 +914,9 @@ SyncEngine.prototype = {
   },
 
   set toFetch(ids) {
+    if (ids.constructor.name != "SerializableSet") {
+      throw new Error("Bug: Attempted to set toFetch to something that isn't a SerializableSet");
+    }
     this._toFetchStorage.data = { ids };
     this._toFetchStorage.saveSoon();
   },
@@ -919,7 +925,12 @@ SyncEngine.prototype = {
     this._previousFailedStorage.ensureDataReady();
     return this._previousFailedStorage.data.ids;
   },
+
   set previousFailed(ids) {
+    if (ids.constructor.name != "SerializableSet") {
+      throw new Error(
+        "Bug: Attempted to set previousFailed to something that isn't a SerializableSet");
+    }
     this._previousFailedStorage.data = { ids };
     this._previousFailedStorage.saveSoon();
   },
@@ -1100,7 +1111,7 @@ SyncEngine.prototype = {
     // reconciled => number of items that were reconciled.
     let count = {applied: 0, failed: 0, newFailed: 0, reconciled: 0};
     let recordsToApply = [];
-    let failedInCurrentSync = [];
+    let failedInCurrentSync = new SerializableSet();
 
     let oldestModified = this.lastModified;
     let downloadedIDs = new Set();
@@ -1124,7 +1135,7 @@ SyncEngine.prototype = {
 
         let { shouldApply, error } = await this._maybeReconcile(record);
         if (error) {
-          failedInCurrentSync.push(record.id);
+          failedInCurrentSync.add(record.id);
           count.failed++;
           continue;
         }
@@ -1136,7 +1147,7 @@ SyncEngine.prototype = {
       }
 
       let failedToApply = await this._applyRecords(recordsToApply);
-      failedInCurrentSync.push(...failedToApply);
+      Utils.setAddAll(failedInCurrentSync, failedToApply);
 
       // `applied` is a bit of a misnomer: it counts records that *should* be
       // applied, so it also includes records that we tried to apply and failed.
@@ -1165,7 +1176,7 @@ SyncEngine.prototype = {
       // that in case the Sync server doesn't support `older` (bug 1316110).
       let remainingIDs = guids.obj.filter(id => !downloadedIDs.has(id));
       if (remainingIDs.length > 0) {
-        this.toFetch = Utils.arrayUnion(this.toFetch, remainingIDs);
+        this.toFetch = Utils.setAddAll(this.toFetch, remainingIDs);
       }
     }
 
@@ -1180,8 +1191,9 @@ SyncEngine.prototype = {
     // download limit, to prevent a large backlog for one engine from blocking
     // the others. We'll keep processing the backlog on subsequent engine syncs.
     let failedInPreviousSync = this.previousFailed;
-    let idsToBackfill = Utils.arrayUnion(this.toFetch.slice(0, downloadLimit),
-      failedInPreviousSync);
+    let idsToBackfill = Array.from(
+      Utils.setAddAll(Utils.subsetOfSize(this.toFetch, downloadLimit),
+                      failedInPreviousSync));
 
     // Note that we intentionally overwrite the previously failed list here.
     // Records that fail to decrypt or apply in two consecutive syncs are likely
@@ -1230,20 +1242,21 @@ SyncEngine.prototype = {
       count.failed += failedToApply.length;
       count.applied += backfilledRecordsToApply.length;
 
-      this.toFetch = Utils.arraySub(this.toFetch, ids);
-      this.previousFailed = Utils.arrayUnion(this.previousFailed, failedInBackfill);
+      this.toFetch = Utils.setDeleteAll(this.toFetch, ids);
+      this.previousFailed = Utils.setAddAll(this.previousFailed, failedInBackfill);
 
       if (this.lastSync < this.lastModified) {
         this.lastSync = this.lastModified;
       }
     }
 
-    count.newFailed = this.previousFailed.reduce((count, engine) => {
-      if (failedInPreviousSync.indexOf(engine) == -1) {
-        count++;
+    count.newFailed = 0;
+    for (let item of this.previousFailed) {
+      if (!failedInPreviousSync.has(item)) {
+        ++count.newFailed;
       }
-      return count;
-    }, 0);
+    }
+
     count.succeeded = Math.max(0, count.applied - count.failed);
     this._log.info(["Records:",
                     count.applied, "applied,",
@@ -1811,8 +1824,8 @@ SyncEngine.prototype = {
 
   async _resetClient() {
     this.resetLastSync();
-    this.previousFailed = [];
-    this.toFetch = [];
+    this.previousFailed = new SerializableSet();
+    this.toFetch = new SerializableSet();
     this._needWeakUpload.clear();
   },
 
diff --git a/services/sync/modules/util.js b/services/sync/modules/util.js
index 920991aee69e..304da82e1e31 100644
--- a/services/sync/modules/util.js
+++ b/services/sync/modules/util.js
@@ -2,7 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-this.EXPORTED_SYMBOLS = ["Utils", "Svc"];
+this.EXPORTED_SYMBOLS = ["Utils", "Svc", "SerializableSet"];
 
 var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
 
@@ -524,6 +524,47 @@ this.Utils = {
     return foo.concat(Utils.arraySub(bar, foo));
   },
 
+  /**
+   * Add all the items in `items` to the provided Set in-place.
+   *
+   * @return The provided set.
+   */
+  setAddAll(set, items) {
+    for (let item of items) {
+      set.add(item);
+    }
+    return set;
+  },
+
+  /**
+   * Delete every items in `items` to the provided Set in-place.
+   *
+   * @return The provided set.
+   */
+  setDeleteAll(set, items) {
+    for (let item of items) {
+      set.delete(item);
+    }
+    return set;
+  },
+
+  /**
+   * Take the first `size` items from the Set `items`.
+   *
+   * @return A Set of size at most `size`
+   */
+  subsetOfSize(items, size) {
+    let result = new Set();
+    let count = 0;
+    for (let item of items) {
+      if (count++ == size) {
+        return result;
+      }
+      result.add(item);
+    }
+    return result;
+  },
+
   bind2: function Async_bind2(object, method) {
     return function innerBind() { return method.apply(object, arguments); };
   },
@@ -703,6 +744,15 @@ this.Utils = {
   }
 };
 
+/**
+ * A subclass of Set that serializes as an Array when passed to JSON.stringify.
+ */
+class SerializableSet extends Set {
+  toJSON() {
+    return Array.from(this);
+  }
+}
+
 XPCOMUtils.defineLazyGetter(Utils, "_utf8Converter", function() {
   let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
                     .createInstance(Ci.nsIScriptableUnicodeConverter);
diff --git a/services/sync/tests/unit/test_bookmark_engine.js b/services/sync/tests/unit/test_bookmark_engine.js
index ef9f5ed559ab..8621a8bfeefa 100644
--- a/services/sync/tests/unit/test_bookmark_engine.js
+++ b/services/sync/tests/unit/test_bookmark_engine.js
@@ -170,7 +170,7 @@ add_task(async function test_processIncoming_error_orderChildren() {
     // Make the 10 minutes old so it will only be synced in the toFetch phase.
     bogus_record.modified = Date.now() / 1000 - 60 * 10;
     engine.lastSync = Date.now() / 1000 - 60;
-    engine.toFetch = [BOGUS_GUID];
+    engine.toFetch = new SerializableSet([BOGUS_GUID]);
 
     let error;
     try {
diff --git a/services/sync/tests/unit/test_history_engine.js b/services/sync/tests/unit/test_history_engine.js
index 67ebe516ecef..2547f0c656d1 100644
--- a/services/sync/tests/unit/test_history_engine.js
+++ b/services/sync/tests/unit/test_history_engine.js
@@ -69,7 +69,7 @@ add_task(async function test_history_download_limit() {
   let ping = await sync_engine_and_validate_telem(engine, false);
   deepEqual(ping.engines[0].incoming, { applied: 5 });
 
-  let backlogAfterFirstSync = engine.toFetch.slice(0);
+  let backlogAfterFirstSync = Array.from(engine.toFetch).sort();
   deepEqual(backlogAfterFirstSync, ["place0000000", "place0000001",
     "place0000002", "place0000003", "place0000004", "place0000005",
     "place0000006", "place0000007", "place0000008", "place0000009"]);
@@ -84,7 +84,7 @@ add_task(async function test_history_download_limit() {
   // After the second sync, our backlog still contains the same GUIDs: we
   // weren't able to make progress on fetching them, since our
   // `guidFetchBatchSize` is 0.
-  let backlogAfterSecondSync = engine.toFetch.slice(0);
+  let backlogAfterSecondSync = Array.from(engine.toFetch).sort();
   deepEqual(backlogAfterFirstSync, backlogAfterSecondSync);
 
   // Now add a newer record to the server.
@@ -105,7 +105,7 @@ add_task(async function test_history_download_limit() {
   deepEqual(ping.engines[0].incoming, { applied: 1 });
 
   // Our backlog should remain the same.
-  let backlogAfterThirdSync = engine.toFetch.slice(0);
+  let backlogAfterThirdSync = Array.from(engine.toFetch).sort();
   deepEqual(backlogAfterSecondSync, backlogAfterThirdSync);
 
   equal(engine.lastSync, lastSync + 20);
@@ -118,15 +118,16 @@ add_task(async function test_history_download_limit() {
   ping = await sync_engine_and_validate_telem(engine, false);
   deepEqual(ping.engines[0].incoming, { applied: 5 });
 
-  deepEqual(engine.toFetch, ["place0000005", "place0000006", "place0000007",
-    "place0000008", "place0000009"]);
+  deepEqual(
+    Array.from(engine.toFetch).sort(),
+    ["place0000005", "place0000006", "place0000007", "place0000008", "place0000009"]);
 
   // Sync again to clear out the backlog.
   engine.lastModified = collection.modified;
   ping = await sync_engine_and_validate_telem(engine, false);
   deepEqual(ping.engines[0].incoming, { applied: 5 });
 
-  deepEqual(engine.toFetch, []);
+  deepEqual(Array.from(engine.toFetch), []);
   await PlacesTestUtils.clearHistory();
 });
 
diff --git a/services/sync/tests/unit/test_syncengine.js b/services/sync/tests/unit/test_syncengine.js
index baf2665acb48..f92e5650180a 100644
--- a/services/sync/tests/unit/test_syncengine.js
+++ b/services/sync/tests/unit/test_syncengine.js
@@ -12,6 +12,16 @@ async function makeSteamEngine() {
   return engine;
 }
 
+function guidSetOfSize(length) {
+  return new SerializableSet(
+    Array.from({ length }, () => Utils.makeGUID()));
+}
+
+function assertSetsEqual(a, b) {
+  // Assert.deepEqual doesn't understand Set.
+  Assert.deepEqual(Array.from(a).sort(), Array.from(b).sort());
+}
+
 async function testSteamEngineStorage(test) {
   try {
     let setupEngine = await makeSteamEngine();
@@ -113,10 +123,10 @@ add_task(async function test_toFetch() {
   const filename = "weave/toFetch/steam.json";
 
   await testSteamEngineStorage({
-    toFetch: [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()],
+    toFetch: guidSetOfSize(3),
     setup(engine) {
       // Ensure pristine environment
-      Assert.equal(engine.toFetch.length, 0);
+      Assert.equal(engine.toFetch.size, 0);
 
       // Write file to disk
       engine.toFetch = this.toFetch;
@@ -124,13 +134,13 @@ add_task(async function test_toFetch() {
     },
     check(engine) {
       // toFetch is written asynchronously
-      Assert.deepEqual(engine.toFetch, this.toFetch);
+      assertSetsEqual(engine.toFetch, this.toFetch);
     },
   });
 
   await testSteamEngineStorage({
-    toFetch: [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()],
-    toFetch2: [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()],
+    toFetch: guidSetOfSize(4),
+    toFetch2: guidSetOfSize(5),
     setup(engine) {
       // Make sure it work for consecutive writes before the callback is executed.
       engine.toFetch = this.toFetch;
@@ -140,12 +150,12 @@ add_task(async function test_toFetch() {
       Assert.equal(engine.toFetch, this.toFetch2);
     },
     check(engine) {
-      Assert.deepEqual(engine.toFetch, this.toFetch2);
+      assertSetsEqual(engine.toFetch, this.toFetch2);
     },
   });
 
   await testSteamEngineStorage({
-    toFetch: [Utils.makeGUID(), Utils.makeGUID()],
+    toFetch: guidSetOfSize(2),
     async beforeCheck() {
       let toFetchPath = OS.Path.join(OS.Constants.Path.profileDir, filename);
       let bytes = new TextEncoder().encode(JSON.stringify(this.toFetch));
@@ -154,7 +164,7 @@ add_task(async function test_toFetch() {
     },
     check(engine) {
       // Read file from disk
-      Assert.deepEqual(engine.toFetch, this.toFetch);
+      assertSetsEqual(engine.toFetch, this.toFetch);
     },
   });
 });
@@ -165,10 +175,10 @@ add_task(async function test_previousFailed() {
   const filename = "weave/failed/steam.json";
 
   await testSteamEngineStorage({
-    previousFailed: [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()],
+    previousFailed: guidSetOfSize(3),
     setup(engine) {
       // Ensure pristine environment
-      Assert.equal(engine.previousFailed.length, 0);
+      Assert.equal(engine.previousFailed.size, 0);
 
       // Write file to disk
       engine.previousFailed = this.previousFailed;
@@ -176,13 +186,13 @@ add_task(async function test_previousFailed() {
     },
     check(engine) {
       // previousFailed is written asynchronously
-      Assert.deepEqual(engine.previousFailed, this.previousFailed);
+      assertSetsEqual(engine.previousFailed, this.previousFailed);
     },
   });
 
   await testSteamEngineStorage({
-    previousFailed: [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()],
-    previousFailed2: [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()],
+    previousFailed: guidSetOfSize(4),
+    previousFailed2: guidSetOfSize(5),
     setup(engine) {
       // Make sure it work for consecutive writes before the callback is executed.
       engine.previousFailed = this.previousFailed;
@@ -192,12 +202,12 @@ add_task(async function test_previousFailed() {
       Assert.equal(engine.previousFailed, this.previousFailed2);
     },
     check(engine) {
-      Assert.deepEqual(engine.previousFailed, this.previousFailed2);
+      assertSetsEqual(engine.previousFailed, this.previousFailed2);
     },
   });
 
   await testSteamEngineStorage({
-    previousFailed: [Utils.makeGUID(), Utils.makeGUID()],
+    previousFailed: guidSetOfSize(2),
     async beforeCheck() {
       let previousFailedPath = OS.Path.join(OS.Constants.Path.profileDir,
                                             filename);
@@ -207,7 +217,7 @@ add_task(async function test_previousFailed() {
     },
     check(engine) {
       // Read file from disk
-      Assert.deepEqual(engine.previousFailed, this.previousFailed);
+      assertSetsEqual(engine.previousFailed, this.previousFailed);
     },
   });
 });
@@ -220,18 +230,18 @@ add_task(async function test_resetClient() {
     // Ensure pristine environment
     Assert.equal(Svc.Prefs.get("steam.lastSync"), undefined);
     Assert.equal(Svc.Prefs.get("steam.lastSyncLocal"), undefined);
-    Assert.equal(engine.toFetch.length, 0);
+    Assert.equal(engine.toFetch.size, 0);
 
     engine.lastSync = 123.45;
     engine.lastSyncLocal = 67890;
-    engine.toFetch = [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()];
-    engine.previousFailed = [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()];
+    engine.toFetch = guidSetOfSize(4);
+    engine.previousFailed = guidSetOfSize(3);
 
     await engine.resetClient();
     Assert.equal(engine.lastSync, 0);
     Assert.equal(engine.lastSyncLocal, 0);
-    Assert.equal(engine.toFetch.length, 0);
-    Assert.equal(engine.previousFailed.length, 0);
+    Assert.equal(engine.toFetch.size, 0);
+    Assert.equal(engine.previousFailed.size, 0);
   } finally {
     Svc.Prefs.resetBranch("");
   }
@@ -252,13 +262,13 @@ add_task(async function test_wipeServer() {
   try {
     // Some data to reset.
     engine.lastSync = 123.45;
-    engine.toFetch = [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()];
+    engine.toFetch = guidSetOfSize(3),
 
     _("Wipe server data and reset client.");
     await engine.wipeServer();
     Assert.equal(steamCollection.payload, undefined);
     Assert.equal(engine.lastSync, 0);
-    Assert.equal(engine.toFetch.length, 0);
+    Assert.equal(engine.toFetch.size, 0);
 
   } finally {
     steamServer.stop(do_test_finished);
diff --git a/services/sync/tests/unit/test_syncengine_sync.js b/services/sync/tests/unit/test_syncengine_sync.js
index a30d8512e52f..3788f19bfbd2 100644
--- a/services/sync/tests/unit/test_syncengine_sync.js
+++ b/services/sync/tests/unit/test_syncengine_sync.js
@@ -627,8 +627,8 @@ add_task(async function test_processIncoming_resume_toFetch() {
   // Time travel 10 seconds into the future but still download the above WBOs.
   let engine = makeRotaryEngine();
   engine.lastSync = LASTSYNC;
-  engine.toFetch = ["flying", "scotsman"];
-  engine.previousFailed = ["failed0", "failed1", "failed2"];
+  engine.toFetch = new SerializableSet(["flying", "scotsman"]);
+  engine.previousFailed = new SerializableSet(["failed0", "failed1", "failed2"]);
 
   let server = sync_httpd_setup({
       "/1.1/foo/storage/rotary": collection.handler()
@@ -657,7 +657,7 @@ add_task(async function test_processIncoming_resume_toFetch() {
     Assert.equal(engine._store.items.failed0, "Record No. 0");
     Assert.equal(engine._store.items.failed1, "Record No. 1");
     Assert.equal(engine._store.items.failed2, "Record No. 2");
-    Assert.equal(engine.previousFailed.length, 0);
+    Assert.equal(engine.previousFailed.size, 0);
   } finally {
     await cleanAndGo(engine, server);
   }
@@ -703,8 +703,8 @@ add_task(async function test_processIncoming_notify_count() {
   try {
     // Confirm initial environment.
     Assert.equal(engine.lastSync, 0);
-    Assert.equal(engine.toFetch.length, 0);
-    Assert.equal(engine.previousFailed.length, 0);
+    Assert.equal(engine.toFetch.size, 0);
+    Assert.equal(engine.previousFailed.size, 0);
     do_check_empty(engine._store.items);
 
     let called = 0;
@@ -722,8 +722,8 @@ add_task(async function test_processIncoming_notify_count() {
 
     // Confirm failures.
     do_check_attribute_count(engine._store.items, 12);
-    Assert.deepEqual(engine.previousFailed, ["record-no-00", "record-no-05",
-      "record-no-10"]);
+    Assert.deepEqual(Array.from(engine.previousFailed).sort(),
+                     ["record-no-00", "record-no-05", "record-no-10"].sort());
 
     // There are newly failed records and they are reported.
     Assert.equal(called, 1);
@@ -737,7 +737,7 @@ add_task(async function test_processIncoming_notify_count() {
 
     // Confirming removed failures.
     do_check_attribute_count(engine._store.items, 14);
-    Assert.deepEqual(engine.previousFailed, ["record-no-00"]);
+    Assert.deepEqual(Array.from(engine.previousFailed), ["record-no-00"]);
 
     Assert.equal(called, 2);
     Assert.equal(counts.failed, 1);
@@ -792,12 +792,12 @@ add_task(async function test_processIncoming_previousFailed() {
   try {
     // Confirm initial environment.
     Assert.equal(engine.lastSync, 0);
-    Assert.equal(engine.toFetch.length, 0);
-    Assert.equal(engine.previousFailed.length, 0);
+    Assert.equal(engine.toFetch.size, 0);
+    Assert.equal(engine.previousFailed.size, 0);
     do_check_empty(engine._store.items);
 
     // Initial failed items in previousFailed to be reset.
-    let previousFailed = [Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()];
+    let previousFailed = new SerializableSet([Utils.makeGUID(), Utils.makeGUID(), Utils.makeGUID()]);
     engine.previousFailed = previousFailed;
     Assert.equal(engine.previousFailed, previousFailed);
 
@@ -807,9 +807,10 @@ add_task(async function test_processIncoming_previousFailed() {
 
     // Expected result: 4 sync batches with 2 failures each => 8 failures
     do_check_attribute_count(engine._store.items, 6);
-    Assert.deepEqual(engine.previousFailed, ["record-no-00", "record-no-01",
-      "record-no-04", "record-no-05", "record-no-08", "record-no-09",
-      "record-no-12", "record-no-13"]);
+    Assert.deepEqual(
+      Array.from(engine.previousFailed).sort(),
+      ["record-no-00", "record-no-01", "record-no-04", "record-no-05",
+       "record-no-08", "record-no-09", "record-no-12", "record-no-13"].sort());
 
     // Sync again with the same failed items (records 0, 1, 8, 9).
     await engine._processIncoming();
@@ -817,8 +818,9 @@ add_task(async function test_processIncoming_previousFailed() {
     // A second sync with the same failed items should not add the same items again.
     // Items that did not fail a second time should no longer be in previousFailed.
     do_check_attribute_count(engine._store.items, 10);
-    Assert.deepEqual(engine.previousFailed, ["record-no-00", "record-no-01",
-      "record-no-08", "record-no-09"]);
+    Assert.deepEqual(
+      Array.from(engine.previousFailed).sort(),
+      ["record-no-00", "record-no-01", "record-no-08", "record-no-09"].sort());
 
     // Refetched items that didn't fail the second time are in engine._store.items.
     Assert.equal(engine._store.items["record-no-04"], "Record No. 4");
@@ -900,8 +902,8 @@ add_task(async function test_processIncoming_failed_records() {
 
     // Confirm initial environment
     Assert.equal(engine.lastSync, 0);
-    Assert.equal(engine.toFetch.length, 0);
-    Assert.equal(engine.previousFailed.length, 0);
+    Assert.equal(engine.toFetch.size, 0);
+    Assert.equal(engine.previousFailed.size, 0);
     do_check_empty(engine._store.items);
 
     let observerSubject;
@@ -920,8 +922,8 @@ add_task(async function test_processIncoming_failed_records() {
                              NUMBER_OF_RECORDS - BOGUS_RECORDS.length);
 
     // Ensure that the bogus records will be fetched again on the next sync.
-    Assert.equal(engine.previousFailed.length, BOGUS_RECORDS.length);
-    Assert.deepEqual(engine.previousFailed.sort(), BOGUS_RECORDS.sort());
+    Assert.equal(engine.previousFailed.size, BOGUS_RECORDS.length);
+    Assert.deepEqual(Array.from(engine.previousFailed).sort(), BOGUS_RECORDS.sort());
 
     // Ensure the observer was notified
     Assert.equal(observerData, engine.name);
@@ -999,8 +1001,8 @@ add_task(async function test_processIncoming_decrypt_failed() {
   try {
 
     // Confirm initial state
-    Assert.equal(engine.toFetch.length, 0);
-    Assert.equal(engine.previousFailed.length, 0);
+    Assert.equal(engine.toFetch.size, 0);
+    Assert.equal(engine.previousFailed.size, 0);
 
     let observerSubject;
     let observerData;
@@ -1016,11 +1018,11 @@ add_task(async function test_processIncoming_decrypt_failed() {
     Assert.equal(ping.engines[0].incoming.failed, 4);
     Assert.equal(ping.engines[0].incoming.newFailed, 4);
 
-    Assert.equal(engine.previousFailed.length, 4);
-    Assert.equal(engine.previousFailed[0], "nojson");
-    Assert.equal(engine.previousFailed[1], "nojson2");
-    Assert.equal(engine.previousFailed[2], "nodecrypt");
-    Assert.equal(engine.previousFailed[3], "nodecrypt2");
+    Assert.equal(engine.previousFailed.size, 4);
+    Assert.ok(engine.previousFailed.has("nojson"));
+    Assert.ok(engine.previousFailed.has("nojson2"));
+    Assert.ok(engine.previousFailed.has("nodecrypt"));
+    Assert.ok(engine.previousFailed.has("nodecrypt2"));
 
     // Ensure the observer was notified
     Assert.equal(observerData, engine.name);
diff --git a/services/sync/tests/unit/test_telemetry.js b/services/sync/tests/unit/test_telemetry.js
index 82f0e0f827d2..d5720c18b81c 100644
--- a/services/sync/tests/unit/test_telemetry.js
+++ b/services/sync/tests/unit/test_telemetry.js
@@ -122,7 +122,7 @@ add_task(async function test_processIncoming_error() {
     // Make the 10 minutes old so it will only be synced in the toFetch phase.
     bogus_record.modified = Date.now() / 1000 - 60 * 10;
     engine.lastSync = Date.now() / 1000 - 60;
-    engine.toFetch = [BOGUS_GUID];
+    engine.toFetch = new SerializableSet([BOGUS_GUID]);
 
     let error, pingPayload, fullPing;
     try {
diff --git a/tools/lint/eslint/modules.json b/tools/lint/eslint/modules.json
index 71aedc68d180..f3454136d5f3 100644
--- a/tools/lint/eslint/modules.json
+++ b/tools/lint/eslint/modules.json
@@ -208,7 +208,7 @@
   "Traversal.jsm": ["TraversalRules", "TraversalHelper"],
   "UpdateTelemetry.jsm": ["AUSTLMY"],
   "UpdateTopLevelContentWindowIDHelper.jsm": ["trackBrowserWindow"],
-  "util.js": ["getChromeWindow", "Utils", "Svc"],
+  "util.js": ["getChromeWindow", "Utils", "Svc", "SerializableSet"],
   "utils.js": ["btoa", "encryptPayload", "makeIdentityConfig", "makeFxAccountsInternalMock", "configureFxAccountIdentity", "configureIdentity", "SyncTestingInfrastructure", "waitForZeroTimer", "Promise", "MockFxaStorageManager", "AccountState", "sumHistogram", "CommonUtils", "CryptoUtils", "TestingUtils", "promiseZeroTimer", "promiseNamedTimer", "getLoginTelemetryScalar", "syncTestLogging"],
   "Utils.jsm": ["Utils", "Logger", "PivotContext", "PrefCache"],
   "VariablesView.jsm": ["VariablesView", "escapeHTML"],

From cf332b3975180c59f52f09afb1cb8af1c34228f3 Mon Sep 17 00:00:00 2001
From: Josh Matthews <josh@joshmatthews.net>
Date: Fri, 26 Jan 2018 12:58:50 -0600
Subject: [PATCH 06/46] servo: Merge #18726 - Add script to automate pulling
 changes from upstream WPT (from jdm:automate-wpt-sync); r=asajeffrey

I want to make automated WPT syncing happen this quarter. This script should allow us to run a nightly job on the builder that grabs the latest changes from upstream runs our testsuite, updates the expected test results, then open a PR with the changes. It still requires saltfs changes to allow it to actually run, but I don't see any harm in getting it reviewed and merged before everything is ready.

Source-Repo: https://github.com/servo/servo
Source-Revision: ce17959f7c5f817bc5739c6693c93cafb1855f4f

--HG--
extra : subtree_source : https%3A//hg.mozilla.org/projects/converted-servo-linear
extra : subtree_revision : 21ffe7b18cf46b8811793fa78c685973c238f7b9
---
 servo/etc/ci/buildbot_steps.yml  |   3 +
 servo/etc/ci/update-wpt-checkout | 186 +++++++++++++++++++++++++++++++
 2 files changed, 189 insertions(+)
 create mode 100755 servo/etc/ci/update-wpt-checkout

diff --git a/servo/etc/ci/buildbot_steps.yml b/servo/etc/ci/buildbot_steps.yml
index 3c6d7ca67355..df86cee287e1 100644
--- a/servo/etc/ci/buildbot_steps.yml
+++ b/servo/etc/ci/buildbot_steps.yml
@@ -72,6 +72,9 @@ mac-nightly:
   - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release
   - ./mach package --release
   - ./mach upload-nightly mac
+  - ./etc/ci/update-wpt-checkout fetch-and-update-expectations
+  - ./etc/ci/update-wpt-checkout open-pr
+  - ./etc/ci/update-wpt-checkout cleanup
 
 linux-rel-intermittent:
   - ./mach clean-nightlies --keep 3 --force
diff --git a/servo/etc/ci/update-wpt-checkout b/servo/etc/ci/update-wpt-checkout
new file mode 100755
index 000000000000..6203d4641917
--- /dev/null
+++ b/servo/etc/ci/update-wpt-checkout
@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REMOTE_NAME=sync-fork
+LOG_FILE=test-wpt.log
+CURRENT_DATE=$(date +"%d-%m-%Y")
+BRANCH_NAME="wpt_update_${CURRENT_DATE}"
+
+export GIT_AUTHOR_NAME="WPT Sync Bot"
+export GIT_AUTHOR_EMAIL="josh+wptsync@joshmatthews.net"
+
+# Retrieve the HEAD commit and extract its hash
+function latest_git_commit() {
+    git log -1 --oneline | cut -f 1 -d ' '
+}
+
+# Create a new branch for this sync, pull down all changes from the upstream
+# web-platform-tests repository, and commit the changes.
+function unsafe_pull_from_upstream() {
+    git checkout -b "${1}" || return 1
+
+    OLD_COMMIT=$(latest_git_commit)
+
+    # Fetch all changes from upstream WPT and automatically transpose them
+    # into a single servo commit.
+    ./mach update-wpt --sync --no-upstream --patch || return 2
+
+    # If there was no new commit created, there are no changes that need syncing.
+    # Skip the remaining steps.
+    if [[ "$(latest_git_commit)" == "${OLD_COMMIT}" ]]; then
+        return 255
+    fi
+
+    # Update the manifest to include the new changes.
+    ./mach update-manifest || return 3
+
+    # Amend the existing commit with the new changes from updating the manifest.
+    git commit -a --amend --no-edit || return 4
+}
+
+# Remove all local traces of this sync operation.
+function cleanup() {
+    git remote rm "${REMOTE_NAME}" || true
+    git reset --hard || true
+    git checkout master || true
+    git branch -D "${BRANCH_NAME}" || true
+    ./mach update-wpt --abort || true
+}
+
+# Build Servo and run the full WPT testsuite, saving the results to a log file.
+function unsafe_run_tests() {
+    # Run the full testsuite and record the new test results.
+    ./mach test-wpt --release --processes 24 --log-raw "${1}" \
+           --always-succeed || return 1
+}
+
+# Using an existing log file, update the expected test results and amend the
+# last commit with the new results.
+function unsafe_update_metadata() {
+    ./mach update-wpt "${1}" || return 1
+    # Ignore any metadata changes to files that weren't modified by this sync.
+    git checkout -- tests/wpt/mozilla/meta || return 2
+    # Ensure any new directories or ini files are included in these changes.
+    git add tests/wpt/metadata || return 3
+    # Merge all changes with the existing commit.
+    git commit -a --amend --no-edit || return 4
+}
+
+# Push the branch to a remote branch, then open a PR for the branch
+# against servo/servo.
+function unsafe_open_pull_request() {
+    WPT_SYNC_USER=servo-wpt-sync
+
+    # If the branch doesn't exist, we'll silently exit. This deals with the
+    # case where an earlier step either failed or discovered that syncing
+    # is unnecessary.
+    git checkout "${BRANCH_NAME}" || return 0
+
+    if [[ -z "${WPT_SYNC_TOKEN+set}" ]]; then
+        echo "Github auth token missing from WPT_SYNC_TOKEN."
+        return 1
+    fi
+
+    # Push the changes to a remote branch owned by the bot.
+    AUTH="${WPT_SYNC_USER}:${WPT_SYNC_TOKEN}"
+    UPSTREAM="https://${AUTH}@github.com/${WPT_SYNC_USER}/servo.git"
+    git remote add "${REMOTE_NAME}" "${UPSTREAM}" || return 2
+    git push -f "${REMOTE_NAME}" "${BRANCH_NAME}" || return 3
+
+    # Prepare the pull request metadata.
+    BODY="Automated downstream sync of changes from upstream as of "
+    BODY+="${CURRENT_DATE}.\n"
+    BODY+="[no-wpt-sync]"
+    cat <<EOF >prdata.json || return 4
+{
+  "title": "Sync WPT with upstream (${CURRENT_DATE})",
+  "head": "${WPT_SYNC_USER}:${BRANCH_NAME}",
+  "base": "master",
+  "body": "${BODY}",
+  "maintainer_can_modify": true
+}
+EOF
+
+    # Open a pull request using the new branch.
+    curl -H "Authorization: token ${WPT_SYNC_TOKEN}" \
+         -H "Content-Type: application/json" \
+         --data @prdata.json \
+         https://api.github.com/repos/servo/servo/pulls || return 4
+}
+
+function pull_from_upstream() {
+    unsafe_pull_from_upstream "${1}" || { code="${?}"; cleanup; return "${code}"; }
+}
+
+function run_tests() {
+    unsafe_run_tests "${1}" || { code="${?}"; cleanup; return "${code}"; }
+}
+
+function update_metadata() {
+    unsafe_update_metadata "${1}" || { code="${?}"; cleanup; return "${code}"; }
+}
+
+function open_pull_request() {
+    unsafe_open_pull_request || { code="${?}"; cleanup; return "${code}"; }
+}
+
+SCRIPT_NAME="${0}"
+
+function update_test_results() {
+    run_tests "${LOG_FILE}"
+    update_metadata "${LOG_FILE}"
+}
+
+function fetch_upstream_changes() {
+    pull_from_upstream "${BRANCH_NAME}"
+}
+
+function usage() {
+    echo "usage: ${SCRIPT_NAME} [cmd]"
+    echo "  commands:"
+    echo "  - fetch-upstream-changes: create a branch with the latest changes from upstream"
+    echo "  - update-test-results: run the tests, update the expected test results, and commit the changes"
+    echo "  - fetch-and-update-expectations: combines fetch-upstream-changes and update-test-results"
+    echo "  - open-pr: open a pull request with the latest changes"
+    echo "  - cleanup: cleanup all traces of an in-progress sync and checkout the master branch"
+    exit 1
+}
+
+function main() {
+    if [[ "${1}" == "fetch-upstream-changes" ]] || [[ "${1}" == "fetch-and-update-expectations" ]]; then
+        code=""
+        fetch_upstream_changes || code="${?}"
+        if [[ "${code}" == "255" ]]; then
+            echo "No changes to sync."
+            return 0
+        fi
+    fi
+
+    if [[ "${1}" == "update-test-results" ]] || [[ "${1}" == "fetch-and-update-expectations" ]]; then
+        update_test_results
+
+    elif [[ "${1}" == "open-pr" ]]; then
+        open_pull_request
+
+    elif [[ "${1}" == "cleanup" ]]; then
+        cleanup
+
+    else
+        usage
+    fi
+}
+
+if [[ "$#" != 1 ]]; then
+    usage
+fi
+
+# Ensure we clean up after ourselves if this script is interrupted.
+trap 'cleanup' SIGINT SIGTERM
+main "${1}"

From 78404351388d8ddb2319389d8120380319f7f0bd Mon Sep 17 00:00:00 2001
From: David Keeler <dkeeler@mozilla.com>
Date: Fri, 19 Jan 2018 13:09:22 -0800
Subject: [PATCH 07/46] bug 1431791 - remove "Others" tab from certificate
 manager r=jaws,mgoodwin

MozReview-Commit-ID: HUPkIAMhNEX

--HG--
extra : rebase_source : a737297d8cb8800b03532d62b39e85711d6f0663
---
 .../preferences/in-content/privacy.xul        |  2 -
 .../en-US/chrome/pippki/certManager.dtd       |  2 -
 .../en-US/chrome/pippki/pippki.properties     |  3 --
 .../pki/resources/content/OrphanOverlay.xul   | 47 -------------------
 .../pki/resources/content/certManager.js      | 32 -------------
 .../pki/resources/content/certManager.xul     |  3 --
 .../pki/resources/content/deletecert.js       |  5 --
 security/manager/pki/resources/jar.mn         |  1 -
 security/manager/ssl/nsCertTree.cpp           |  8 ----
 .../browser/browser_deleteCert_ui.js          | 10 ----
 10 files changed, 113 deletions(-)
 delete mode 100644 security/manager/pki/resources/content/OrphanOverlay.xul

diff --git a/browser/components/preferences/in-content/privacy.xul b/browser/components/preferences/in-content/privacy.xul
index ec37e90060c7..4396861d12cc 100644
--- a/browser/components/preferences/in-content/privacy.xul
+++ b/browser/components/preferences/in-content/privacy.xul
@@ -672,12 +672,10 @@
                                 &certmgr.tab.others2;
                                 &certmgr.tab.websites3;
                                 &certmgr.tab.ca;
-                                &certmgr.tab.orphan2;
                                 &certmgr.mine2;
                                 &certmgr.others2;
                                 &certmgr.websites3;
                                 &certmgr.cas2;
-                                &certmgr.orphans2;
                                 &certmgr.certname;
                                 &certmgr.tokenname;
                                 &certmgr.view2.label;
diff --git a/security/manager/locales/en-US/chrome/pippki/certManager.dtd b/security/manager/locales/en-US/chrome/pippki/certManager.dtd
index e6318e1ce096..2133f3047688 100644
--- a/security/manager/locales/en-US/chrome/pippki/certManager.dtd
+++ b/security/manager/locales/en-US/chrome/pippki/certManager.dtd
@@ -8,13 +8,11 @@
 <!ENTITY certmgr.tab.others2                  "People">
 <!ENTITY certmgr.tab.websites3                "Servers">
 <!ENTITY certmgr.tab.ca                       "Authorities">
-<!ENTITY certmgr.tab.orphan2                  "Others">
 
 <!ENTITY certmgr.mine2                        "You have certificates from these organizations that identify you">
 <!ENTITY certmgr.others2                      "You have certificates on file that identify these people">
 <!ENTITY certmgr.websites3                    "You have certificates on file that identify these servers">
 <!ENTITY certmgr.cas2                         "You have certificates on file that identify these certificate authorities">
-<!ENTITY certmgr.orphans2                     "You have certificates on file that do not fit in any of the other categories">
 
 <!ENTITY certmgr.detail.general_tab.title     "General">
 <!ENTITY certmgr.detail.general_tab.accesskey "G">
diff --git a/security/manager/locales/en-US/chrome/pippki/pippki.properties b/security/manager/locales/en-US/chrome/pippki/pippki.properties
index 163384651067..ed3b867ed260 100644
--- a/security/manager/locales/en-US/chrome/pippki/pippki.properties
+++ b/security/manager/locales/en-US/chrome/pippki/pippki.properties
@@ -32,9 +32,6 @@ deleteEmailCertConfirm=Are you sure you want to delete these people’s e-mail c
 deleteEmailCertImpactDesc=If you delete a person’s e-mail certificate, you will no longer be able to send encrypted e-mail to that person.
 deleteEmailCertTitle=Delete E-Mail Certificates
 
-deleteOrphanCertConfirm=Are you sure you want to delete these certificates?
-deleteOrphanCertTitle=Delete Certificates
-
 # PKCS#12 file dialogs
 chooseP12RestoreFileDialog2=Certificate File to Import
 chooseP12BackupFileDialog=File Name to Backup
diff --git a/security/manager/pki/resources/content/OrphanOverlay.xul b/security/manager/pki/resources/content/OrphanOverlay.xul
deleted file mode 100644
index c267d0f7bef6..000000000000
--- a/security/manager/pki/resources/content/OrphanOverlay.xul
+++ /dev/null
@@ -1,47 +0,0 @@
-<?xml version="1.0"?>
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-
-<?xml-stylesheet href="chrome://global/skin/" type="text/css"?>
-
-<!DOCTYPE overlay SYSTEM "chrome://pippki/locale/certManager.dtd">
-
-<overlay id="OrphanOverlay"
-         xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-         xmlns:cert="http://netscape.com/rdf-cert#"
-         xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
-
-  <vbox id="OrphanCerts">
-    <description>&certmgr.orphans2;</description>
-    <separator class="thin"/>
-    <tree id="orphan-tree" flex="1" enableColumnDrag="true"
-              onselect="orphan_enableButtons()">
-      <treecols>
-        <treecol id="certcol" label="&certmgr.certname;" primary="true"
-                     persist="hidden width ordinal" flex="1"/>
-        <splitter class="tree-splitter"/>
-        <treecol id="tokencol" label="&certmgr.tokenname;"
-                     persist="hidden width ordinal" flex="1"/>
-      </treecols>
-      <treechildren ondblclick="viewCerts();"/>
-    </tree>
-
-    <separator class="thin"/>
-
-    <hbox>
-       <button id="orphan_viewButton" class="normal"
-               label="&certmgr.view2.label;"
-               accesskey="&certmgr.view2.accesskey;"
-               disabled="true" oncommand="viewCerts();"/>
-       <button id="orphan_exportButton" class="normal"
-               label="&certmgr.export.label;"
-               accesskey="&certmgr.export.accesskey;"
-               disabled="true" oncommand="exportCerts();"/>
-       <button id="orphan_deleteButton" class="normal"
-               label="&certmgr.delete2.label;"
-               accesskey="&certmgr.delete2.accesskey;"
-               disabled="true" oncommand="deleteCerts();"/>
-    </hbox>
-  </vbox>
-</overlay>
diff --git a/security/manager/pki/resources/content/certManager.js b/security/manager/pki/resources/content/certManager.js
index 5145895b507e..b1bad26efc53 100644
--- a/security/manager/pki/resources/content/certManager.js
+++ b/security/manager/pki/resources/content/certManager.js
@@ -47,11 +47,6 @@ var emailTreeView;
  * @type nsICertTree
  */
 var userTreeView;
-/**
- * Cert tree for the "Other" tab.
- * @type nsICertTree
- */
-var orphanTreeView;
 
 function LoadCerts() {
   certdb = Components.classes[nsX509CertDB].getService(nsIX509CertDB);
@@ -77,11 +72,6 @@ function LoadCerts() {
   userTreeView.loadCertsFromCache(certcache, nsIX509Cert.USER_CERT);
   document.getElementById("user-tree").view = userTreeView;
 
-  orphanTreeView = Components.classes[nsCertTree]
-                      .createInstance(nsICertTree);
-  orphanTreeView.loadCertsFromCache(certcache, nsIX509Cert.UNKNOWN_CERT);
-  document.getElementById("orphan-tree").view = orphanTreeView;
-
   enableBackupAllButton();
 }
 
@@ -95,7 +85,6 @@ function getSelectedCerts() {
   var mine_tab = document.getElementById("mine_tab");
   var others_tab = document.getElementById("others_tab");
   var websites_tab = document.getElementById("websites_tab");
-  var orphan_tab = document.getElementById("orphan_tab");
   var items = null;
   if (ca_tab.selected) {
     items = caTreeView.selection;
@@ -105,8 +94,6 @@ function getSelectedCerts() {
     items = emailTreeView.selection;
   } else if (websites_tab.selected) {
     items = serverTreeView.selection;
-  } else if (orphan_tab.selected) {
-    items = orphanTreeView.selection;
   }
   selected_certs = [];
   var cert = null;
@@ -128,8 +115,6 @@ function getSelectedCerts() {
           cert = emailTreeView.getCert(j);
         } else if (websites_tab.selected) {
           cert = serverTreeView.getCert(j);
-        } else if (orphan_tab.selected) {
-          cert = orphanTreeView.getCert(j);
         }
         if (cert) {
           var sc = selected_certs.length;
@@ -146,7 +131,6 @@ function getSelectedTreeItems() {
   var mine_tab = document.getElementById("mine_tab");
   var others_tab = document.getElementById("others_tab");
   var websites_tab = document.getElementById("websites_tab");
-  var orphan_tab = document.getElementById("orphan_tab");
   var items = null;
   if (ca_tab.selected) {
     items = caTreeView.selection;
@@ -156,8 +140,6 @@ function getSelectedTreeItems() {
     items = emailTreeView.selection;
   } else if (websites_tab.selected) {
     items = serverTreeView.selection;
-  } else if (orphan_tab.selected) {
-    items = orphanTreeView.selection;
   }
   selected_certs = [];
   selected_tree_items = [];
@@ -181,8 +163,6 @@ function getSelectedTreeItems() {
           tree_item = emailTreeView.getTreeItem(j);
         } else if (websites_tab.selected) {
           tree_item = serverTreeView.getTreeItem(j);
-        } else if (orphan_tab.selected) {
-          tree_item = orphanTreeView.getTreeItem(j);
         }
         if (tree_item) {
           var sc = selected_tree_items.length;
@@ -278,15 +258,6 @@ function email_enableButtons() {
   enableButtonsForCertTree(emailTreeView, idList);
 }
 
-function orphan_enableButtons() {
-  let idList = [
-    "orphan_viewButton",
-    "orphan_exportButton",
-    "orphan_deleteButton",
-  ];
-  enableButtonsForCertTree(orphanTreeView, idList);
-}
-
 function backupCerts() {
   getSelectedCerts();
   var numcerts = selected_certs.length;
@@ -405,7 +376,6 @@ function deleteCerts() {
     "websites_tab": serverTreeView,
     "ca_tab": caTreeView,
     "others_tab": emailTreeView,
-    "orphan_tab": orphanTreeView,
   };
   let selTab = document.getElementById("certMgrTabbox").selectedItem;
   let selTabID = selTab.getAttribute("id");
@@ -490,6 +460,4 @@ function addException() {
   var certcache = certdb.getCerts();
   serverTreeView.loadCertsFromCache(certcache, nsIX509Cert.SERVER_CERT);
   serverTreeView.selection.clearSelection();
-  orphanTreeView.loadCertsFromCache(certcache, nsIX509Cert.UNKNOWN_CERT);
-  orphanTreeView.selection.clearSelection();
 }
diff --git a/security/manager/pki/resources/content/certManager.xul b/security/manager/pki/resources/content/certManager.xul
index 5d42efd140e5..de9cc0eb35bf 100644
--- a/security/manager/pki/resources/content/certManager.xul
+++ b/security/manager/pki/resources/content/certManager.xul
@@ -9,7 +9,6 @@
 <?xul-overlay href="chrome://pippki/content/OthersOverlay.xul"?>
 <?xul-overlay href="chrome://pippki/content/WebSitesOverlay.xul"?>
 <?xul-overlay href="chrome://pippki/content/CAOverlay.xul"?>
-<?xul-overlay href="chrome://pippki/content/OrphanOverlay.xul"?>
 
 <!DOCTYPE dialog SYSTEM "chrome://pippki/locale/certManager.dtd">
 
@@ -34,14 +33,12 @@
         <tab id="others_tab" label="&certmgr.tab.others2;"/>
         <tab id="websites_tab" label="&certmgr.tab.websites3;"/>
         <tab id="ca_tab" label="&certmgr.tab.ca;" selected="true"/>
-        <tab id="orphan_tab" label="&certmgr.tab.orphan2;"/>
       </tabs>
       <tabpanels flex="1">
         <vbox id="myCerts" flex="1"/>
         <vbox id="othersCerts" flex="1"/>
         <vbox id="webCerts" flex="1"/>
         <vbox id="CACerts" flex="1"/>
-        <vbox id="OrphanCerts" flex="1"/>
       </tabpanels>
     </tabbox>
 
diff --git a/security/manager/pki/resources/content/deletecert.js b/security/manager/pki/resources/content/deletecert.js
index 24519e7a7952..eebfb688f956 100644
--- a/security/manager/pki/resources/content/deletecert.js
+++ b/security/manager/pki/resources/content/deletecert.js
@@ -83,11 +83,6 @@ function onLoad() {
       confirm = bundle.getString("deleteEmailCertConfirm");
       impact = bundle.getString("deleteEmailCertImpactDesc");
       break;
-    case "orphan_tab":
-      title = bundle.getString("deleteOrphanCertTitle");
-      confirm = bundle.getString("deleteOrphanCertConfirm");
-      impact = "";
-      break;
     default:
       return;
   }
diff --git a/security/manager/pki/resources/jar.mn b/security/manager/pki/resources/jar.mn
index 960b740acc63..c07bdd09bfdd 100644
--- a/security/manager/pki/resources/jar.mn
+++ b/security/manager/pki/resources/jar.mn
@@ -6,7 +6,6 @@ pippki.jar:
 % content pippki %content/pippki/
     content/pippki/CAOverlay.xul             (content/CAOverlay.xul)
     content/pippki/MineOverlay.xul           (content/MineOverlay.xul)
-    content/pippki/OrphanOverlay.xul         (content/OrphanOverlay.xul)
     content/pippki/OthersOverlay.xul         (content/OthersOverlay.xul)
     content/pippki/WebSitesOverlay.xul       (content/WebSitesOverlay.xul)
     content/pippki/certDump.xul              (content/certDump.xul)
diff --git a/security/manager/ssl/nsCertTree.cpp b/security/manager/ssl/nsCertTree.cpp
index f040ede88675..70e5b099d514 100644
--- a/security/manager/ssl/nsCertTree.cpp
+++ b/security/manager/ssl/nsCertTree.cpp
@@ -483,14 +483,6 @@ nsCertTree::GetCertsByTypeFromCertList(CERTCertList *aCertList,
         addOverrides = true;
       }
       else
-      if (aWantedType == nsIX509Cert::UNKNOWN_CERT
-          && thisCertType == nsIX509Cert::UNKNOWN_CERT) {
-        // This unknown cert was stored without trust.
-        // If there are associated overrides, do not show as unknown.
-        // If there are no associated overrides, display as unknown.
-        wantThisCertIfNoOverrides = true;
-      }
-      else
       if (aWantedType == nsIX509Cert::SERVER_CERT
           && thisCertType == nsIX509Cert::SERVER_CERT) {
         // This server cert is explicitly marked as a web site peer,
diff --git a/security/manager/ssl/tests/mochitest/browser/browser_deleteCert_ui.js b/security/manager/ssl/tests/mochitest/browser/browser_deleteCert_ui.js
index f08811333f4c..05d278469b2f 100644
--- a/security/manager/ssl/tests/mochitest/browser/browser_deleteCert_ui.js
+++ b/security/manager/ssl/tests/mochitest/browser/browser_deleteCert_ui.js
@@ -181,16 +181,6 @@ add_task(async function testDeleteCACerts() {
                     expectedImpact);
 });
 
-// Test deleting certs from the "Other" tab.
-add_task(async function testDeleteOtherCerts() {
-  const expectedTitle = "Delete Certificates";
-  const expectedConfirmMsg =
-    "Are you sure you want to delete these certificates?";
-  const expectedImpact = "";
-  await testHelper("orphan_tab", expectedTitle, expectedConfirmMsg,
-                    expectedImpact);
-});
-
 // Test that the right values are returned when the dialog is accepted.
 add_task(async function testAcceptDialogReturnValues() {
   let [win, retVals] = await openDeleteCertConfirmDialog("ca_tab" /* arbitrary */);

From 0375f07548f1bf18db70959d911cfc95930561f6 Mon Sep 17 00:00:00 2001
From: David Keeler <dkeeler@mozilla.com>
Date: Thu, 25 Jan 2018 16:56:56 -0800
Subject: [PATCH 08/46] bug 1432923 - fix a leak in some TransportTests r=drno

Before this patch, TransportTest::Reset would assign to p1_ and p2_ without
releasing their resources first. Since they're not RAII types, they would just
leak. This fixes the leak by a) explicitly initializing them to nullptr in the
constructor and b) deleting them if they previously had a non-null value.

MozReview-Commit-ID: 20U6sqRWg06

--HG--
extra : rebase_source : e304c7636a9b31df53d3ff0e8d7e0042f7e82494
---
 media/mtransport/test/transport_unittests.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/media/mtransport/test/transport_unittests.cpp b/media/mtransport/test/transport_unittests.cpp
index 01ec1cf1eb76..167a14be1fd5 100644
--- a/media/mtransport/test/transport_unittests.cpp
+++ b/media/mtransport/test/transport_unittests.cpp
@@ -830,6 +830,8 @@ class TransportTest : public MtransportTest {
   TransportTest() {
     fds_[0] = nullptr;
     fds_[1] = nullptr;
+    p1_ = nullptr;
+    p2_ = nullptr;
   }
 
   void TearDown() override {
@@ -858,6 +860,12 @@ class TransportTest : public MtransportTest {
   }
 
   void Reset() {
+    if (p1_) {
+      delete p1_;
+    }
+    if (p2_) {
+      delete p2_;
+    }
     p1_ = new TransportTestPeer(target_, "P1", test_utils_);
     p2_ = new TransportTestPeer(target_, "P2", test_utils_);
   }

From 52ab84dbf15ea79280905fdc89124e1a6b06d7a1 Mon Sep 17 00:00:00 2001
From: Noemi Erli <nerli@mozilla.com>
Date: Wed, 24 Jan 2018 20:04:34 +0200
Subject: [PATCH 09/46] Backed out 4 changesets (bug 1381645) for failing in
 dom/tests/mochitest/general/test_interfaces.html on a CLOSED TREE

Backed out changeset 7b2482a5eb04 (bug 1381645)
Backed out changeset ef3252b0ba21 (bug 1381645)
Backed out changeset 7c022a5e28f2 (bug 1381645)
Backed out changeset 2f71c8989cc0 (bug 1381645)
---
 .../mochitest/general/test_interfaces.js      |  16 +-
 dom/vr/test/mochitest/mochitest.ini           |   1 -
 dom/webidl/Navigator.webidl                   |   4 +-
 dom/webidl/VRDisplay.webidl                   |   7 -
 .../meta/webvr/idlharness.html.ini            | 204 ------------------
 5 files changed, 10 insertions(+), 222 deletions(-)

diff --git a/dom/tests/mochitest/general/test_interfaces.js b/dom/tests/mochitest/general/test_interfaces.js
index 6fa4c17816af..f5fa0a3b9f36 100644
--- a/dom/tests/mochitest/general/test_interfaces.js
+++ b/dom/tests/mochitest/general/test_interfaces.js
@@ -1181,21 +1181,21 @@ var interfaceNamesInGlobalScope =
 // IMPORTANT: Do not change this list without review from a DOM peer!
     {name: "VideoStreamTrack", insecureContext: true},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRDisplay", releaseNonWindows: false},
+    {name: "VRDisplay", insecureContext: true, releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRDisplayCapabilities", releaseNonWindows: false},
+    {name: "VRDisplayCapabilities", insecureContext: true, releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRDisplayEvent", releaseNonWindows: false},
+    {name: "VRDisplayEvent", insecureContext: true, releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VREyeParameters", releaseNonWindows: false},
+    {name: "VREyeParameters", insecureContext: true, releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRFieldOfView", releaseNonWindows: false},
+    {name: "VRFieldOfView", insecureContext: true, releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRFrameData", releaseNonWindows: false},
+    {name: "VRFrameData", insecureContext: true, releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRPose", releaseNonWindows: false},
+    {name: "VRPose", insecureContext: true, releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
-    {name: "VRStageParameters", releaseNonWindows: false},
+    {name: "VRStageParameters", insecureContext: true, releaseNonWindows: false},
 // IMPORTANT: Do not change this list without review from a DOM peer!
     {name: "VTTCue", insecureContext: true},
 // IMPORTANT: Do not change this list without review from a DOM peer!
diff --git a/dom/vr/test/mochitest/mochitest.ini b/dom/vr/test/mochitest/mochitest.ini
index cdd6ee4e02a9..c5b82e719e6d 100644
--- a/dom/vr/test/mochitest/mochitest.ini
+++ b/dom/vr/test/mochitest/mochitest.ini
@@ -1,6 +1,5 @@
 # Please confirm there is no other VR display connected. Otherwise, VRPuppetDisplay can't be attached.
 [DEFAULT]
-scheme = https
 support-files =
   VRSimulationDriver.js
   requestPresent.js
diff --git a/dom/webidl/Navigator.webidl b/dom/webidl/Navigator.webidl
index 46b97771c1dc..d9af6d027115 100644
--- a/dom/webidl/Navigator.webidl
+++ b/dom/webidl/Navigator.webidl
@@ -217,10 +217,10 @@ partial interface Navigator {
 };
 
 partial interface Navigator {
-  [SecureContext, Throws, Pref="dom.vr.enabled"]
+  [Throws, Pref="dom.vr.enabled"]
   Promise<sequence<VRDisplay>> getVRDisplays();
   // TODO: Use FrozenArray once available. (Bug 1236777)
-  [SecureContext, Frozen, Cached, Pure, Pref="dom.vr.enabled"]
+  [Frozen, Cached, Pure, Pref="dom.vr.enabled"]
   readonly attribute sequence<VRDisplay> activeVRDisplays;
   [ChromeOnly, Pref="dom.vr.enabled"]
   readonly attribute boolean isWebVRContentDetected;
diff --git a/dom/webidl/VRDisplay.webidl b/dom/webidl/VRDisplay.webidl
index c75fb9a6de5c..43bcd3e00ebc 100644
--- a/dom/webidl/VRDisplay.webidl
+++ b/dom/webidl/VRDisplay.webidl
@@ -9,7 +9,6 @@ enum VREye {
 };
 
 [Pref="dom.vr.enabled",
- SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRFieldOfView {
   readonly attribute double upDegrees;
@@ -50,7 +49,6 @@ dictionary VRLayer {
  * These are expected to be static per-device/per-user.
  */
 [Pref="dom.vr.enabled",
- SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRDisplayCapabilities {
   /**
@@ -92,7 +90,6 @@ interface VRDisplayCapabilities {
  * that support room-scale experiences.
  */
 [Pref="dom.vr.enabled",
- SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRStageParameters {
   /**
@@ -118,7 +115,6 @@ interface VRStageParameters {
 };
 
 [Pref="dom.vr.enabled",
- SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRPose
 {
@@ -140,7 +136,6 @@ interface VRPose
 
 [Constructor,
  Pref="dom.vr.enabled",
- SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRFrameData {
   readonly attribute DOMHighResTimeStamp timestamp;
@@ -163,7 +158,6 @@ interface VRSubmitFrameResult {
 };
 
 [Pref="dom.vr.enabled",
- SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VREyeParameters {
   /**
@@ -187,7 +181,6 @@ interface VREyeParameters {
 };
 
 [Pref="dom.vr.enabled",
- SecureContext,
  HeaderFile="mozilla/dom/VRDisplay.h"]
 interface VRDisplay : EventTarget {
   /**
diff --git a/testing/web-platform/meta/webvr/idlharness.html.ini b/testing/web-platform/meta/webvr/idlharness.html.ini
index 926c1fcf9500..f784e49ab6da 100644
--- a/testing/web-platform/meta/webvr/idlharness.html.ini
+++ b/testing/web-platform/meta/webvr/idlharness.html.ini
@@ -11,207 +11,3 @@
   [HTMLIFrameElement interface: attribute allowvr]
     expected: FAIL
 
-  [Navigator interface: operation getVRDisplays()]
-    expected: FAIL
-
-  [Navigator interface: attribute activeVRDisplays]
-    expected: FAIL
-
-  [VRDisplay interface: existence and properties of interface object]
-    expected: FAIL
-
-  [VRDisplay interface object length]
-    expected: FAIL
-
-  [VRDisplay interface object name]
-    expected: FAIL
-
-  [VRDisplay interface: existence and properties of interface prototype object]
-    expected: FAIL
-
-  [VRDisplay interface: existence and properties of interface prototype object's "constructor" property]
-    expected: FAIL
-
-  [VRDisplay interface: attribute isPresenting]
-    expected: FAIL
-
-  [VRDisplay interface: attribute capabilities]
-    expected: FAIL
-
-  [VRDisplay interface: attribute stageParameters]
-    expected: FAIL
-
-  [VRDisplay interface: operation getEyeParameters(VREye)]
-    expected: FAIL
-
-  [VRDisplay interface: attribute displayId]
-    expected: FAIL
-
-  [VRDisplay interface: attribute displayName]
-    expected: FAIL
-
-  [VRDisplay interface: operation getFrameData(VRFrameData)]
-    expected: FAIL
-
-  [VRDisplay interface: attribute depthNear]
-    expected: FAIL
-
-  [VRDisplay interface: attribute depthFar]
-    expected: FAIL
-
-  [VRDisplay interface: operation requestAnimationFrame(FrameRequestCallback)]
-    expected: FAIL
-
-  [VRDisplay interface: operation cancelAnimationFrame(long)]
-    expected: FAIL
-
-  [VRDisplay interface: operation requestPresent([object Object\])]
-    expected: FAIL
-
-  [VRDisplay interface: operation exitPresent()]
-    expected: FAIL
-
-  [VRDisplay interface: operation getLayers()]
-    expected: FAIL
-
-  [VRDisplay interface: operation submitFrame()]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface: existence and properties of interface object]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface object length]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface object name]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface: existence and properties of interface prototype object]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface: existence and properties of interface prototype object's "constructor" property]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface: attribute hasPosition]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface: attribute hasExternalDisplay]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface: attribute canPresent]
-    expected: FAIL
-
-  [VRDisplayCapabilities interface: attribute maxLayers]
-    expected: FAIL
-
-  [VRPose interface: existence and properties of interface object]
-    expected: FAIL
-
-  [VRPose interface object length]
-    expected: FAIL
-
-  [VRPose interface object name]
-    expected: FAIL
-
-  [VRPose interface: existence and properties of interface prototype object]
-    expected: FAIL
-
-  [VRPose interface: existence and properties of interface prototype object's "constructor" property]
-    expected: FAIL
-
-  [VRPose interface: attribute position]
-    expected: FAIL
-
-  [VRPose interface: attribute linearVelocity]
-    expected: FAIL
-
-  [VRPose interface: attribute linearAcceleration]
-    expected: FAIL
-
-  [VRPose interface: attribute orientation]
-    expected: FAIL
-
-  [VRPose interface: attribute angularVelocity]
-    expected: FAIL
-
-  [VRPose interface: attribute angularAcceleration]
-    expected: FAIL
-
-  [VRFrameData interface: existence and properties of interface object]
-    expected: FAIL
-
-  [VRFrameData interface object length]
-    expected: FAIL
-
-  [VRFrameData interface object name]
-    expected: FAIL
-
-  [VRFrameData interface: existence and properties of interface prototype object]
-    expected: FAIL
-
-  [VRFrameData interface: existence and properties of interface prototype object's "constructor" property]
-    expected: FAIL
-
-  [VRFrameData interface: attribute leftProjectionMatrix]
-    expected: FAIL
-
-  [VRFrameData interface: attribute leftViewMatrix]
-    expected: FAIL
-
-  [VRFrameData interface: attribute rightProjectionMatrix]
-    expected: FAIL
-
-  [VRFrameData interface: attribute rightViewMatrix]
-    expected: FAIL
-
-  [VRFrameData interface: attribute pose]
-    expected: FAIL
-
-  [VREyeParameters interface: existence and properties of interface object]
-    expected: FAIL
-
-  [VREyeParameters interface object length]
-    expected: FAIL
-
-  [VREyeParameters interface object name]
-    expected: FAIL
-
-  [VREyeParameters interface: existence and properties of interface prototype object]
-    expected: FAIL
-
-  [VREyeParameters interface: existence and properties of interface prototype object's "constructor" property]
-    expected: FAIL
-
-  [VREyeParameters interface: attribute offset]
-    expected: FAIL
-
-  [VREyeParameters interface: attribute renderWidth]
-    expected: FAIL
-
-  [VREyeParameters interface: attribute renderHeight]
-    expected: FAIL
-
-  [VRStageParameters interface: existence and properties of interface object]
-    expected: FAIL
-
-  [VRStageParameters interface object length]
-    expected: FAIL
-
-  [VRStageParameters interface object name]
-    expected: FAIL
-
-  [VRStageParameters interface: existence and properties of interface prototype object]
-    expected: FAIL
-
-  [VRStageParameters interface: existence and properties of interface prototype object's "constructor" property]
-    expected: FAIL
-
-  [VRStageParameters interface: attribute sittingToStandingTransform]
-    expected: FAIL
-
-  [VRStageParameters interface: attribute sizeX]
-    expected: FAIL
-
-  [VRStageParameters interface: attribute sizeZ]
-    expected: FAIL
-

From 74425c36d719300cb0eb93b190ce2a19e4114bf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Fri, 19 Jan 2018 23:51:30 +0100
Subject: [PATCH 10/46] Bug 1431855: Flush notifications via the document in
 SVGDocumentWrapper. r=dholbert

MozReview-Commit-ID: BMmhvqFE7O7

--HG--
extra : rebase_source : 516aaaae3b9b6de9e72d2fc724b08385ae09e4e7
---
 image/SVGDocumentWrapper.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/image/SVGDocumentWrapper.cpp b/image/SVGDocumentWrapper.cpp
index a0bab41a7090..054d0b2b55c5 100644
--- a/image/SVGDocumentWrapper.cpp
+++ b/image/SVGDocumentWrapper.cpp
@@ -419,10 +419,8 @@ SVGDocumentWrapper::UnregisterForXPCOMShutdown()
 void
 SVGDocumentWrapper::FlushLayout()
 {
-  nsCOMPtr<nsIPresShell> presShell;
-  mViewer->GetPresShell(getter_AddRefs(presShell));
-  if (presShell) {
-    presShell->FlushPendingNotifications(FlushType::Layout);
+  if (nsIDocument* doc = GetDocument()) {
+    doc->FlushPendingNotifications(FlushType::Layout);
   }
 }
 

From dbd4f2f3f13aabcfac543754ef543baf1aed721f Mon Sep 17 00:00:00 2001
From: Jon Leighton <j@jonathanleighton.com>
Date: Fri, 26 Jan 2018 13:58:01 -0600
Subject: [PATCH 11/46] servo: Merge #19544 - Text selection API conformance
 (from jonleighton:issue-19171-5); r=nox

This is my next batch of changes for issue #19171. All the tests in tests/wpt/metadata/html/semantics/forms/textfieldselection/ are now passing (and also some tests outside of there).

I've made detailed notes about the changes in each commit message.

r? @KiChjang

Source-Repo: https://github.com/servo/servo
Source-Revision: c2dfece49f1d59f51a3207cd3d88c282ee1adf70

--HG--
extra : subtree_source : https%3A//hg.mozilla.org/projects/converted-servo-linear
extra : subtree_revision : 6321a59ea353cb0f8196c63a1e2644ee8171a584
---
 .../components/script/dom/htmlinputelement.rs |  72 ++++-
 .../script/dom/htmltextareaelement.rs         |  51 ++-
 servo/components/script/dom/textcontrol.rs    | 162 +++++++++-
 .../script/dom/webidls/HTMLFormElement.webidl |   8 +
 .../dom/webidls/HTMLInputElement.webidl       |  10 +-
 .../dom/webidls/HTMLTextAreaElement.webidl    |  10 +-
 servo/components/script/textinput.rs          | 304 +++++++++++-------
 servo/tests/unit/script/textinput.rs          |  41 ++-
 8 files changed, 476 insertions(+), 182 deletions(-)

diff --git a/servo/components/script/dom/htmlinputelement.rs b/servo/components/script/dom/htmlinputelement.rs
index ed153529889f..e70afde60c2b 100755
--- a/servo/components/script/dom/htmlinputelement.rs
+++ b/servo/components/script/dom/htmlinputelement.rs
@@ -8,6 +8,7 @@ use dom::attr::Attr;
 use dom::bindings::cell::DomRefCell;
 use dom::bindings::codegen::Bindings::EventBinding::EventMethods;
 use dom::bindings::codegen::Bindings::FileListBinding::FileListMethods;
+use dom::bindings::codegen::Bindings::HTMLFormElementBinding::SelectionMode;
 use dom::bindings::codegen::Bindings::HTMLInputElementBinding;
 use dom::bindings::codegen::Bindings::HTMLInputElementBinding::HTMLInputElementMethods;
 use dom::bindings::codegen::Bindings::KeyboardEventBinding::KeyboardEventMethods;
@@ -52,7 +53,7 @@ use std::ops::Range;
 use style::attr::AttrValue;
 use style::element_state::ElementState;
 use style::str::split_commas;
-use textinput::{Direction, Selection, SelectionDirection, TextInput};
+use textinput::{Direction, SelectionDirection, TextInput};
 use textinput::KeyReaction::{DispatchInput, Nothing, RedrawSelection, TriggerDefaultAction};
 use textinput::Lines::Single;
 
@@ -188,7 +189,6 @@ pub struct HTMLInputElement {
     input_type: Cell<InputType>,
     checked_changed: Cell<bool>,
     placeholder: DomRefCell<DOMString>,
-    value_changed: Cell<bool>,
     size: Cell<u32>,
     maxlength: Cell<i32>,
     minlength: Cell<i32>,
@@ -244,7 +244,6 @@ impl HTMLInputElement {
             input_type: Cell::new(Default::default()),
             placeholder: DomRefCell::new(DOMString::new()),
             checked_changed: Cell::new(false),
-            value_changed: Cell::new(false),
             maxlength: Cell::new(DEFAULT_MAX_LENGTH),
             minlength: Cell::new(DEFAULT_MIN_LENGTH),
             size: Cell::new(DEFAULT_INPUT_SIZE),
@@ -374,7 +373,7 @@ impl LayoutHTMLInputElementHelpers for LayoutDom<HTMLInputElement> {
         match (*self.unsafe_get()).input_type() {
             InputType::Password => {
                 let text = get_raw_textinput_value(self);
-                let sel = textinput.get_absolute_selection_range();
+                let sel = textinput.sorted_selection_offsets_range();
 
                 // Translate indices from the raw value to indices in the replacement value.
                 let char_start = text[.. sel.start].chars().count();
@@ -383,7 +382,7 @@ impl LayoutHTMLInputElementHelpers for LayoutDom<HTMLInputElement> {
                 let bytes_per_char = PASSWORD_REPLACEMENT_CHAR.len_utf8();
                 Some(char_start * bytes_per_char .. char_end * bytes_per_char)
             }
-            input_type if input_type.is_textual() => Some(textinput.get_absolute_selection_range()),
+            input_type if input_type.is_textual() => Some(textinput.sorted_selection_offsets_range()),
             _ => None
         }
     }
@@ -417,6 +416,35 @@ impl TextControl for HTMLInputElement {
             _ => false
         }
     }
+
+    // https://html.spec.whatwg.org/multipage/#concept-input-apply
+    //
+    // Defines input types to which the select() IDL method applies. These are a superset of the
+    // types for which selection_api_applies() returns true.
+    //
+    // Types omitted which could theoretically be included if they were
+    // rendered as a text control: file
+    fn has_selectable_text(&self) -> bool {
+        match self.input_type() {
+            InputType::Text | InputType::Search | InputType::Url
+            | InputType::Tel | InputType::Password | InputType::Email
+            | InputType::Date | InputType::Month | InputType::Week
+            | InputType::Time | InputType::DatetimeLocal | InputType::Number
+            | InputType::Color => {
+                true
+            }
+
+            InputType::Button | InputType::Checkbox | InputType::File
+            | InputType::Hidden | InputType::Image | InputType::Radio
+            | InputType::Range | InputType::Reset | InputType::Submit => {
+                false
+            }
+        }
+    }
+
+    fn set_dirty_value_flag(&self, value: bool) {
+        self.value_dirty.set(value)
+    }
 }
 
 impl HTMLInputElementMethods for HTMLInputElement {
@@ -538,8 +566,7 @@ impl HTMLInputElementMethods for HTMLInputElement {
                 self.sanitize_value();
                 // Step 5.
                 if *self.textinput.borrow().single_line_content() != old_value {
-                    self.textinput.borrow_mut()
-                        .adjust_horizontal_to_limit(Direction::Forward, Selection::NotSelected);
+                    self.textinput.borrow_mut().clear_selection_to_limit(Direction::Forward);
                 }
             }
             ValueMode::Default |
@@ -557,7 +584,6 @@ impl HTMLInputElementMethods for HTMLInputElement {
             }
         }
 
-        self.value_changed.set(true);
         self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
         Ok(())
     }
@@ -687,6 +713,11 @@ impl HTMLInputElementMethods for HTMLInputElement {
         }
     }
 
+    // https://html.spec.whatwg.org/multipage/#dom-textarea/input-select
+    fn Select(&self) {
+        self.dom_select(); // defined in TextControl trait
+    }
+
     // https://html.spec.whatwg.org/multipage/#dom-textarea/input-selectionstart
     fn GetSelectionStart(&self) -> Option<u32> {
         self.get_dom_selection_start()
@@ -722,6 +753,19 @@ impl HTMLInputElementMethods for HTMLInputElement {
         self.set_dom_selection_range(start, end, direction)
     }
 
+    // https://html.spec.whatwg.org/multipage/#dom-textarea/input-setrangetext
+    fn SetRangeText(&self, replacement: DOMString) -> ErrorResult {
+        // defined in TextControl trait
+        self.set_dom_range_text(replacement, None, None, Default::default())
+    }
+
+    // https://html.spec.whatwg.org/multipage/#dom-textarea/input-setrangetext
+    fn SetRangeText_(&self, replacement: DOMString, start: u32, end: u32,
+                     selection_mode: SelectionMode) -> ErrorResult {
+        // defined in TextControl trait
+        self.set_dom_range_text(replacement, Some(start), Some(end), selection_mode)
+    }
+
     // Select the files based on filepaths passed in,
     // enabled by dom.htmlinputelement.select_files.enabled,
     // used for test purpose.
@@ -902,7 +946,6 @@ impl HTMLInputElement {
         self.SetValue(self.DefaultValue())
             .expect("Failed to reset input value to default.");
         self.value_dirty.set(false);
-        self.value_changed.set(false);
         self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
     }
 
@@ -1116,6 +1159,8 @@ impl VirtualMethods for HTMLInputElement {
 
                         // https://html.spec.whatwg.org/multipage/#input-type-change
                         let (old_value_mode, old_idl_value) = (self.value_mode(), self.Value());
+                        let previously_selectable = self.selection_api_applies();
+
                         self.input_type.set(new_type);
 
                         if new_type.is_textual() {
@@ -1167,6 +1212,11 @@ impl VirtualMethods for HTMLInputElement {
 
                         // Step 6
                         self.sanitize_value();
+
+                        // Steps 7-9
+                        if !previously_selectable && self.selection_api_applies() {
+                            self.textinput.borrow_mut().clear_selection_to_limit(Direction::Backward);
+                        }
                     },
                     AttributeMutation::Removed => {
                         if self.input_type() == InputType::Radio {
@@ -1184,7 +1234,7 @@ impl VirtualMethods for HTMLInputElement {
 
                 self.update_placeholder_shown_state();
             },
-            &local_name!("value") if !self.value_changed.get() => {
+            &local_name!("value") if !self.value_dirty.get() => {
                 let value = mutation.new_value(attr).map(|value| (**value).to_owned());
                 self.textinput.borrow_mut().set_content(
                     value.map_or(DOMString::new(), DOMString::from));
@@ -1327,7 +1377,7 @@ impl VirtualMethods for HTMLInputElement {
                                                      keyevent.MetaKey());
                         },
                         DispatchInput => {
-                            self.value_changed.set(true);
+                            self.value_dirty.set(true);
                             self.update_placeholder_shown_state();
                             self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
                             event.mark_as_handled();
diff --git a/servo/components/script/dom/htmltextareaelement.rs b/servo/components/script/dom/htmltextareaelement.rs
index af69e227d606..bdb0020c616b 100755
--- a/servo/components/script/dom/htmltextareaelement.rs
+++ b/servo/components/script/dom/htmltextareaelement.rs
@@ -5,6 +5,7 @@
 use dom::attr::Attr;
 use dom::bindings::cell::DomRefCell;
 use dom::bindings::codegen::Bindings::EventBinding::EventMethods;
+use dom::bindings::codegen::Bindings::HTMLFormElementBinding::SelectionMode;
 use dom::bindings::codegen::Bindings::HTMLTextAreaElementBinding;
 use dom::bindings::codegen::Bindings::HTMLTextAreaElementBinding::HTMLTextAreaElementMethods;
 use dom::bindings::codegen::Bindings::NodeBinding::NodeMethods;
@@ -35,7 +36,7 @@ use std::default::Default;
 use std::ops::Range;
 use style::attr::AttrValue;
 use style::element_state::ElementState;
-use textinput::{Direction, KeyReaction, Lines, Selection, SelectionDirection, TextInput};
+use textinput::{Direction, KeyReaction, Lines, SelectionDirection, TextInput};
 
 #[dom_struct]
 pub struct HTMLTextAreaElement {
@@ -44,7 +45,7 @@ pub struct HTMLTextAreaElement {
     textinput: DomRefCell<TextInput<ScriptToConstellationChan>>,
     placeholder: DomRefCell<DOMString>,
     // https://html.spec.whatwg.org/multipage/#concept-textarea-dirty
-    value_changed: Cell<bool>,
+    value_dirty: Cell<bool>,
     form_owner: MutNullableDom<HTMLFormElement>,
 }
 
@@ -81,7 +82,7 @@ impl LayoutHTMLTextAreaElementHelpers for LayoutDom<HTMLTextAreaElement> {
             return None;
         }
         let textinput = (*self.unsafe_get()).textinput.borrow_for_layout();
-        Some(textinput.get_absolute_selection_range())
+        Some(textinput.sorted_selection_offsets_range())
     }
 
     #[allow(unsafe_code)]
@@ -122,7 +123,7 @@ impl HTMLTextAreaElement {
             placeholder: DomRefCell::new(DOMString::new()),
             textinput: DomRefCell::new(TextInput::new(
                     Lines::Multiple, DOMString::new(), chan, None, None, SelectionDirection::None)),
-            value_changed: Cell::new(false),
+            value_dirty: Cell::new(false),
             form_owner: Default::default(),
         }
     }
@@ -152,6 +153,14 @@ impl TextControl for HTMLTextAreaElement {
     fn selection_api_applies(&self) -> bool {
         true
     }
+
+    fn has_selectable_text(&self) -> bool {
+        true
+    }
+
+    fn set_dirty_value_flag(&self, value: bool) {
+        self.value_dirty.set(value)
+    }
 }
 
 impl HTMLTextAreaElementMethods for HTMLTextAreaElement {
@@ -227,7 +236,7 @@ impl HTMLTextAreaElementMethods for HTMLTextAreaElement {
 
         // if the element's dirty value flag is false, then the element's
         // raw value must be set to the value of the element's textContent IDL attribute
-        if !self.value_changed.get() {
+        if !self.value_dirty.get() {
             self.reset();
         }
     }
@@ -243,19 +252,19 @@ impl HTMLTextAreaElementMethods for HTMLTextAreaElement {
 
         // Step 1
         let old_value = textinput.get_content();
-        let old_selection = textinput.selection_begin;
+        let old_selection = textinput.selection_origin;
 
         // Step 2
         textinput.set_content(value);
 
         // Step 3
-        self.value_changed.set(true);
+        self.value_dirty.set(true);
 
         if old_value != textinput.get_content() {
             // Step 4
-            textinput.adjust_horizontal_to_limit(Direction::Forward, Selection::NotSelected);
+            textinput.clear_selection_to_limit(Direction::Forward);
         } else {
-            textinput.selection_begin = old_selection;
+            textinput.selection_origin = old_selection;
         }
 
         self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
@@ -266,6 +275,11 @@ impl HTMLTextAreaElementMethods for HTMLTextAreaElement {
         self.upcast::<HTMLElement>().labels()
     }
 
+    // https://html.spec.whatwg.org/multipage/#dom-textarea/input-select
+    fn Select(&self) {
+        self.dom_select(); // defined in TextControl trait
+    }
+
     // https://html.spec.whatwg.org/multipage/#dom-textarea/input-selectionstart
     fn GetSelectionStart(&self) -> Option<u32> {
         self.get_dom_selection_start()
@@ -300,6 +314,19 @@ impl HTMLTextAreaElementMethods for HTMLTextAreaElement {
     fn SetSelectionRange(&self, start: u32, end: u32, direction: Option<DOMString>) -> ErrorResult {
         self.set_dom_selection_range(start, end, direction)
     }
+
+    // https://html.spec.whatwg.org/multipage/#dom-textarea/input-setrangetext
+    fn SetRangeText(&self, replacement: DOMString) -> ErrorResult {
+        // defined in TextControl trait
+        self.set_dom_range_text(replacement, None, None, Default::default())
+    }
+
+    // https://html.spec.whatwg.org/multipage/#dom-textarea/input-setrangetext
+    fn SetRangeText_(&self, replacement: DOMString, start: u32, end: u32,
+                     selection_mode: SelectionMode) -> ErrorResult {
+        // defined in TextControl trait
+        self.set_dom_range_text(replacement, Some(start), Some(end), selection_mode)
+    }
 }
 
 
@@ -307,7 +334,7 @@ impl HTMLTextAreaElement {
     pub fn reset(&self) {
         // https://html.spec.whatwg.org/multipage/#the-textarea-element:concept-form-reset-control
         self.SetValue(self.DefaultValue());
-        self.value_changed.set(false);
+        self.value_dirty.set(false);
     }
 }
 
@@ -400,7 +427,7 @@ impl VirtualMethods for HTMLTextAreaElement {
         if let Some(ref s) = self.super_type() {
             s.children_changed(mutation);
         }
-        if !self.value_changed.get() {
+        if !self.value_dirty.get() {
             self.reset();
         }
     }
@@ -423,7 +450,7 @@ impl VirtualMethods for HTMLTextAreaElement {
                 match action {
                     KeyReaction::TriggerDefaultAction => (),
                     KeyReaction::DispatchInput => {
-                        self.value_changed.set(true);
+                        self.value_dirty.set(true);
                         self.update_placeholder_shown_state();
                         self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
                         event.mark_as_handled();
diff --git a/servo/components/script/dom/textcontrol.rs b/servo/components/script/dom/textcontrol.rs
index 9143c2bda233..342b081621d2 100644
--- a/servo/components/script/dom/textcontrol.rs
+++ b/servo/components/script/dom/textcontrol.rs
@@ -3,6 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 use dom::bindings::cell::DomRefCell;
+use dom::bindings::codegen::Bindings::HTMLFormElementBinding::SelectionMode;
 use dom::bindings::conversions::DerivedFrom;
 use dom::bindings::error::{Error, ErrorResult};
 use dom::bindings::str::DOMString;
@@ -10,11 +11,24 @@ use dom::event::{EventBubbles, EventCancelable};
 use dom::eventtarget::EventTarget;
 use dom::node::{Node, NodeDamage, window_from_node};
 use script_traits::ScriptToConstellationChan;
-use textinput::{SelectionDirection, TextInput};
+use textinput::{SelectionDirection, SelectionState, TextInput};
 
 pub trait TextControl: DerivedFrom<EventTarget> + DerivedFrom<Node> {
     fn textinput(&self) -> &DomRefCell<TextInput<ScriptToConstellationChan>>;
     fn selection_api_applies(&self) -> bool;
+    fn has_selectable_text(&self) -> bool;
+    fn set_dirty_value_flag(&self, value: bool);
+
+    // https://html.spec.whatwg.org/multipage/#dom-textarea/input-select
+    fn dom_select(&self) {
+        // Step 1
+        if !self.has_selectable_text() {
+            return;
+        }
+
+        // Step 2
+        self.set_selection_range(Some(0), Some(u32::max_value()), None, None);
+    }
 
     // https://html.spec.whatwg.org/multipage/#dom-textarea/input-selectionstart
     fn get_dom_selection_start(&self) -> Option<u32> {
@@ -45,7 +59,7 @@ pub trait TextControl: DerivedFrom<EventTarget> + DerivedFrom<Node> {
         }
 
         // Step 4
-        self.set_selection_range(start, Some(end), Some(self.selection_direction()));
+        self.set_selection_range(start, Some(end), Some(self.selection_direction()), None);
         Ok(())
     }
 
@@ -68,7 +82,7 @@ pub trait TextControl: DerivedFrom<EventTarget> + DerivedFrom<Node> {
         }
 
         // Step 2
-        self.set_selection_range(Some(self.selection_start()), end, Some(self.selection_direction()));
+        self.set_selection_range(Some(self.selection_start()), end, Some(self.selection_direction()), None);
         Ok(())
     }
 
@@ -93,7 +107,8 @@ pub trait TextControl: DerivedFrom<EventTarget> + DerivedFrom<Node> {
         self.set_selection_range(
             Some(self.selection_start()),
             Some(self.selection_end()),
-            direction.map(|d| SelectionDirection::from(d))
+            direction.map(|d| SelectionDirection::from(d)),
+            None
         );
         Ok(())
     }
@@ -106,16 +121,125 @@ pub trait TextControl: DerivedFrom<EventTarget> + DerivedFrom<Node> {
         }
 
         // Step 2
-        self.set_selection_range(Some(start), Some(end), direction.map(|d| SelectionDirection::from(d)));
+        self.set_selection_range(Some(start), Some(end), direction.map(|d| SelectionDirection::from(d)), None);
+        Ok(())
+    }
+
+    // https://html.spec.whatwg.org/multipage/#dom-textarea/input-setrangetext
+    fn set_dom_range_text(&self, replacement: DOMString, start: Option<u32>, end: Option<u32>,
+                          selection_mode: SelectionMode) -> ErrorResult {
+        // Step 1
+        if !self.selection_api_applies() {
+            return Err(Error::InvalidState);
+        }
+
+        // Step 2
+        self.set_dirty_value_flag(true);
+
+        // Step 3
+        let mut start = start.unwrap_or_else(|| self.selection_start());
+        let mut end = end.unwrap_or_else(|| self.selection_end());
+
+        // Step 4
+        if start > end {
+            return Err(Error::IndexSize);
+        }
+
+        // Save the original selection state to later pass to set_selection_range, because we will
+        // change the selection state in order to replace the text in the range.
+        let original_selection_state = self.textinput().borrow().selection_state();
+
+        let content_length = self.textinput().borrow().len() as u32;
+
+        // Step 5
+        if start > content_length {
+            start = content_length;
+        }
+
+        // Step 6
+        if end > content_length {
+            end = content_length;
+        }
+
+        // Step 7
+        let mut selection_start = self.selection_start();
+
+        // Step 8
+        let mut selection_end = self.selection_end();
+
+        // Step 11
+        // Must come before the textinput.replace_selection() call, as replacement gets moved in
+        // that call.
+        let new_length = replacement.len() as u32;
+
+        {
+            let mut textinput = self.textinput().borrow_mut();
+
+            // Steps 9-10
+            textinput.set_selection_range(start, end, SelectionDirection::None);
+            textinput.replace_selection(replacement);
+        }
+
+        // Step 12
+        let new_end = start + new_length;
+
+        // Step 13
+        match selection_mode {
+            SelectionMode::Select => {
+                selection_start = start;
+                selection_end = new_end;
+            },
+
+            SelectionMode::Start => {
+                selection_start = start;
+                selection_end = start;
+            },
+
+            SelectionMode::End => {
+                selection_start = new_end;
+                selection_end = new_end;
+            },
+
+            SelectionMode::Preserve => {
+                // Sub-step 1
+                let old_length = end - start;
+
+                // Sub-step 2
+                let delta = (new_length as isize) - (old_length as isize);
+
+                // Sub-step 3
+                if selection_start > end {
+                    selection_start = ((selection_start as isize) + delta) as u32;
+                } else if selection_start > start {
+                    selection_start = start;
+                }
+
+                // Sub-step 4
+                if selection_end > end {
+                    selection_end = ((selection_end as isize) + delta) as u32;
+                } else if selection_end > start {
+                    selection_end = new_end;
+                }
+            },
+        }
+
+        // Step 14
+        self.set_selection_range(
+            Some(selection_start),
+            Some(selection_end),
+            None,
+            Some(original_selection_state)
+        );
+
         Ok(())
     }
 
     fn selection_start(&self) -> u32 {
-        self.textinput().borrow().get_selection_start()
+        self.textinput().borrow().selection_start_offset() as u32
     }
 
     fn selection_end(&self) -> u32 {
-        self.textinput().borrow().get_absolute_insertion_point() as u32
+        self.textinput().borrow().selection_end_offset() as u32
     }
 
     fn selection_direction(&self) -> SelectionDirection {
@@ -123,7 +247,11 @@ pub trait TextControl: DerivedFrom<EventTarget> + DerivedFrom<Node> {
     }
 
     // https://html.spec.whatwg.org/multipage/#set-the-selection-range
-    fn set_selection_range(&self, start: Option<u32>, end: Option<u32>, direction: Option<SelectionDirection>) {
+    fn set_selection_range(&self, start: Option<u32>, end: Option<u32>, direction: Option<SelectionDirection>,
+                           original_selection_state: Option<SelectionState>) {
+        let mut textinput = self.textinput().borrow_mut();
+        let original_selection_state = original_selection_state.unwrap_or_else(|| textinput.selection_state());
+
         // Step 1
         let start = start.unwrap_or(0);
 
@@ -131,16 +259,18 @@ pub trait TextControl: DerivedFrom<EventTarget> + DerivedFrom<Node> {
         let end = end.unwrap_or(0);
 
         // Steps 3-5
-        self.textinput().borrow_mut().set_selection_range(start, end, direction.unwrap_or(SelectionDirection::None));
+        textinput.set_selection_range(start, end, direction.unwrap_or(SelectionDirection::None));
 
         // Step 6
-        let window = window_from_node(self);
-        let _ = window.user_interaction_task_source().queue_event(
-            &self.upcast::<EventTarget>(),
-            atom!("select"),
-            EventBubbles::Bubbles,
-            EventCancelable::NotCancelable,
-            &window);
+        if textinput.selection_state() != original_selection_state {
+            let window = window_from_node(self);
+            window.user_interaction_task_source().queue_event(
+                &self.upcast::<EventTarget>(),
+                atom!("select"),
+                EventBubbles::Bubbles,
+                EventCancelable::NotCancelable,
+                &window);
+        }
 
         self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
     }
diff --git a/servo/components/script/dom/webidls/HTMLFormElement.webidl b/servo/components/script/dom/webidls/HTMLFormElement.webidl
index b4a2504fd60f..1514e38b6f58 100644
--- a/servo/components/script/dom/webidls/HTMLFormElement.webidl
+++ b/servo/components/script/dom/webidls/HTMLFormElement.webidl
@@ -35,3 +35,11 @@ interface HTMLFormElement : HTMLElement {
   //boolean checkValidity();
   //boolean reportValidity();
 };
+
+// https://html.spec.whatwg.org/multipage/#selectionmode
+enum SelectionMode {
+  "preserve", // default
+  "select",
+  "start",
+  "end"
+};
diff --git a/servo/components/script/dom/webidls/HTMLInputElement.webidl b/servo/components/script/dom/webidls/HTMLInputElement.webidl
index 93a5a7f108b4..e76ca054a6f1 100644
--- a/servo/components/script/dom/webidls/HTMLInputElement.webidl
+++ b/servo/components/script/dom/webidls/HTMLInputElement.webidl
@@ -89,16 +89,18 @@ interface HTMLInputElement : HTMLElement {
 
   readonly attribute NodeList labels;
 
-  //void select();
+  void select();
   [SetterThrows]
            attribute unsigned long? selectionStart;
   [SetterThrows]
            attribute unsigned long? selectionEnd;
   [SetterThrows]
            attribute DOMString? selectionDirection;
-  //void setRangeText(DOMString replacement);
-  //void setRangeText(DOMString replacement, unsigned long start, unsigned long end,
-  //                  optional SelectionMode selectionMode = "preserve");
+  [Throws]
+           void setRangeText(DOMString replacement);
+  [Throws]
+           void setRangeText(DOMString replacement, unsigned long start, unsigned long end,
+                             optional SelectionMode selectionMode = "preserve");
   [Throws]
            void setSelectionRange(unsigned long start, unsigned long end, optional DOMString direction);
 
diff --git a/servo/components/script/dom/webidls/HTMLTextAreaElement.webidl b/servo/components/script/dom/webidls/HTMLTextAreaElement.webidl
index f0e8a0be1181..3ff20242bfee 100644
--- a/servo/components/script/dom/webidls/HTMLTextAreaElement.webidl
+++ b/servo/components/script/dom/webidls/HTMLTextAreaElement.webidl
@@ -50,16 +50,18 @@ interface HTMLTextAreaElement : HTMLElement {
 
   readonly attribute NodeList labels;
 
-  // void select();
+  void select();
   [SetterThrows]
            attribute unsigned long? selectionStart;
   [SetterThrows]
            attribute unsigned long? selectionEnd;
   [SetterThrows]
            attribute DOMString? selectionDirection;
-  // void setRangeText(DOMString replacement);
-  // void setRangeText(DOMString replacement, unsigned long start, unsigned long end,
-  //                   optional SelectionMode selectionMode = "preserve");
+  [Throws]
+           void setRangeText(DOMString replacement);
+  [Throws]
+           void setRangeText(DOMString replacement, unsigned long start, unsigned long end,
+                             optional SelectionMode selectionMode = "preserve");
   [Throws]
            void setSelectionRange(unsigned long start, unsigned long end, optional DOMString direction);
 };
diff --git a/servo/components/script/textinput.rs b/servo/components/script/textinput.rs
index 0fe0024a116b..937ab38ed5d6 100644
--- a/servo/components/script/textinput.rs
+++ b/servo/components/script/textinput.rs
@@ -48,7 +48,7 @@ impl From<SelectionDirection> for DOMString {
     }
 }
 
-#[derive(Clone, Copy, JSTraceable, MallocSizeOf, PartialEq)]
+#[derive(Clone, Copy, Debug, JSTraceable, MallocSizeOf, PartialEq, PartialOrd)]
 pub struct TextPoint {
     /// 0-based line number
     pub line: usize,
@@ -56,6 +56,13 @@ pub struct TextPoint {
     pub index: usize,
 }
 
+#[derive(Clone, Copy, PartialEq)]
+pub struct SelectionState {
+    start: TextPoint,
+    end: TextPoint,
+    direction: SelectionDirection,
+}
+
 /// Encapsulated state for handling keyboard input in a single or multiline text input control.
 #[derive(JSTraceable, MallocSizeOf)]
 pub struct TextInput<T: ClipboardProvider> {
@@ -63,8 +70,9 @@ pub struct TextInput<T: ClipboardProvider> {
     lines: Vec<DOMString>,
     /// Current cursor input point
     pub edit_point: TextPoint,
-    /// Beginning of selection range with edit_point as end that can span multiple lines.
-    pub selection_begin: Option<TextPoint>,
+    /// The current selection goes from the selection_origin until the edit_point. Note that the
+    /// selection_origin may be after the edit_point, in the case of a backward selection.
+    pub selection_origin: Option<TextPoint>,
     /// Is this a multiline input?
     multiline: bool,
     #[ignore_malloc_size_of = "Can't easily measure this generic type"]
@@ -156,7 +164,7 @@ impl<T: ClipboardProvider> TextInput<T> {
         let mut i = TextInput {
             lines: vec!(),
             edit_point: Default::default(),
-            selection_begin: None,
+            selection_origin: None,
             multiline: lines == Lines::Multiple,
             clipboard_provider: clipboard_provider,
             max_length: max_length,
@@ -169,7 +177,7 @@ impl<T: ClipboardProvider> TextInput<T> {
 
     /// Remove a character at the current editing point
     pub fn delete_char(&mut self, dir: Direction) {
-        if self.selection_begin.is_none() || self.selection_begin == Some(self.edit_point) {
+        if self.selection_origin.is_none() || self.selection_origin == Some(self.edit_point) {
             self.adjust_horizontal_by_one(dir, Selection::Selected);
         }
         self.replace_selection(DOMString::new());
@@ -182,46 +190,93 @@ impl<T: ClipboardProvider> TextInput<T> {
 
     /// Insert a string at the current editing point
     pub fn insert_string<S: Into<String>>(&mut self, s: S) {
-        if self.selection_begin.is_none() {
-            self.selection_begin = Some(self.edit_point);
+        if self.selection_origin.is_none() {
+            self.selection_origin = Some(self.edit_point);
         }
         self.replace_selection(DOMString::from(s.into()));
     }
 
-    pub fn get_sorted_selection(&self) -> Option<(TextPoint, TextPoint)> {
-        self.selection_begin.map(|begin| {
-            let end = self.edit_point;
-
-            if begin.line < end.line || (begin.line == end.line && begin.index < end.index) {
-                (begin, end)
-            } else {
-                (end, begin)
-            }
-        })
+    /// The selection origin, or the edit point if there is no selection. Note that the selection
+    /// origin may be after the edit point, in the case of a backward selection.
+    pub fn selection_origin_or_edit_point(&self) -> TextPoint {
+        self.selection_origin.unwrap_or(self.edit_point)
     }
 
-    // Check that the selection is valid.
-    fn assert_ok_selection(&self) {
-        if let Some(begin) = self.selection_begin {
-            debug_assert!(begin.line < self.lines.len());
-            debug_assert!(begin.index <= self.lines[begin.line].len());
+    /// The start of the selection (or the edit point, if there is no selection). Always less than
+    /// or equal to selection_end(), regardless of the selection direction.
+    pub fn selection_start(&self) -> TextPoint {
+        match self.selection_direction {
+            SelectionDirection::None | SelectionDirection::Forward => self.selection_origin_or_edit_point(),
+            SelectionDirection::Backward => self.edit_point,
         }
-        debug_assert!(self.edit_point.line < self.lines.len());
-        debug_assert!(self.edit_point.index <= self.lines[self.edit_point.line].len());
+    }
+
+    /// The UTF-8 byte offset of the selection_start()
+    pub fn selection_start_offset(&self) -> usize {
+        self.text_point_to_offset(&self.selection_start())
+    }
+
+    /// The end of the selection (or the edit point, if there is no selection). Always greater
+    /// than or equal to selection_start(), regardless of the selection direction.
+    pub fn selection_end(&self) -> TextPoint {
+        match self.selection_direction {
+            SelectionDirection::None | SelectionDirection::Forward => self.edit_point,
+            SelectionDirection::Backward => self.selection_origin_or_edit_point(),
+        }
+    }
+
+    /// The UTF-8 byte offset of the selection_end()
+    pub fn selection_end_offset(&self) -> usize {
+        self.text_point_to_offset(&self.selection_end())
+    }
+
+    /// Whether or not there is an active selection (the selection may be zero-length)
+    #[inline]
+    pub fn has_selection(&self) -> bool {
+        self.selection_origin.is_some()
+    }
+
+    /// Returns a tuple of (start, end) giving the bounds of the current selection. start is always
+    /// less than or equal to end.
+    pub fn sorted_selection_bounds(&self) -> (TextPoint, TextPoint) {
+        (self.selection_start(), self.selection_end())
     }
 
     /// Return the selection range as UTF-8 byte offsets from the start of the content.
     ///
-    /// If there is no selection, returns an empty range at the insertion point.
-    pub fn get_absolute_selection_range(&self) -> Range<usize> {
-        match self.get_sorted_selection() {
-            Some((begin, end)) => self.get_absolute_point_for_text_point(&begin) ..
-                                  self.get_absolute_point_for_text_point(&end),
-            None => {
-                let insertion_point = self.get_absolute_insertion_point();
-                insertion_point .. insertion_point
+    /// If there is no selection, returns an empty range at the edit point.
+    pub fn sorted_selection_offsets_range(&self) -> Range<usize> {
+        self.selection_start_offset() .. self.selection_end_offset()
+    }
+
+    /// The state of the current selection. Can be used to compare whether selection state has changed.
+    pub fn selection_state(&self) -> SelectionState {
+        SelectionState {
+            start: self.selection_start(),
+            end: self.selection_end(),
+            direction: self.selection_direction,
+        }
+    }
+
+    // Check that the selection is valid.
+    fn assert_ok_selection(&self) {
+        if let Some(begin) = self.selection_origin {
+            debug_assert!(begin.line < self.lines.len());
+            debug_assert!(begin.index <= self.lines[begin.line].len());
+
+            match self.selection_direction {
+                SelectionDirection::None | SelectionDirection::Forward => {
+                    debug_assert!(begin <= self.edit_point)
+                },
+
+                SelectionDirection::Backward => {
+                    debug_assert!(self.edit_point <= begin)
+                },
             }
         }
+
+        debug_assert!(self.edit_point.line < self.lines.len());
+        debug_assert!(self.edit_point.index <= self.lines[self.edit_point.line].len());
     }
 
     pub fn get_selection_text(&self) -> Option<String> {
@@ -242,78 +297,83 @@ impl<T: ClipboardProvider> TextInput<T> {
     ///
     /// The accumulator `acc` can be mutated by the callback, and will be returned at the end.
     fn fold_selection_slices<B, F: FnMut(&mut B, &str)>(&self, mut acc: B, mut f: F) -> B {
-        match self.get_sorted_selection() {
-            Some((begin, end)) if begin.line == end.line => {
-                f(&mut acc, &self.lines[begin.line][begin.index..end.index])
-            }
-            Some((begin, end)) => {
-                f(&mut acc, &self.lines[begin.line][begin.index..]);
-                for line in &self.lines[begin.line + 1 .. end.line] {
+        if self.has_selection() {
+            let (start, end) = self.sorted_selection_bounds();
+
+            if start.line == end.line {
+                f(&mut acc, &self.lines[start.line][start.index..end.index])
+            } else {
+                f(&mut acc, &self.lines[start.line][start.index..]);
+                for line in &self.lines[start.line + 1 .. end.line] {
                     f(&mut acc, "\n");
                     f(&mut acc, line);
                 }
                 f(&mut acc, "\n");
                 f(&mut acc, &self.lines[end.line][..end.index])
             }
-            None => {}
         }
+
         acc
     }
 
     pub fn replace_selection(&mut self, insert: DOMString) {
-        if let Some((begin, end)) = self.get_sorted_selection() {
-            let allowed_to_insert_count = if let Some(max_length) = self.max_length {
-                let len_after_selection_replaced = self.utf16_len() - self.selection_utf16_len();
-                if len_after_selection_replaced >= max_length {
-                    // If, after deleting the selection, the len is still greater than the max
-                    // length, then don't delete/insert anything
-                    return
-                }
-
-                max_length - len_after_selection_replaced
-            } else {
-                usize::MAX
-            };
-
-            let last_char_index = len_of_first_n_code_units(&*insert, allowed_to_insert_count);
-            let chars_to_insert = &insert[..last_char_index];
-
-            self.clear_selection();
-
-            let new_lines = {
-                let prefix = &self.lines[begin.line][..begin.index];
-                let suffix = &self.lines[end.line][end.index..];
-                let lines_prefix = &self.lines[..begin.line];
-                let lines_suffix = &self.lines[end.line + 1..];
-
-                let mut insert_lines = if self.multiline {
-                    chars_to_insert.split('\n').map(|s| DOMString::from(s)).collect()
-                } else {
-                    vec!(DOMString::from(chars_to_insert))
-                };
-
-                // FIXME(ajeffrey): effecient append for DOMStrings
-                let mut new_line = prefix.to_owned();
-
-                new_line.push_str(&insert_lines[0]);
-                insert_lines[0] = DOMString::from(new_line);
-
-                let last_insert_lines_index = insert_lines.len() - 1;
-                self.edit_point.index = insert_lines[last_insert_lines_index].len();
-                self.edit_point.line = begin.line + last_insert_lines_index;
-
-                // FIXME(ajeffrey): effecient append for DOMStrings
-                insert_lines[last_insert_lines_index].push_str(suffix);
-
-                let mut new_lines = vec!();
-                new_lines.extend_from_slice(lines_prefix);
-                new_lines.extend_from_slice(&insert_lines);
-                new_lines.extend_from_slice(lines_suffix);
-                new_lines
-            };
-
-            self.lines = new_lines;
+        if !self.has_selection() {
+            return
         }
+
+        let (start, end) = self.sorted_selection_bounds();
+
+        let allowed_to_insert_count = if let Some(max_length) = self.max_length {
+            let len_after_selection_replaced = self.utf16_len() - self.selection_utf16_len();
+            if len_after_selection_replaced >= max_length {
+                // If, after deleting the selection, the len is still greater than the max
+                // length, then don't delete/insert anything
+                return
+            }
+
+            max_length - len_after_selection_replaced
+        } else {
+            usize::MAX
+        };
+
+        let last_char_index = len_of_first_n_code_units(&*insert, allowed_to_insert_count);
+        let chars_to_insert = &insert[..last_char_index];
+
+        self.clear_selection();
+
+        let new_lines = {
+            let prefix = &self.lines[start.line][..start.index];
+            let suffix = &self.lines[end.line][end.index..];
+            let lines_prefix = &self.lines[..start.line];
+            let lines_suffix = &self.lines[end.line + 1..];
+
+            let mut insert_lines = if self.multiline {
+                chars_to_insert.split('\n').map(|s| DOMString::from(s)).collect()
+            } else {
+                vec!(DOMString::from(chars_to_insert))
+            };
+
+            // FIXME(ajeffrey): effecient append for DOMStrings
+            let mut new_line = prefix.to_owned();
+
+            new_line.push_str(&insert_lines[0]);
+            insert_lines[0] = DOMString::from(new_line);
+
+            let last_insert_lines_index = insert_lines.len() - 1;
+            self.edit_point.index = insert_lines[last_insert_lines_index].len();
+            self.edit_point.line = start.line + last_insert_lines_index;
+
+            // FIXME(ajeffrey): effecient append for DOMStrings
+            insert_lines[last_insert_lines_index].push_str(suffix);
+
+            let mut new_lines = vec!();
+            new_lines.extend_from_slice(lines_prefix);
+            new_lines.extend_from_slice(&insert_lines);
+            new_lines.extend_from_slice(lines_suffix);
+            new_lines
+        };
+
+        self.lines = new_lines;
         self.assert_ok_selection();
     }
 
@@ -330,8 +390,8 @@ impl<T: ClipboardProvider> TextInput<T> {
         }
 
         if select == Selection::Selected {
-            if self.selection_begin.is_none() {
-                self.selection_begin = Some(self.edit_point);
+            if self.selection_origin.is_none() {
+                self.selection_origin = Some(self.edit_point);
             }
         } else {
             self.clear_selection();
@@ -398,14 +458,19 @@ impl<T: ClipboardProvider> TextInput<T> {
     fn adjust_selection_for_horizontal_change(&mut self, adjust: Direction, select: Selection)
                                               -> bool {
         if select == Selection::Selected {
-            if self.selection_begin.is_none() {
-                self.selection_begin = Some(self.edit_point);
+            if self.selection_origin.is_none() {
+                self.selection_origin = Some(self.edit_point);
             }
+
+            self.selection_direction = match adjust {
+                Direction::Backward => SelectionDirection::Backward,
+                Direction::Forward => SelectionDirection::Forward,
+            };
         } else {
-            if let Some((begin, end)) = self.get_sorted_selection() {
+            if self.has_selection() {
                 self.edit_point = match adjust {
-                    Direction::Backward => begin,
-                    Direction::Forward => end,
+                    Direction::Backward => self.selection_start(),
+                    Direction::Forward => self.selection_end(),
                 };
                 self.clear_selection();
                 return true
@@ -451,7 +516,7 @@ impl<T: ClipboardProvider> TextInput<T> {
 
     /// Select all text in the input control.
     pub fn select_all(&mut self) {
-        self.selection_begin = Some(TextPoint {
+        self.selection_origin = Some(TextPoint {
             line: 0,
             index: 0,
         });
@@ -463,7 +528,14 @@ impl<T: ClipboardProvider> TextInput<T> {
 
     /// Remove the current selection.
     pub fn clear_selection(&mut self) {
-        self.selection_begin = None;
+        self.selection_origin = None;
+        self.selection_direction = SelectionDirection::None;
+    }
+
+    /// Remove the current selection and set the edit point to the end of the content.
+    pub fn clear_selection_to_limit(&mut self, direction: Direction) {
+        self.clear_selection();
+        self.adjust_horizontal_to_limit(direction, Selection::NotSelected);
     }
 
     pub fn adjust_horizontal_by_word(&mut self, direction: Direction, select: Selection) {
@@ -780,17 +852,12 @@ impl<T: ClipboardProvider> TextInput<T> {
         };
         self.edit_point.line = min(self.edit_point.line, self.lines.len() - 1);
         self.edit_point.index = min(self.edit_point.index, self.current_line_length());
-        self.selection_begin = None;
+        self.selection_origin = None;
         self.assert_ok_selection();
     }
 
-    /// Get the insertion point as a byte offset from the start of the content.
-    pub fn get_absolute_insertion_point(&self) -> usize {
-        self.get_absolute_point_for_text_point(&self.edit_point)
-    }
-
     /// Convert a TextPoint into a byte offset from the start of the content.
-    pub fn get_absolute_point_for_text_point(&self, text_point: &TextPoint) -> usize {
+    fn text_point_to_offset(&self, text_point: &TextPoint) -> usize {
         self.lines.iter().enumerate().fold(0, |acc, (i, val)| {
             if i < text_point.line {
                 acc + val.len() + 1 // +1 for the \n
@@ -801,7 +868,7 @@ impl<T: ClipboardProvider> TextInput<T> {
     }
 
     /// Convert a byte offset from the start of the content into a TextPoint.
-    pub fn get_text_point_for_absolute_point(&self, abs_point: usize) -> TextPoint {
+    fn offset_to_text_point(&self, abs_point: usize) -> TextPoint {
         let mut index = abs_point;
         let mut line = 0;
 
@@ -842,28 +909,17 @@ impl<T: ClipboardProvider> TextInput<T> {
         match direction {
             SelectionDirection::None |
             SelectionDirection::Forward => {
-                self.selection_begin = Some(self.get_text_point_for_absolute_point(start));
-                self.edit_point = self.get_text_point_for_absolute_point(end);
+                self.selection_origin = Some(self.offset_to_text_point(start));
+                self.edit_point = self.offset_to_text_point(end);
             },
             SelectionDirection::Backward => {
-                self.selection_begin = Some(self.get_text_point_for_absolute_point(end));
-                self.edit_point = self.get_text_point_for_absolute_point(start);
+                self.selection_origin = Some(self.offset_to_text_point(end));
+                self.edit_point = self.offset_to_text_point(start);
             }
         }
         self.assert_ok_selection();
     }
 
-    pub fn get_selection_start(&self) -> u32 {
-        let selection_start = match self.selection_begin {
-            Some(selection_begin_point) => {
-                self.get_absolute_point_for_text_point(&selection_begin_point)
-            },
-            None => self.get_absolute_insertion_point()
-        };
-
-        selection_start as u32
-    }
-
     pub fn set_edit_point_index(&mut self, index: usize) {
         let byte_size = self.lines[self.edit_point.line]
             .graphemes(true)
diff --git a/servo/tests/unit/script/textinput.rs b/servo/tests/unit/script/textinput.rs
index 5b98dc934c69..2167efb0b6d1 100644
--- a/servo/tests/unit/script/textinput.rs
+++ b/servo/tests/unit/script/textinput.rs
@@ -222,7 +222,7 @@ fn test_textinput_delete_char() {
     let mut textinput = text_input(Lines::Single, "abcdefg");
     textinput.adjust_horizontal(2, Selection::NotSelected);
     // Set an empty selection range.
-    textinput.selection_begin = Some(textinput.edit_point);
+    textinput.selection_origin = Some(textinput.edit_point);
     textinput.delete_char(Direction::Backward);
     assert_eq!(textinput.get_content(), "acdefg");
 }
@@ -252,15 +252,15 @@ fn test_textinput_get_sorted_selection() {
     let mut textinput = text_input(Lines::Single, "abcdefg");
     textinput.adjust_horizontal(2, Selection::NotSelected);
     textinput.adjust_horizontal(2, Selection::Selected);
-    let (begin, end) = textinput.get_sorted_selection().unwrap();
-    assert_eq!(begin.index, 2);
+    let (start, end) = textinput.sorted_selection_bounds();
+    assert_eq!(start.index, 2);
     assert_eq!(end.index, 4);
 
     textinput.clear_selection();
 
     textinput.adjust_horizontal(-2, Selection::Selected);
-    let (begin, end) = textinput.get_sorted_selection().unwrap();
-    assert_eq!(begin.index, 2);
+    let (start, end) = textinput.sorted_selection_bounds();
+    assert_eq!(start.index, 2);
     assert_eq!(end.index, 4);
 }
 
@@ -588,18 +588,18 @@ fn test_textinput_set_selection_with_direction() {
     assert_eq!(textinput.edit_point.index, 6);
     assert_eq!(textinput.selection_direction, SelectionDirection::Forward);
 
-    assert!(textinput.selection_begin.is_some());
-    assert_eq!(textinput.selection_begin.unwrap().line, 0);
-    assert_eq!(textinput.selection_begin.unwrap().index, 2);
+    assert!(textinput.selection_origin.is_some());
+    assert_eq!(textinput.selection_origin.unwrap().line, 0);
+    assert_eq!(textinput.selection_origin.unwrap().index, 2);
 
     textinput.set_selection_range(2, 6, SelectionDirection::Backward);
     assert_eq!(textinput.edit_point.line, 0);
     assert_eq!(textinput.edit_point.index, 2);
     assert_eq!(textinput.selection_direction, SelectionDirection::Backward);
 
-    assert!(textinput.selection_begin.is_some());
-    assert_eq!(textinput.selection_begin.unwrap().line, 0);
-    assert_eq!(textinput.selection_begin.unwrap().index, 6);
+    assert!(textinput.selection_origin.is_some());
+    assert_eq!(textinput.selection_origin.unwrap().line, 0);
+    assert_eq!(textinput.selection_origin.unwrap().index, 6);
 }
 
 #[test]
@@ -611,3 +611,22 @@ fn test_textinput_unicode_handling() {
     textinput.set_edit_point_index(4);
     assert_eq!(textinput.edit_point.index, 8);
 }
+
+#[test]
+fn test_selection_bounds() {
+    let mut textinput = text_input(Lines::Single, "abcdef");
+
+    textinput.set_selection_range(2, 5, SelectionDirection::Forward);
+    assert_eq!(TextPoint { line: 0, index: 2 }, textinput.selection_origin_or_edit_point());
+    assert_eq!(TextPoint { line: 0, index: 2 }, textinput.selection_start());
+    assert_eq!(TextPoint { line: 0, index: 5 }, textinput.selection_end());
+    assert_eq!(2, textinput.selection_start_offset());
+    assert_eq!(5, textinput.selection_end_offset());
+
+    textinput.set_selection_range(3, 6, SelectionDirection::Backward);
+    assert_eq!(TextPoint { line: 0, index: 6 }, textinput.selection_origin_or_edit_point());
+    assert_eq!(TextPoint { line: 0, index: 3 }, textinput.selection_start());
+    assert_eq!(TextPoint { line: 0, index: 6 }, textinput.selection_end());
+    assert_eq!(3, textinput.selection_start_offset());
+    assert_eq!(6, textinput.selection_end_offset());
+}

From 47baed1a64958e5e977942f652d046de945df167 Mon Sep 17 00:00:00 2001
From: Hiroyuki Ikezoe <hikezoe@mozilla.com>
Date: Fri, 26 Jan 2018 20:10:07 +0900
Subject: [PATCH 12/46] Bug 1433336 - Drop unused SubDocEnumArgs in
 nsDocument.cpp. r=smaug

MozReview-Commit-ID: HuVOdxGZIuN

--HG--
extra : rebase_source : c5709c71a7ac6d2d4e174b4806294d0928c99197
---
 dom/base/nsDocument.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/dom/base/nsDocument.cpp b/dom/base/nsDocument.cpp
index 2d2039f1f80a..b589811f1822 100644
--- a/dom/base/nsDocument.cpp
+++ b/dom/base/nsDocument.cpp
@@ -8580,12 +8580,6 @@ nsDocument::Sanitize()
   }
 }
 
-struct SubDocEnumArgs
-{
-  nsIDocument::nsSubDocEnumFunc callback;
-  void *data;
-};
-
 void
 nsDocument::EnumerateSubDocuments(nsSubDocEnumFunc aCallback, void *aData)
 {

From 911b60cb08e842ab0c527bf4ff7e0ade9cbc1ea4 Mon Sep 17 00:00:00 2001
From: Hiroyuki Ikezoe <hikezoe@mozilla.com>
Date: Fri, 26 Jan 2018 20:11:09 +0900
Subject: [PATCH 13/46] Bug 1433336 - Add
 nsIDocument::CollectDescendantDocument(). r=smaug

In contrast to nsIDocument::EnumerateSubDocuments(), this new function is
supposed to work with the callback function which does not mutate any state
in the document, and also this function checks all descendant documents even
if there is a descendant that the callback function returns false.

MozReview-Commit-ID: 4f6zD29qQD2

--HG--
extra : rebase_source : 221269fb08bb73d938a6019e1274b2236ee34264
---
 dom/base/nsDocument.cpp | 21 +++++++++++++++++++++
 dom/base/nsDocument.h   |  3 +++
 dom/base/nsIDocument.h  |  9 +++++++++
 3 files changed, 33 insertions(+)

diff --git a/dom/base/nsDocument.cpp b/dom/base/nsDocument.cpp
index b589811f1822..07a008b1e0fd 100644
--- a/dom/base/nsDocument.cpp
+++ b/dom/base/nsDocument.cpp
@@ -8604,6 +8604,27 @@ nsDocument::EnumerateSubDocuments(nsSubDocEnumFunc aCallback, void *aData)
   }
 }
 
+void
+nsDocument::CollectDescendantDocuments(
+  nsTArray<nsCOMPtr<nsIDocument>>& aDescendants,
+  nsDocTestFunc aCallback) const
+{
+  if (!mSubDocuments) {
+    return;
+  }
+
+  for (auto iter = mSubDocuments->Iter(); !iter.Done(); iter.Next()) {
+    auto entry = static_cast<SubDocMapEntry*>(iter.Get());
+    const nsIDocument* subdoc = entry->mSubDocument;
+    if (subdoc) {
+      if (aCallback(subdoc)) {
+        aDescendants.AppendElement(entry->mSubDocument);
+      }
+      subdoc->CollectDescendantDocuments(aDescendants, aCallback);
+    }
+  }
+}
+
 #ifdef DEBUG_bryner
 #define DEBUG_PAGE_CACHE
 #endif
diff --git a/dom/base/nsDocument.h b/dom/base/nsDocument.h
index c96f0ef7b379..a80035e605ac 100644
--- a/dom/base/nsDocument.h
+++ b/dom/base/nsDocument.h
@@ -678,6 +678,9 @@ public:
 
   virtual void EnumerateSubDocuments(nsSubDocEnumFunc aCallback,
                                                  void *aData) override;
+  virtual void CollectDescendantDocuments(
+    nsTArray<nsCOMPtr<nsIDocument>>& aDescendants,
+    nsDocTestFunc aCallback) const override;
 
   virtual bool CanSavePresentation(nsIRequest *aNewRequest) override;
   virtual void Destroy() override;
diff --git a/dom/base/nsIDocument.h b/dom/base/nsIDocument.h
index fe30ebb14c31..0b2eb5ea6990 100644
--- a/dom/base/nsIDocument.h
+++ b/dom/base/nsIDocument.h
@@ -1916,6 +1916,15 @@ public:
   virtual void EnumerateSubDocuments(nsSubDocEnumFunc aCallback,
                                      void *aData) = 0;
 
+  /**
+   * Collect all the descendant documents for which |aCalback| returns true.
+   * The callback function must not mutate any state for the given document.
+   */
+  typedef bool (*nsDocTestFunc)(const nsIDocument* aDocument);
+  virtual void CollectDescendantDocuments(
+    nsTArray<nsCOMPtr<nsIDocument>>& aDescendants,
+    nsDocTestFunc aCallback) const = 0;
+
   /**
    * Check whether it is safe to cache the presentation of this document
    * and all of its subdocuments. This method checks the following conditions

From 471558b3c113879020a61e9d98b222bdfdab7fcf Mon Sep 17 00:00:00 2001
From: Hiroyuki Ikezoe <hikezoe@mozilla.com>
Date: Fri, 26 Jan 2018 20:11:17 +0900
Subject: [PATCH 14/46] Bug 1433336 - Collect only documents which are
 observerd by IntersectionObserver in nsRefreshDriver::Tick(). r=smaug

We don't need to collect whole descendants documents.

This patch intentionally leaves animation events handling which is another
caller of CollectDocuments in nsRefreshDriver since the animation part will
be fixed in a different way in bug 1415780.

MozReview-Commit-ID: INAJm1NHLuI

--HG--
extra : rebase_source : 4e255cef604d4455d6efa5dc2ebddd5dc649ee65
---
 dom/base/nsDocument.h           |  4 ++++
 dom/base/nsIDocument.h          |  1 +
 layout/base/nsRefreshDriver.cpp | 30 +++++++++++++++++++++++-------
 layout/base/nsRefreshDriver.h   |  1 +
 4 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/dom/base/nsDocument.h b/dom/base/nsDocument.h
index a80035e605ac..dcb96dfad1b0 100644
--- a/dom/base/nsDocument.h
+++ b/dom/base/nsDocument.h
@@ -620,6 +620,10 @@ public:
   virtual void UpdateIntersectionObservations() override;
   virtual void ScheduleIntersectionObserverNotification() override;
   virtual void NotifyIntersectionObservers() override;
+  virtual bool HasIntersectionObservers() const override
+  {
+    return !mIntersectionObservers.IsEmpty();
+  }
 
   virtual void NotifyLayerManagerRecreated() override;
 
diff --git a/dom/base/nsIDocument.h b/dom/base/nsIDocument.h
index 0b2eb5ea6990..4440120d875a 100644
--- a/dom/base/nsIDocument.h
+++ b/dom/base/nsIDocument.h
@@ -3120,6 +3120,7 @@ public:
   virtual void UpdateIntersectionObservations() = 0;
   virtual void ScheduleIntersectionObserverNotification() = 0;
   virtual void NotifyIntersectionObservers() = 0;
+  virtual bool HasIntersectionObservers() const = 0;
 
   // Dispatch a runnable related to the document.
   virtual nsresult Dispatch(mozilla::TaskCategory aCategory,
diff --git a/layout/base/nsRefreshDriver.cpp b/layout/base/nsRefreshDriver.cpp
index c7ff4022074c..01bb8b5d3666 100644
--- a/layout/base/nsRefreshDriver.cpp
+++ b/layout/base/nsRefreshDriver.cpp
@@ -1587,6 +1587,28 @@ CollectDocuments(nsIDocument* aDocument, void* aDocArray)
   return true;
 }
 
+void
+nsRefreshDriver::UpdateIntersectionObservations()
+{
+  AutoTArray<nsCOMPtr<nsIDocument>, 32> documents;
+
+  if (mPresContext->Document()->HasIntersectionObservers()) {
+    documents.AppendElement(mPresContext->Document());
+  }
+
+  mPresContext->Document()->CollectDescendantDocuments(
+    documents,
+    [](const nsIDocument* document) -> bool {
+      return document->HasIntersectionObservers();
+    });
+
+  for (uint32_t i = 0; i < documents.Length(); ++i) {
+    nsIDocument* doc = documents[i];
+    doc->UpdateIntersectionObservations();
+    doc->ScheduleIntersectionObserverNotification();
+  }
+}
+
 void
 nsRefreshDriver::DispatchAnimationEvents()
 {
@@ -1955,13 +1977,7 @@ nsRefreshDriver::Tick(int64_t aNowEpoch, TimeStamp aNowTime)
   }
 #endif
 
-  AutoTArray<nsCOMPtr<nsIDocument>, 32> documents;
-  CollectDocuments(mPresContext->Document(), &documents);
-  for (uint32_t i = 0; i < documents.Length(); ++i) {
-    nsIDocument* doc = documents[i];
-    doc->UpdateIntersectionObservations();
-    doc->ScheduleIntersectionObserverNotification();
-  }
+  UpdateIntersectionObservations();
 
   /*
    * Perform notification to imgIRequests subscribed to listen
diff --git a/layout/base/nsRefreshDriver.h b/layout/base/nsRefreshDriver.h
index 70db2f3a627e..10babae47898 100644
--- a/layout/base/nsRefreshDriver.h
+++ b/layout/base/nsRefreshDriver.h
@@ -385,6 +385,7 @@ private:
   void DispatchPendingEvents();
   void DispatchAnimationEvents();
   void RunFrameRequestCallbacks(mozilla::TimeStamp aNowTime);
+  void UpdateIntersectionObservations();
   void Tick(int64_t aNowEpoch, mozilla::TimeStamp aNowTime);
 
   enum EnsureTimerStartedFlags {

From 94906d61109a7f6902b0748f0b497b2d62e97fd1 Mon Sep 17 00:00:00 2001
From: Chris Manchester <cmanchester@mozilla.com>
Date: Tue, 23 Jan 2018 14:55:11 -0800
Subject: [PATCH 15/46] Bug 1432643 - Store `gn gen` output in the objdir per
 project. r=nalexander

MozReview-Commit-ID: 5yPGlwqfhho

--HG--
extra : rebase_source : 16b82d0d17347ec891089eefb5b9bfa07634a53b
---
 python/mozbuild/mozbuild/gn_processor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/mozbuild/mozbuild/gn_processor.py b/python/mozbuild/mozbuild/gn_processor.py
index 5f934ff8a337..355f0c0fa000 100644
--- a/python/mozbuild/mozbuild/gn_processor.py
+++ b/python/mozbuild/mozbuild/gn_processor.py
@@ -510,7 +510,7 @@ def generate_gn_config(config, srcdir, output, non_unified_sources, gn_binary,
     gn_args = '--args=%s' % ' '.join(['%s=%s' % (k, str_for_arg(v)) for k, v
                                       in input_variables.iteritems()])
     gn_arg_string = '_'.join([str(input_variables[k]) for k in sorted(input_variables.keys())])
-    out_dir = mozpath.join(config.topobjdir, 'gn-output')
+    out_dir = mozpath.join(output, 'gn-output')
     gen_args = [
         config.substs['GN'], 'gen', out_dir, gn_args, '--ide=json',
     ]
@@ -560,7 +560,7 @@ class GnMozbuildWriterBackend(BuildBackend):
                 # Check the objdir for a gn-config in to aide debugging in cases
                 # someone is running both steps on the same machine and want to
                 # sanity check moz.build generation for a particular config.
-                gn_config_files = glob.glob(mozpath.join(obj.topobjdir,
+                gn_config_files = glob.glob(mozpath.join(obj.objdir, obj.target_dir,
                                                          'gn-output', '*.json'))
             if gn_config_files:
                 print("Writing moz.build files based on the following gn configs: %s" %

From 71c85d4c2254c85351cfee0f818fc5bb5970dce7 Mon Sep 17 00:00:00 2001
From: Kartikaya Gupta <kgupta@mozilla.com>
Date: Fri, 26 Jan 2018 15:54:36 -0500
Subject: [PATCH 16/46] Bug 1432789 - Update webrender to
 1d8157c71f88d5c673f5d084f02515ab74263814. r=jrmuizel

MozReview-Commit-ID: JakzOipPChe

--HG--
extra : rebase_source : 5f3db52bd59749bc79c0e5c530ae04ab19c03d1d
---
 gfx/doc/README.webrender                |   12 +-
 gfx/webrender/Cargo.toml                |    2 +-
 gfx/webrender/res/cs_text_run.glsl      |    9 +-
 gfx/webrender/res/ps_border_corner.glsl |    7 +-
 gfx/webrender/res/ps_border_edge.glsl   |    8 +-
 gfx/webrender/src/capture.rs            |    5 +-
 gfx/webrender/src/frame_builder.rs      |    4 +-
 gfx/webrender/src/gpu_cache.rs          |   22 +-
 gfx/webrender/src/internal_types.rs     |   10 +-
 gfx/webrender/src/render_backend.rs     |   96 +-
 gfx/webrender/src/renderer.rs           |   18 +-
 gfx/webrender/src/tiling.rs             |   26 +-
 gfx/wrench/.gitignore                   |    7 +
 gfx/wrench/Cargo.toml                   |   44 +
 gfx/wrench/README.md                    |   35 +
 gfx/wrench/build.rs                     |   28 +
 gfx/wrench/res/wrench.exe.manifest      |   24 +
 gfx/wrench/src/args.yaml                |  164 +++
 gfx/wrench/src/binary_frame_reader.rs   |  237 ++++
 gfx/wrench/src/blob.rs                  |  162 +++
 gfx/wrench/src/cgfont_to_data.rs        |  124 ++
 gfx/wrench/src/json_frame_writer.rs     |  317 +++++
 gfx/wrench/src/main.rs                  |  597 +++++++++
 gfx/wrench/src/parse_function.rs        |  121 ++
 gfx/wrench/src/perf.rs                  |  279 +++++
 gfx/wrench/src/png.rs                   |  113 ++
 gfx/wrench/src/premultiply.rs           |   55 +
 gfx/wrench/src/rawtest.rs               |  567 +++++++++
 gfx/wrench/src/reftest.rs               |  495 ++++++++
 gfx/wrench/src/ron_frame_writer.rs      |  195 +++
 gfx/wrench/src/scene.rs                 |  134 ++
 gfx/wrench/src/wrench.rs                |  595 +++++++++
 gfx/wrench/src/yaml_frame_reader.rs     | 1510 +++++++++++++++++++++++
 gfx/wrench/src/yaml_frame_writer.rs     | 1207 ++++++++++++++++++
 gfx/wrench/src/yaml_helper.rs           |  577 +++++++++
 35 files changed, 7724 insertions(+), 82 deletions(-)
 create mode 100644 gfx/wrench/.gitignore
 create mode 100644 gfx/wrench/Cargo.toml
 create mode 100644 gfx/wrench/README.md
 create mode 100644 gfx/wrench/build.rs
 create mode 100644 gfx/wrench/res/wrench.exe.manifest
 create mode 100644 gfx/wrench/src/args.yaml
 create mode 100644 gfx/wrench/src/binary_frame_reader.rs
 create mode 100644 gfx/wrench/src/blob.rs
 create mode 100644 gfx/wrench/src/cgfont_to_data.rs
 create mode 100644 gfx/wrench/src/json_frame_writer.rs
 create mode 100644 gfx/wrench/src/main.rs
 create mode 100644 gfx/wrench/src/parse_function.rs
 create mode 100644 gfx/wrench/src/perf.rs
 create mode 100644 gfx/wrench/src/png.rs
 create mode 100644 gfx/wrench/src/premultiply.rs
 create mode 100644 gfx/wrench/src/rawtest.rs
 create mode 100644 gfx/wrench/src/reftest.rs
 create mode 100644 gfx/wrench/src/ron_frame_writer.rs
 create mode 100644 gfx/wrench/src/scene.rs
 create mode 100644 gfx/wrench/src/wrench.rs
 create mode 100644 gfx/wrench/src/yaml_frame_reader.rs
 create mode 100644 gfx/wrench/src/yaml_frame_writer.rs
 create mode 100644 gfx/wrench/src/yaml_helper.rs

diff --git a/gfx/doc/README.webrender b/gfx/doc/README.webrender
index 2fe7fa4b07dc..6e5233af4359 100644
--- a/gfx/doc/README.webrender
+++ b/gfx/doc/README.webrender
@@ -48,10 +48,12 @@ Option B:
    roughly:
    - Update your mozilla-central checkout to the latest code on mozilla-central.
    - Check out and update the webrender repo to the version you want
-   - Copy over the webrender and webrender_api folders into gfx/. The best way
-     to do this is to simply delete the gfx/webrender and gfx/webrender_api
-     folders and use |cp -R| to copy them in again from the webrender repo. Update
-     the "latest commit" information at the bottom of this file with the version.
+   - Copy over the webrender, webrender_api, and part of the wrench folders into
+     gfx/. The best way to do this is to simply delete the gfx/webrender,
+     gfx/webrender_api, and gfx/wrench folders and use |cp -R| to copy them in
+     again from the webrender repo, and then delete the gfx/wrench/reftests,
+     gfx/wrench/benchmarks, and gfx/wrench/script folders. Update the "latest
+     commit" information at the bottom of this file with the version.
    - If you need to modify webrender_bindings/Cargo.toml file, do so now. Changes
      at this step usually consist of:
      (a) Updating version numbers. Go through the version numbers of ALL the
@@ -175,4 +177,4 @@ Troubleshooting tips:
 -------------------------------------------------------------------------------
 
 The version of WebRender currently in the tree is:
-c0943271eb8c6440a61db37e2f1e84201dcac2e3
+1d8157c71f88d5c673f5d084f02515ab74263814
diff --git a/gfx/webrender/Cargo.toml b/gfx/webrender/Cargo.toml
index c4f86cf93baa..ffb3a71bfd99 100644
--- a/gfx/webrender/Cargo.toml
+++ b/gfx/webrender/Cargo.toml
@@ -36,7 +36,7 @@ serde_json = { optional = true, version = "1.0" }
 serde = { optional = true, version = "1.0", features = ["serde_derive"] }
 image = { optional = true, version = "0.17" }
 base64 = { optional = true, version = "0.3.0" }
-ron = { optional = true, version = "0.1.5" }
+ron = { optional = true, version = "0.1.7" }
 
 [dev-dependencies]
 angle = {git = "https://github.com/servo/angle", branch = "servo"}
diff --git a/gfx/webrender/res/cs_text_run.glsl b/gfx/webrender/res/cs_text_run.glsl
index b6ff2eead619..796fc789c66a 100644
--- a/gfx/webrender/res/cs_text_run.glsl
+++ b/gfx/webrender/res/cs_text_run.glsl
@@ -6,6 +6,7 @@
 
 varying vec3 vUv;
 flat varying vec4 vColor;
+flat varying vec4 vStRect;
 
 #ifdef WR_VERTEX_SHADER
 // Draw a text run to a cache target. These are always
@@ -41,12 +42,18 @@ void main(void) {
 
     vUv = vec3(mix(st0, st1, aPosition.xy), res.layer);
     vColor = prim.task.color;
+
+    // We clamp the texture coordinates to the half-pixel offset from the borders
+    // in order to avoid sampling outside of the texture area.
+    vec2 half_texel = vec2(0.5) / texture_size;
+    vStRect = vec4(min(st0, st1) + half_texel, max(st0, st1) - half_texel);
 }
 #endif
 
 #ifdef WR_FRAGMENT_SHADER
 void main(void) {
-    float a = texture(sColor0, vUv).a;
+    vec2 uv = clamp(vUv.xy, vStRect.xy, vStRect.zw);
+    float a = texture(sColor0, vec3(uv, vUv.z)).a;
     oFragColor = vColor * a;
 }
 #endif
diff --git a/gfx/webrender/res/ps_border_corner.glsl b/gfx/webrender/res/ps_border_corner.glsl
index bea4ffcfea88..fc82716fc1e4 100644
--- a/gfx/webrender/res/ps_border_corner.glsl
+++ b/gfx/webrender/res/ps_border_corner.glsl
@@ -153,6 +153,7 @@ void main(void) {
     vec4 edge_distances;
     vec4 color0, color1;
     vec2 color_delta;
+    vec4 edge_mask;
 
     // TODO(gw): Now that all border styles are supported, the switch
     //           statement below can be tidied up quite a bit.
@@ -180,6 +181,7 @@ void main(void) {
             color_delta = vec2(1.0);
             vIsBorderRadiusLessThanBorderWidth = any(lessThan(border.radii[0].xy,
                                                               border.widths.xy)) ? 1.0 : 0.0;
+            edge_mask = vec4(1.0, 1.0, 0.0, 0.0);
             break;
         }
         case 1: {
@@ -206,6 +208,7 @@ void main(void) {
             color_delta = vec2(1.0, -1.0);
             vIsBorderRadiusLessThanBorderWidth = any(lessThan(border.radii[0].zw,
                                                               border.widths.zy)) ? 1.0 : 0.0;
+            edge_mask = vec4(0.0, 1.0, 1.0, 0.0);
             break;
         }
         case 2: {
@@ -232,6 +235,7 @@ void main(void) {
             color_delta = vec2(-1.0);
             vIsBorderRadiusLessThanBorderWidth = any(lessThan(border.radii[1].xy,
                                                               border.widths.zw)) ? 1.0 : 0.0;
+            edge_mask = vec4(0.0, 0.0, 1.0, 1.0);
             break;
         }
         case 3: {
@@ -258,6 +262,7 @@ void main(void) {
             color_delta = vec2(-1.0, 1.0);
             vIsBorderRadiusLessThanBorderWidth = any(lessThan(border.radii[1].zw,
                                                               border.widths.xw)) ? 1.0 : 0.0;
+            edge_mask = vec4(1.0, 0.0, 0.0, 1.0);
             break;
         }
     }
@@ -301,7 +306,7 @@ void main(void) {
     VertexInfo vi = write_transform_vertex(segment_rect,
                                            prim.local_rect,
                                            prim.local_clip_rect,
-                                           vec4(1.0),
+                                           edge_mask,
                                            prim.z,
                                            prim.scroll_node,
                                            prim.task);
diff --git a/gfx/webrender/res/ps_border_edge.glsl b/gfx/webrender/res/ps_border_edge.glsl
index 7dc070828515..3fac7a5ff973 100644
--- a/gfx/webrender/res/ps_border_edge.glsl
+++ b/gfx/webrender/res/ps_border_edge.glsl
@@ -152,6 +152,8 @@ void main(void) {
     bool color_flip;
 
     RectWithSize segment_rect;
+    vec4 edge_mask;
+
     switch (sub_part) {
         case 0: {
             segment_rect.p0 = vec2(corners.tl_outer.x, corners.tl_inner.y);
@@ -165,6 +167,7 @@ void main(void) {
                               segment_rect.size.y,
                               segment_rect.p0.y,
                               segment_rect.p0.x + 0.5 * segment_rect.size.x);
+            edge_mask = vec4(1.0, 0.0, 1.0, 0.0);
             break;
         }
         case 1: {
@@ -179,6 +182,7 @@ void main(void) {
                               segment_rect.size.x,
                               segment_rect.p0.x,
                               segment_rect.p0.y + 0.5 * segment_rect.size.y);
+            edge_mask = vec4(0.0, 1.0, 0.0, 1.0);
             break;
         }
         case 2: {
@@ -193,6 +197,7 @@ void main(void) {
                               segment_rect.size.y,
                               segment_rect.p0.y,
                               segment_rect.p0.x + 0.5 * segment_rect.size.x);
+            edge_mask = vec4(1.0, 0.0, 1.0, 0.0);
             break;
         }
         case 3: {
@@ -207,6 +212,7 @@ void main(void) {
                               segment_rect.size.x,
                               segment_rect.p0.x,
                               segment_rect.p0.y + 0.5 * segment_rect.size.y);
+            edge_mask = vec4(0.0, 1.0, 0.0, 1.0);
             break;
         }
     }
@@ -219,7 +225,7 @@ void main(void) {
     VertexInfo vi = write_transform_vertex(segment_rect,
                                            prim.local_rect,
                                            prim.local_clip_rect,
-                                           vec4(1.0),
+                                           edge_mask,
                                            prim.z,
                                            prim.scroll_node,
                                            prim.task);
diff --git a/gfx/webrender/src/capture.rs b/gfx/webrender/src/capture.rs
index e5d612b73a7a..c0f75c8d30a7 100644
--- a/gfx/webrender/src/capture.rs
+++ b/gfx/webrender/src/capture.rs
@@ -24,7 +24,10 @@ impl CaptureConfig {
         CaptureConfig {
             root,
             bits,
-            pretty: ser::PrettyConfig::default(),
+            pretty: ser::PrettyConfig {
+                enumerate_arrays: true,
+                .. ser::PrettyConfig::default()
+            },
         }
     }
 
diff --git a/gfx/webrender/src/frame_builder.rs b/gfx/webrender/src/frame_builder.rs
index a2b24036d168..100e29c63e64 100644
--- a/gfx/webrender/src/frame_builder.rs
+++ b/gfx/webrender/src/frame_builder.rs
@@ -1793,7 +1793,7 @@ impl FrameBuilder {
             }
         }
 
-        let gpu_cache_updates = gpu_cache.end_frame(gpu_cache_profile);
+        let gpu_cache_frame_id = gpu_cache.end_frame(gpu_cache_profile);
 
         render_tasks.build();
 
@@ -1811,7 +1811,7 @@ impl FrameBuilder {
             clip_chain_local_clip_rects,
             render_tasks,
             deferred_resolves,
-            gpu_cache_updates: Some(gpu_cache_updates),
+            gpu_cache_frame_id,
             has_been_rendered: false,
             has_texture_cache_tasks,
         }
diff --git a/gfx/webrender/src/gpu_cache.rs b/gfx/webrender/src/gpu_cache.rs
index ae70b63aa055..25f4160fdcbc 100644
--- a/gfx/webrender/src/gpu_cache.rs
+++ b/gfx/webrender/src/gpu_cache.rs
@@ -221,15 +221,18 @@ pub enum GpuCacheUpdate {
     },
 }
 
+#[must_use]
 #[cfg_attr(feature = "capture", derive(Deserialize, Serialize))]
 pub struct GpuCacheUpdateList {
-    // The current height of the texture. The render thread
-    // should resize the texture if required.
+    /// The frame current update list was generated from.
+    pub frame_id: FrameId,
+    /// The current height of the texture. The render thread
+    /// should resize the texture if required.
     pub height: u32,
-    // List of updates to apply.
+    /// List of updates to apply.
     pub updates: Vec<GpuCacheUpdate>,
-    // A flat list of GPU blocks that are pending upload
-    // to GPU memory.
+    /// A flat list of GPU blocks that are pending upload
+    /// to GPU memory.
     pub blocks: Vec<GpuBlockData>,
 }
 
@@ -595,9 +598,9 @@ impl GpuCache {
     /// End the frame. Return the list of updates to apply to the
     /// device specific cache texture.
     pub fn end_frame(
-        &mut self,
+        &self,
         profile_counters: &mut GpuCacheProfileCounters,
-    ) -> GpuCacheUpdateList {
+    ) -> FrameId {
         profile_counters
             .allocated_rows
             .set(self.texture.rows.len());
@@ -607,8 +610,13 @@ impl GpuCache {
         profile_counters
             .saved_blocks
             .set(self.saved_block_count);
+        self.frame_id
+    }
 
+    /// Extract the pending updates from the cache.
+    pub fn extract_updates(&mut self) -> GpuCacheUpdateList {
         GpuCacheUpdateList {
+            frame_id: self.frame_id,
             height: self.texture.height,
             updates: mem::replace(&mut self.texture.updates, Vec::new()),
             blocks: mem::replace(&mut self.texture.pending_blocks, Vec::new()),
diff --git a/gfx/webrender/src/internal_types.rs b/gfx/webrender/src/internal_types.rs
index ae335e3deb51..571d33d2e6ba 100644
--- a/gfx/webrender/src/internal_types.rs
+++ b/gfx/webrender/src/internal_types.rs
@@ -7,6 +7,7 @@ use api::{ExternalImageData, ExternalImageId};
 use api::{ImageFormat, PipelineId};
 use api::DebugCommand;
 use device::TextureFilter;
+use gpu_cache::GpuCacheUpdateList;
 use fxhash::FxHasher;
 use profiler::BackendProfileCounters;
 use std::{usize, i32};
@@ -161,14 +162,15 @@ pub enum ResultMsg {
     DebugCommand(DebugCommand),
     DebugOutput(DebugOutput),
     RefreshShader(PathBuf),
+    UpdateGpuCache(GpuCacheUpdateList),
+    UpdateResources {
+        updates: TextureUpdateList,
+        cancel_rendering: bool,
+    },
     PublishDocument(
         DocumentId,
         RenderedDocument,
         TextureUpdateList,
         BackendProfileCounters,
     ),
-    UpdateResources {
-        updates: TextureUpdateList,
-        cancel_rendering: bool,
-    },
 }
diff --git a/gfx/webrender/src/render_backend.rs b/gfx/webrender/src/render_backend.rs
index e7188ccdde4a..962f95bb9fa4 100644
--- a/gfx/webrender/src/render_backend.rs
+++ b/gfx/webrender/src/render_backend.rs
@@ -6,9 +6,9 @@ use api::{ApiMsg, BlobImageRenderer, BuiltDisplayList, DebugCommand, DeviceIntPo
 #[cfg(feature = "debugger")]
 use api::{BuiltDisplayListIter, SpecificDisplayItem};
 use api::{DevicePixelScale, DeviceUintPoint, DeviceUintRect, DeviceUintSize};
-use api::{DocumentId, DocumentLayer, DocumentMsg};
-use api::{IdNamespace, PipelineId, RenderNotifier};
-use api::channel::{MsgReceiver, PayloadReceiver, PayloadReceiverHelperMethods};
+use api::{DocumentId, DocumentLayer, DocumentMsg, HitTestFlags, HitTestResult};
+use api::{IdNamespace, PipelineId, RenderNotifier, WorldPoint};
+use api::channel::{MsgReceiver, MsgSender, PayloadReceiver, PayloadReceiverHelperMethods};
 use api::channel::{PayloadSender, PayloadSenderHelperMethods};
 #[cfg(feature = "capture")]
 use api::CapturedDocument;
@@ -20,7 +20,7 @@ use frame::FrameContext;
 use frame_builder::{FrameBuilder, FrameBuilderConfig};
 use gpu_cache::GpuCache;
 use internal_types::{DebugOutput, FastHashMap, FastHashSet, RenderedDocument, ResultMsg};
-use profiler::{BackendProfileCounters, ResourceProfileCounters};
+use profiler::{BackendProfileCounters, IpcProfileCounters, ResourceProfileCounters};
 use rayon::ThreadPool;
 use record::ApiRecordingReceiver;
 use resource_cache::ResourceCache;
@@ -153,10 +153,13 @@ impl Document {
     }
 }
 
+type HitTestQuery = (Option<PipelineId>, WorldPoint, HitTestFlags, MsgSender<HitTestResult>);
+
 struct DocumentOps {
     scroll: bool,
     build: bool,
     render: bool,
+    queries: Vec<HitTestQuery>,
 }
 
 impl DocumentOps {
@@ -165,6 +168,7 @@ impl DocumentOps {
             scroll: false,
             build: false,
             render: false,
+            queries: vec![],
         }
     }
 
@@ -175,10 +179,11 @@ impl DocumentOps {
         }
     }
 
-    fn combine(&mut self, other: Self) {
+    fn combine(&mut self, mut other: Self) {
         self.scroll = self.scroll || other.scroll;
         self.build = self.build || other.build;
         self.render = self.render || other.render;
+        self.queries.extend(other.queries.drain(..));
     }
 }
 
@@ -260,7 +265,8 @@ impl RenderBackend {
         document_id: DocumentId,
         message: DocumentMsg,
         frame_counter: u32,
-        profile_counters: &mut BackendProfileCounters,
+        ipc_profile_counters: &mut IpcProfileCounters,
+        resource_profile_counters: &mut ResourceProfileCounters,
     ) -> DocumentOps {
         let doc = self.documents.get_mut(&document_id).expect("No document?");
 
@@ -332,7 +338,6 @@ impl RenderBackend {
                 let display_list_received_time = precise_time_ns();
 
                 {
-                    let _timer = profile_counters.total_time.timer();
                     doc.scene.set_display_list(
                         pipeline_id,
                         epoch,
@@ -352,7 +357,7 @@ impl RenderBackend {
                 // really simple and cheap to access, so it's not a big deal.
                 let display_list_consumed_time = precise_time_ns();
 
-                profile_counters.ipc.set(
+                ipc_profile_counters.set(
                     builder_start_time,
                     builder_finish_time,
                     send_start_time,
@@ -368,7 +373,7 @@ impl RenderBackend {
 
                 self.resource_cache.update_resources(
                     updates,
-                    &mut profile_counters.resources
+                    resource_profile_counters
                 );
 
                 DocumentOps::nop()
@@ -396,59 +401,44 @@ impl RenderBackend {
             }
             DocumentMsg::Scroll(delta, cursor, move_phase) => {
                 profile_scope!("Scroll");
-                let _timer = profile_counters.total_time.timer();
 
                 let should_render = doc.frame_ctx.scroll(delta, cursor, move_phase)
                     && doc.render_on_scroll == Some(true);
 
                 DocumentOps {
                     scroll: true,
-                    build: false,
                     render: should_render,
+                    ..DocumentOps::nop()
                 }
             }
             DocumentMsg::HitTest(pipeline_id, point, flags, tx) => {
-                profile_scope!("HitTest");
-                if doc.render_on_hittest {
-                    doc.render(
-                        &mut self.resource_cache,
-                        &mut self.gpu_cache,
-                        &mut profile_counters.resources,
-                    );
-                    doc.render_on_hittest = false;
+                DocumentOps {
+                    render: doc.render_on_hittest,
+                    queries: vec![(pipeline_id, point, flags, tx)],
+                    ..DocumentOps::nop()
                 }
-
-                let cst = doc.frame_ctx.get_clip_scroll_tree();
-                let result = doc.frame_builder
-                    .as_ref()
-                    .unwrap()
-                    .hit_test(cst, pipeline_id, point, flags);
-                tx.send(result).unwrap();
-                DocumentOps::nop()
             }
             DocumentMsg::ScrollNodeWithId(origin, id, clamp) => {
                 profile_scope!("ScrollNodeWithScrollId");
-                let _timer = profile_counters.total_time.timer();
 
                 let should_render = doc.frame_ctx.scroll_node(origin, id, clamp)
                     && doc.render_on_scroll == Some(true);
 
                 DocumentOps {
                     scroll: true,
-                    build: false,
                     render: should_render,
+                    ..DocumentOps::nop()
                 }
             }
             DocumentMsg::TickScrollingBounce => {
                 profile_scope!("TickScrollingBounce");
-                let _timer = profile_counters.total_time.timer();
 
                 doc.frame_ctx.tick_scrolling_bounce_animations();
 
                 DocumentOps {
                     scroll: true,
-                    build: false,
                     render: doc.render_on_scroll == Some(true),
+                    ..DocumentOps::nop()
                 }
             }
             DocumentMsg::GetScrollNodeState(tx) => {
@@ -471,8 +461,6 @@ impl RenderBackend {
                 DocumentOps::build()
             }
             DocumentMsg::GenerateFrame => {
-                let _timer = profile_counters.total_time.timer();
-
                 let mut op = DocumentOps::nop();
 
                 if let Some(ref mut ros) = doc.render_on_scroll {
@@ -656,12 +644,14 @@ impl RenderBackend {
     ) {
         let mut op = DocumentOps::nop();
         for doc_msg in doc_msgs {
+            let _timer = profile_counters.total_time.timer();
             op.combine(
                 self.process_document(
                     document_id,
                     doc_msg,
                     *frame_counter,
-                    profile_counters,
+                    &mut profile_counters.ipc,
+                    &mut profile_counters.resources,
                 )
             );
         }
@@ -669,6 +659,7 @@ impl RenderBackend {
         let doc = self.documents.get_mut(&document_id).unwrap();
 
         if op.build {
+            let _timer = profile_counters.total_time.timer();
             profile_scope!("build scene");
             doc.build_scene(&mut self.resource_cache);
             doc.render_on_hittest = true;
@@ -678,17 +669,28 @@ impl RenderBackend {
             profile_scope!("generate frame");
 
             *frame_counter += 1;
-            let rendered_document = doc.render(
-                &mut self.resource_cache,
-                &mut self.gpu_cache,
-                &mut profile_counters.resources,
-            );
 
-            info!("generated frame for document {:?} with {} passes",
-                document_id, rendered_document.frame.passes.len());
+            // borrow ck hack for profile_counters
+            let (pending_update, rendered_document) = {
+                let _timer = profile_counters.total_time.timer();
+
+                let rendered_document = doc.render(
+                    &mut self.resource_cache,
+                    &mut self.gpu_cache,
+                    &mut profile_counters.resources,
+                );
+
+                info!("generated frame for document {:?} with {} passes",
+                    document_id, rendered_document.frame.passes.len());
+
+                let msg = ResultMsg::UpdateGpuCache(self.gpu_cache.extract_updates());
+                self.result_tx.send(msg).unwrap();
+
+                let pending_update = self.resource_cache.pending_updates();
+                (pending_update, rendered_document)
+            };
 
             // Publish the frame
-            let pending_update = self.resource_cache.pending_updates();
             let msg = ResultMsg::PublishDocument(
                 document_id,
                 rendered_document,
@@ -703,6 +705,16 @@ impl RenderBackend {
         if op.render || op.scroll {
             self.notifier.new_document_ready(document_id, op.scroll, op.render);
         }
+
+        for (pipeline_id, point, flags, tx) in op.queries {
+            profile_scope!("HitTest");
+            let cst = doc.frame_ctx.get_clip_scroll_tree();
+            let result = doc.frame_builder
+                .as_ref()
+                .unwrap()
+                .hit_test(cst, pipeline_id, point, flags);
+            tx.send(result).unwrap();
+        }
     }
 
     #[cfg(not(feature = "debugger"))]
diff --git a/gfx/webrender/src/renderer.rs b/gfx/webrender/src/renderer.rs
index 42946a4a3be7..232a9c1c6197 100644
--- a/gfx/webrender/src/renderer.rs
+++ b/gfx/webrender/src/renderer.rs
@@ -1656,6 +1656,8 @@ pub struct Renderer {
     render_task_texture: VertexDataTexture,
     gpu_cache_texture: CacheTexture,
 
+    gpu_cache_frame_id: FrameId,
+
     pipeline_epoch_map: FastHashMap<PipelineId, Epoch>,
 
     // Manages and resolves source textures IDs to real texture IDs.
@@ -2313,6 +2315,7 @@ impl Renderer {
             cpu_profiles: VecDeque::new(),
             gpu_profiles: VecDeque::new(),
             gpu_cache_texture,
+            gpu_cache_frame_id: FrameId::new(0),
             texture_cache_upload_pbo,
             texture_resolver,
             renderer_errors: Vec::new(),
@@ -2368,19 +2371,17 @@ impl Renderer {
     /// Should be called before `render()`, as texture cache updates are done here.
     pub fn update(&mut self) {
         profile_scope!("update");
-
         // Pull any pending results and return the most recent.
         while let Ok(msg) = self.result_rx.try_recv() {
             match msg {
                 ResultMsg::PublishDocument(
                     document_id,
-                    mut doc,
+                    doc,
                     texture_update_list,
                     profile_counters,
                 ) => {
                     //TODO: associate `document_id` with target window
                     self.pending_texture_updates.push(texture_update_list);
-                    self.pending_gpu_cache_updates.extend(doc.frame.gpu_cache_updates.take());
                     self.backend_profile_counters = profile_counters;
 
                     // Update the list of available epochs for use during reftests.
@@ -2404,6 +2405,9 @@ impl Renderer {
                         None => self.active_documents.push((document_id, doc)),
                     }
                 }
+                ResultMsg::UpdateGpuCache(list) => {
+                    self.pending_gpu_cache_updates.push(list);
+                }
                 ResultMsg::UpdateResources {
                     updates,
                     cancel_rendering,
@@ -2753,7 +2757,6 @@ impl Renderer {
         framebuffer_size: Option<DeviceUintSize>
     ) -> Result<RendererStats, Vec<RendererError>> {
         profile_scope!("render");
-
         if self.active_documents.is_empty() {
             self.last_time = precise_time_ns();
             return Ok(RendererStats::empty());
@@ -2844,6 +2847,7 @@ impl Renderer {
 
             for &mut (_, RenderedDocument { ref mut frame, .. }) in &mut active_documents {
                 self.prepare_gpu_cache(frame);
+                assert!(frame.gpu_cache_frame_id <= self.gpu_cache_frame_id);
 
                 self.draw_tile_frame(
                     frame,
@@ -2896,6 +2900,7 @@ impl Renderer {
             }
         }
 
+        self.backend_profile_counters.reset();
         self.profile_counters.reset();
         self.profile_counters.frame_counter.inc();
 
@@ -2931,6 +2936,7 @@ impl Renderer {
         let gpu_cache_height = self.gpu_cache_texture.get_height();
         if gpu_cache_height != 0 &&  GPU_CACHE_RESIZE_TEST {
             self.pending_gpu_cache_updates.push(GpuCacheUpdateList {
+                frame_id: FrameId::new(0),
                 height: gpu_cache_height,
                 blocks: vec![[1f32; 4].into()],
                 updates: Vec::new(),
@@ -2955,6 +2961,9 @@ impl Renderer {
 
         for update_list in self.pending_gpu_cache_updates.drain(..) {
             assert!(update_list.height <= max_requested_height);
+            if update_list.frame_id > self.gpu_cache_frame_id {
+                self.gpu_cache_frame_id = update_list.frame_id
+            }
             self.gpu_cache_texture
                 .update(&mut self.device, &update_list);
         }
@@ -4089,6 +4098,7 @@ impl Renderer {
             .expect("Found external image, but no handler set!");
 
         let mut list = GpuCacheUpdateList {
+            frame_id: FrameId::new(0),
             height: self.gpu_cache_texture.get_height(),
             blocks: Vec::new(),
             updates: Vec::new(),
diff --git a/gfx/webrender/src/tiling.rs b/gfx/webrender/src/tiling.rs
index c25ff55e68dc..81f592f35446 100644
--- a/gfx/webrender/src/tiling.rs
+++ b/gfx/webrender/src/tiling.rs
@@ -9,8 +9,8 @@ use api::{LayerRect, MixBlendMode, PipelineId};
 use batch::{AlphaBatcher, ClipBatcher, resolve_image};
 use clip::{ClipStore};
 use clip_scroll_tree::{ClipScrollTree};
-use device::Texture;
-use gpu_cache::{GpuCache, GpuCacheUpdateList};
+use device::{FrameId, Texture};
+use gpu_cache::{GpuCache};
 use gpu_types::{BlurDirection, BlurInstance, BrushInstance, ClipChainRectIndex};
 use gpu_types::{ClipScrollNodeData, ClipScrollNodeIndex};
 use gpu_types::{PrimitiveInstance};
@@ -861,6 +861,7 @@ impl CompositeOps {
 /// and presented to the renderer.
 #[cfg_attr(feature = "capture", derive(Deserialize, Serialize))]
 pub struct Frame {
+    //TODO: share the fields with DocumentView struct
     pub window_size: DeviceUintSize,
     pub inner_rect: DeviceUintRect,
     pub background_color: Option<ColorF>,
@@ -874,22 +875,21 @@ pub struct Frame {
     pub clip_chain_local_clip_rects: Vec<LayerRect>,
     pub render_tasks: RenderTaskTree,
 
-    // List of updates that need to be pushed to the
-    // gpu resource cache.
-    pub gpu_cache_updates: Option<GpuCacheUpdateList>,
+    /// The GPU cache frame that the contents of Self depend on
+    pub gpu_cache_frame_id: FrameId,
 
-    // List of textures that we don't know about yet
-    // from the backend thread. The render thread
-    // will use a callback to resolve these and
-    // patch the data structures.
+    /// List of textures that we don't know about yet
+    /// from the backend thread. The render thread
+    /// will use a callback to resolve these and
+    /// patch the data structures.
     pub deferred_resolves: Vec<DeferredResolve>,
 
-    // True if this frame contains any render tasks
-    // that write to the texture cache.
+    /// True if this frame contains any render tasks
+    /// that write to the texture cache.
     pub has_texture_cache_tasks: bool,
 
-    // True if this frame has been drawn by the
-    // renderer.
+    /// True if this frame has been drawn by the
+    /// renderer.
     pub has_been_rendered: bool,
 }
 
diff --git a/gfx/wrench/.gitignore b/gfx/wrench/.gitignore
new file mode 100644
index 000000000000..341d13d0f754
--- /dev/null
+++ b/gfx/wrench/.gitignore
@@ -0,0 +1,7 @@
+Cargo.lock
+target/
+*#
+*~
+yaml_frames/
+json_frames/
+bin_frames/
diff --git a/gfx/wrench/Cargo.toml b/gfx/wrench/Cargo.toml
new file mode 100644
index 000000000000..635619d319b2
--- /dev/null
+++ b/gfx/wrench/Cargo.toml
@@ -0,0 +1,44 @@
+[package]
+name = "wrench"
+version = "0.2.6"
+authors = ["Vladimir Vukicevic <vladimir@pobox.com>"]
+build = "build.rs"
+license = "MPL-2.0"
+
+[dependencies]
+base64 = "0.3"
+bincode = "0.9"
+byteorder = "1.0"
+env_logger = { version = "0.4", optional = true }
+euclid = "0.16"
+gleam = "0.4"
+servo-glutin = "0.13"
+app_units = "0.6"
+image = "0.17"
+clap = { version = "2", features = ["yaml"] }
+lazy_static = "1"
+log = "0.3"
+yaml-rust = { git = "https://github.com/vvuk/yaml-rust", features = ["preserve_order"] }
+serde_json = "1.0"
+ron = "0.1.5"
+time = "0.1"
+crossbeam = "0.2"
+osmesa-sys = { version = "0.1.2", optional = true }
+osmesa-src = { git = "https://github.com/servo/osmesa-src", optional = true }
+webrender = {path = "../webrender", features=["capture","debugger","png","profiler"]}
+webrender_api = {path = "../webrender_api", features=["debug-serialization"]}
+serde = {version = "1.0", features = ["derive"] }
+
+[target.'cfg(target_os = "macos")'.dependencies]
+core-graphics = "0.12.4"
+core-foundation = "0.4"
+
+[features]
+headless = [ "osmesa-sys", "osmesa-src" ]
+logging = [ "env_logger" ]
+
+[target.'cfg(target_os = "windows")'.dependencies]
+dwrote = "0.4.1"
+
+[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
+font-loader = "0.5"
diff --git a/gfx/wrench/README.md b/gfx/wrench/README.md
new file mode 100644
index 000000000000..d8dc09370058
--- /dev/null
+++ b/gfx/wrench/README.md
@@ -0,0 +1,35 @@
+# wrench
+
+`wrench` is a tool for debugging webrender outside of a browser engine.
+
+## headless
+
+`wrench` has an optional headless mode for use in continuous integration. To run in headless mode, instead of using `cargo run -- args`, use `./headless.py args`.
+
+## `replay` and `show`
+
+Binary recordings can be generated by webrender and replayed with `wrench replay`. Enable binary recording in `RendererOptions`.
+
+```rust
+RendererOptions {
+    ...
+    recorder: Some(Box::new(BinaryRecorder::new("wr-frame.bin"))),
+    ...
+}
+```
+
+If you are working on gecko integration you can enable recording in `webrender_bindings/src/bindings.rs` by setting
+
+```rust
+static ENABLE_RECORDING: bool = true;
+```
+
+`wrench replay --save yaml` will convert the recording into frames described in yaml. Frames can then be replayed with `wrench show`.
+
+## `reftest`
+
+Wrench also has a reftest system for catching regressions.
+* To run all reftests, run `./headless.py reftest`
+* To run specific reftests, run `./headless.py reftest path/to/test/or/dir`
+* To examine test failures, use the [reftest analyzer](https://hg.mozilla.org/mozilla-central/raw-file/tip/layout/tools/reftest/reftest-analyzer.xhtml)
+* To add a new reftest, create an example frame and a reference frame in `reftests/` and then add an entry to `reftests/reftest.list`
diff --git a/gfx/wrench/build.rs b/gfx/wrench/build.rs
new file mode 100644
index 000000000000..974ef3180cca
--- /dev/null
+++ b/gfx/wrench/build.rs
@@ -0,0 +1,28 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+
+fn main() {
+    let target = env::var("TARGET").unwrap();
+    let out_dir = env::var_os("OUT_DIR").unwrap();
+    let out_dir = PathBuf::from(out_dir);
+
+    println!("cargo:rerun-if-changed=res/wrench.exe.manifest");
+    if target.contains("windows") {
+        let src = PathBuf::from("res/wrench.exe.manifest");
+        let mut dst = out_dir
+            .parent()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .to_owned();
+        dst.push("wrench.exe.manifest");
+        fs::copy(&src, &dst).unwrap();
+    }
+}
diff --git a/gfx/wrench/res/wrench.exe.manifest b/gfx/wrench/res/wrench.exe.manifest
new file mode 100644
index 000000000000..223d04d93fb0
--- /dev/null
+++ b/gfx/wrench/res/wrench.exe.manifest
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1"
+          manifestVersion="1.0"
+          xmlns:asmv3="urn:schemas-microsoft-com:asm.v3">
+  <assemblyIdentity type="win32"
+                    name="webrender.Wrench"
+                    version="0.1.0.0"/>
+
+  <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
+    <application>
+      <supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/> <!-- Windows 7 -->
+      <supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/> <!-- Windows 8 -->
+      <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/> <!-- Windows 8.1 -->
+      <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/> <!-- Windows 10 -->
+    </application>
+  </compatibility>
+
+  <asmv3:application>
+    <asmv3:windowsSettings xmlns="http://schemas.microsoft.com/SMI/2005/WindowsSettings">
+      <dpiAware>true</dpiAware> 
+    </asmv3:windowsSettings>
+  </asmv3:application>
+</assembly>
+
diff --git a/gfx/wrench/src/args.yaml b/gfx/wrench/src/args.yaml
new file mode 100644
index 000000000000..2ca3b150cb05
--- /dev/null
+++ b/gfx/wrench/src/args.yaml
@@ -0,0 +1,164 @@
+name: wrench
+version: "0.1"
+author: Vladimir Vukicevic <vladimir@pobox.com>
+about: WebRender testing and debugging utility
+
+args:
+  - precache:
+      short: c
+      long: precache
+      help: Precache shaders
+  - verbose:
+      short: v
+      long: verbose
+      help: Enable verbose display
+  - zoom:
+      short: z
+      long: zoom
+      help: Set zoom factor
+      takes_value: true
+  - debug:
+      short: d
+      long: debug
+      help: Enable debug renderer
+  - shaders:
+      long: shaders
+      help: Override path for shaders
+      takes_value: true
+  - rebuild:
+      short: r
+      long: rebuild
+      help: Rebuild display list from scratch every frame
+  - save:
+      long: save
+      help: 'Save frames, one of: yaml, json, ron, or binary'
+      takes_value: true
+  - no_subpixel_aa:
+      short: a
+      long: no-subpixel-aa
+      help: Disable subpixel aa
+  - slow_subpixel:
+      long: slow-subpixel
+      help: Disable dual source blending
+  - headless:
+      short: h
+      long: headless
+      help: Enable headless rendering
+  - dp_ratio:
+      short: p
+      long: device-pixel-ratio
+      help: Device pixel ratio
+      takes_value: true
+  - size:
+      short: s
+      long: size
+      help: Window size, specified as widthxheight (e.g. 1024x768), in pixels
+      takes_value: true
+  - time:
+      short: t
+      long: time
+      help: Time limit (in seconds)
+      takes_value: true
+  - vsync:
+      long: vsync
+      help: Enable vsync for OpenGL window
+  - no_scissor:
+      long: no-scissor
+      help: Disable scissors when clearing render targets
+  - no_batch:
+      long: no-batch
+      help: Disable batching of instanced draw calls
+
+subcommands:
+    - png:
+        about: render frame described by YAML and save it to a png file
+        args:
+          - surface:
+              short: s
+              long: surface
+              help: 'What rendered surface to save as PNG, one of: screen, gpu-cache'
+              takes_value: true
+          - INPUT:
+              help: The input YAML file
+              required: true
+              index: 1
+    - show:
+        about: show frame(s) described by YAML
+        args:
+          - queue:
+              short: q
+              long: queue
+              help: How many frames to submit to WR ahead of time (default 1)
+              takes_value: true
+          - include:
+              long: include
+              help: Include the given element type. Can be specified multiple times. (rect/image/text/glyphs/border)
+              multiple: true
+              takes_value: true
+          - list-resources:
+              long: list-resources
+              help: List the resources used by this YAML file
+          - watch:
+              short: w
+              long: watch
+              help: Watch the given YAML file, reloading whenever it changes
+          - INPUT:
+              help: The input YAML file
+              required: true
+              index: 1
+    - replay:
+        about: replay binary recording
+        args:
+          - api:
+              long: api
+              help: Reissue Api messsages for each frame
+          - skip-uploads:
+              long: skip-uploads
+              help: Skip re-uploads while reissuing Api messages (BROKEN)
+          - play:
+              long: play
+              help: Play entire recording through, then quit (useful with --save)
+          - INPUT:
+              help: The input binary file or directory
+              required: true
+              index: 1
+    - reftest:
+        about: run reftests
+        args:
+          - fuzz_tolerance:
+              long: fuzzy
+              takes_value: true
+              help: Add a minimum fuzziness tolerance to all tests.
+              required: false
+          - REFTEST:
+              help: a specific reftest or directory to run
+              required: false
+              index: 1
+    - rawtest:
+        about: run rawtests
+    - perf:
+        about: run benchmarks
+        args:
+          - filename:
+              help: name of the file to save benchmarks to
+              required: true
+              index: 1
+    - compare_perf:
+        about: compare two benchmark files
+        args:
+          - first_filename:
+              help: first benchmark file to compare
+              required: true
+              index: 1
+          - second_filename:
+              help: second benchmark file to compare
+              required: true
+              index: 2
+    - load:
+        about: load a capture
+        args:
+          - path:
+              help: directory containing the capture
+              takes_value: true
+              required: true
+              index: 1
diff --git a/gfx/wrench/src/binary_frame_reader.rs b/gfx/wrench/src/binary_frame_reader.rs
new file mode 100644
index 000000000000..2efe43039ca9
--- /dev/null
+++ b/gfx/wrench/src/binary_frame_reader.rs
@@ -0,0 +1,237 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use bincode::deserialize;
+use byteorder::{LittleEndian, ReadBytesExt};
+use clap;
+use std::any::TypeId;
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom};
+use std::path::{Path, PathBuf};
+use std::{mem, process};
+use webrender::WEBRENDER_RECORDING_HEADER;
+use webrender::api::{ApiMsg, DocumentMsg};
+use wrench::{Wrench, WrenchThing};
+
+#[derive(Clone)]
+enum Item {
+    Message(ApiMsg),
+    Data(Vec<u8>),
+}
+
+pub struct BinaryFrameReader {
+    file: File,
+    eof: bool,
+    frame_offsets: Vec<u64>,
+
+    skip_uploads: bool,
+    replay_api: bool,
+    play_through: bool,
+
+    frame_data: Vec<Item>,
+    frame_num: u32,
+    frame_read: bool,
+}
+
+impl BinaryFrameReader {
+    pub fn new(file_path: &Path) -> BinaryFrameReader {
+        let mut file = File::open(&file_path).expect("Can't open recording file");
+        let header = file.read_u64::<LittleEndian>().unwrap();
+        assert_eq!(
+            header,
+            WEBRENDER_RECORDING_HEADER,
+            "Binary recording is missing recording header!"
+        );
+
+        let apimsg_type_id = unsafe {
+            assert_eq!(mem::size_of::<TypeId>(), mem::size_of::<u64>());
+            mem::transmute::<TypeId, u64>(TypeId::of::<ApiMsg>())
+        };
+
+        let written_apimsg_type_id = file.read_u64::<LittleEndian>().unwrap();
+        if written_apimsg_type_id != apimsg_type_id {
+            println!(
+                "Warning: binary file ApiMsg type mismatch: expected 0x{:x}, found 0x{:x}",
+                apimsg_type_id,
+                written_apimsg_type_id
+            );
+        }
+
+        BinaryFrameReader {
+            file,
+            eof: false,
+            frame_offsets: vec![],
+
+            skip_uploads: false,
+            replay_api: false,
+            play_through: false,
+
+            frame_data: vec![],
+            frame_num: 0,
+            frame_read: false,
+        }
+    }
+
+    pub fn new_from_args(args: &clap::ArgMatches) -> BinaryFrameReader {
+        let bin_file = args.value_of("INPUT").map(|s| PathBuf::from(s)).unwrap();
+        let mut r = BinaryFrameReader::new(&bin_file);
+        r.skip_uploads = args.is_present("skip-uploads");
+        r.replay_api = args.is_present("api");
+        r.play_through = args.is_present("play");
+        r
+    }
+
+    // FIXME I don't think we can skip uploads without also skipping
+    // payload (I think? Unused payload ranges may also be ignored.). But
+    // either way we'd need to track image updates and deletions -- if we
+    // delete an image, we can't go back to a previous frame.
+    //
+    // We could probably introduce a mode where going backwards replays all
+    // frames up until that frame, so that we know we can be accurate.
+    fn should_skip_upload_msg(&self, msg: &ApiMsg) -> bool {
+        if !self.skip_uploads {
+            return false;
+        }
+
+        match *msg {
+            ApiMsg::UpdateResources(..) => true,
+            _ => false,
+        }
+    }
+
+    // a frame exists if we either haven't hit eof yet, or if
+    // we have, then if we've seen its offset.
+    fn frame_exists(&self, frame: u32) -> bool {
+        !self.eof || (frame as usize) < self.frame_offsets.len()
+    }
+}
+
+impl WrenchThing for BinaryFrameReader {
+    fn do_frame(&mut self, wrench: &mut Wrench) -> u32 {
+        // save where the frame begins as we read through the file
+        if self.frame_num as usize >= self.frame_offsets.len() {
+            self.frame_num = self.frame_offsets.len() as u32;
+            let pos = self.file.seek(SeekFrom::Current(0)).unwrap();
+            println!("Frame {} offset: {}", self.frame_offsets.len(), pos);
+            self.frame_offsets.push(pos);
+        }
+
+        let first_time = !self.frame_read;
+        if first_time {
+            let offset = self.frame_offsets[self.frame_num as usize];
+            self.file.seek(SeekFrom::Start(offset)).unwrap();
+
+            wrench.set_title(&format!("frame {}", self.frame_num));
+
+            self.frame_data.clear();
+            let mut found_frame_marker = false;
+            let mut found_display_list = false;
+            let mut found_pipeline = false;
+            while let Ok(mut len) = self.file.read_u32::<LittleEndian>() {
+                if len > 0 {
+                    let mut buffer = vec![0; len as usize];
+                    self.file.read_exact(&mut buffer).unwrap();
+                    let msg = deserialize(&buffer).unwrap();
+                    let mut store_message = true;
+                    // In order to detect the first valid frame, we
+                    // need to find:
+                    // (a) SetRootPipeline
+                    // (b) SetDisplayList
+                    // (c) GenerateFrame that occurs *after* (a) and (b)
+                    match msg {
+                        ApiMsg::UpdateDocument(_, ref doc_msgs) => {
+                            for doc_msg in doc_msgs {
+                                match *doc_msg {
+                                    DocumentMsg::GenerateFrame => {
+                                        found_frame_marker = true;
+                                    }
+                                    DocumentMsg::SetDisplayList { .. } => {
+                                        found_frame_marker = false;
+                                        found_display_list = true;
+                                    }
+                                    DocumentMsg::SetRootPipeline(..) => {
+                                        found_frame_marker = false;
+                                        found_pipeline = true;
+                                    }
+                                    _ => {}
+                                }
+                            }
+                        }
+                        // Wrench depends on the document always existing
+                        ApiMsg::DeleteDocument(_) => {
+                            store_message = false;
+                        }
+                        _ => {}
+                    }
+                    if store_message {
+                        self.frame_data.push(Item::Message(msg));
+                    }
+                    // Frames are marked by the GenerateFrame message.
+                    // On the first frame, we additionally need to find at least
+                    // a SetDisplayList and a SetRootPipeline.
+                    // After the first frame, any GenerateFrame message marks a new
+                    // frame being rendered.
+                    if found_frame_marker && (self.frame_num > 0 || (found_display_list && found_pipeline)) {
+                        break;
+                    }
+                } else {
+                    len = self.file.read_u32::<LittleEndian>().unwrap();
+                    let mut buffer = vec![0; len as usize];
+                    self.file.read_exact(&mut buffer).unwrap();
+                    self.frame_data.push(Item::Data(buffer));
+                }
+            }
+
+            if self.eof == false &&
+                self.file.seek(SeekFrom::Current(0)).unwrap() == self.file.metadata().unwrap().len()
+            {
+                self.eof = true;
+            }
+
+            self.frame_read = true;
+        }
+
+        if first_time || self.replay_api {
+            wrench.begin_frame();
+            let frame_items = self.frame_data.clone();
+            for item in frame_items {
+                match item {
+                    Item::Message(msg) => if !self.should_skip_upload_msg(&msg) {
+                        wrench.api.send_message(msg);
+                    },
+                    Item::Data(buf) => {
+                        wrench.api.send_payload(&buf);
+                    }
+                }
+            }
+        } else if self.play_through {
+            if !self.frame_exists(self.frame_num + 1) {
+                process::exit(0);
+            }
+            self.next_frame();
+            self.do_frame(wrench);
+        } else {
+            wrench.refresh();
+        }
+
+        self.frame_num
+    }
+
+    // note that we don't loop here; we could, but that'll require
+    // some tracking to avoid reuploading resources every run.  We
+    // sort of try in should_skip_upload_msg, but this needs work.
+    fn next_frame(&mut self) {
+        if self.frame_exists(self.frame_num + 1) {
+            self.frame_num += 1;
+            self.frame_read = false;
+        }
+    }
+
+    fn prev_frame(&mut self) {
+        if self.frame_num > 0 {
+            self.frame_num -= 1;
+            self.frame_read = false;
+        }
+    }
+}
diff --git a/gfx/wrench/src/blob.rs b/gfx/wrench/src/blob.rs
new file mode 100644
index 000000000000..d67bc2a0595c
--- /dev/null
+++ b/gfx/wrench/src/blob.rs
@@ -0,0 +1,162 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// A very basic BlobImageRenderer that can only render a checkerboard pattern.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::Mutex;
+use webrender::api::*;
+
+// Serialize/deserialze the blob.
+
+pub fn serialize_blob(color: ColorU) -> Vec<u8> {
+    vec![color.r, color.g, color.b, color.a]
+}
+
+fn deserialize_blob(blob: &[u8]) -> Result<ColorU, ()> {
+    let mut iter = blob.iter();
+    return match (iter.next(), iter.next(), iter.next(), iter.next()) {
+        (Some(&r), Some(&g), Some(&b), Some(&a)) => Ok(ColorU::new(r, g, b, a)),
+        (Some(&a), None, None, None) => Ok(ColorU::new(a, a, a, a)),
+        _ => Err(()),
+    };
+}
+
+// This is the function that applies the deserialized drawing commands and generates
+// actual image data.
+fn render_blob(
+    color: ColorU,
+    descriptor: &BlobImageDescriptor,
+    tile: Option<TileOffset>,
+) -> BlobImageResult {
+    // Allocate storage for the result. Right now the resource cache expects the
+    // tiles to have have no stride or offset.
+    let mut texels = Vec::with_capacity((descriptor.width * descriptor.height * 4) as usize);
+
+    // Generate a per-tile pattern to see it in the demo. For a real use case it would not
+    // make sense for the rendered content to depend on its tile.
+    let tile_checker = match tile {
+        Some(tile) => (tile.x % 2 == 0) != (tile.y % 2 == 0),
+        None => true,
+    };
+
+    for y in 0 .. descriptor.height {
+        for x in 0 .. descriptor.width {
+            // Apply the tile's offset. This is important: all drawing commands should be
+            // translated by this offset to give correct results with tiled blob images.
+            let x2 = x + descriptor.offset.x as u32;
+            let y2 = y + descriptor.offset.y as u32;
+
+            // Render a simple checkerboard pattern
+            let checker = if (x2 % 20 >= 10) != (y2 % 20 >= 10) {
+                1
+            } else {
+                0
+            };
+            // ..nested in the per-tile cherkerboard pattern
+            let tc = if tile_checker { 0 } else { (1 - checker) * 40 };
+
+            match descriptor.format {
+                ImageFormat::BGRA8 => {
+                    texels.push(color.b * checker + tc);
+                    texels.push(color.g * checker + tc);
+                    texels.push(color.r * checker + tc);
+                    texels.push(color.a * checker + tc);
+                }
+                ImageFormat::R8 => {
+                    texels.push(color.a * checker + tc);
+                }
+                _ => {
+                    return Err(BlobImageError::Other(
+                        format!("Usupported image format {:?}", descriptor.format),
+                    ));
+                }
+            }
+        }
+    }
+
+    Ok(RasterizedBlobImage {
+        data: texels,
+        width: descriptor.width,
+        height: descriptor.height,
+    })
+}
+
+pub struct BlobCallbacks {
+    pub request: Box<Fn(&BlobImageRequest) + Send + 'static>,
+    pub resolve: Box<Fn() + Send + 'static>,
+}
+
+impl BlobCallbacks {
+    pub fn new() -> Self {
+        BlobCallbacks { request: Box::new(|_|()), resolve: Box::new(|| (())) }
+    }
+}
+
+pub struct CheckerboardRenderer {
+    image_cmds: HashMap<ImageKey, ColorU>,
+    callbacks: Arc<Mutex<BlobCallbacks>>,
+
+    // The images rendered in the current frame (not kept here between frames).
+    rendered_images: HashMap<BlobImageRequest, BlobImageResult>,
+}
+
+impl CheckerboardRenderer {
+    pub fn new(callbacks: Arc<Mutex<BlobCallbacks>>) -> Self {
+        CheckerboardRenderer {
+            callbacks,
+            image_cmds: HashMap::new(),
+            rendered_images: HashMap::new(),
+        }
+    }
+}
+
+impl BlobImageRenderer for CheckerboardRenderer {
+    fn add(&mut self, key: ImageKey, cmds: BlobImageData, _: Option<TileSize>) {
+        self.image_cmds
+            .insert(key, deserialize_blob(&cmds[..]).unwrap());
+    }
+
+    fn update(&mut self, key: ImageKey, cmds: BlobImageData, _dirty_rect: Option<DeviceUintRect>) {
+        // Here, updating is just replacing the current version of the commands with
+        // the new one (no incremental updates).
+        self.image_cmds
+            .insert(key, deserialize_blob(&cmds[..]).unwrap());
+    }
+
+    fn delete(&mut self, key: ImageKey) {
+        self.image_cmds.remove(&key);
+    }
+
+    fn request(
+        &mut self,
+        _resources: &BlobImageResources,
+        request: BlobImageRequest,
+        descriptor: &BlobImageDescriptor,
+        _dirty_rect: Option<DeviceUintRect>,
+    ) {
+        (self.callbacks.lock().unwrap().request)(&request);
+        assert!(!self.rendered_images.contains_key(&request));
+        // This method is where we kick off our rendering jobs.
+        // It should avoid doing work on the calling thread as much as possible.
+        // In this example we will use the thread pool to render individual tiles.
+
+        // Gather the input data to send to a worker thread.
+        let cmds = self.image_cmds.get(&request.key).unwrap();
+
+        let result = render_blob(*cmds, descriptor, request.tile);
+
+        self.rendered_images.insert(request, result);
+    }
+
+    fn resolve(&mut self, request: BlobImageRequest) -> BlobImageResult {
+        (self.callbacks.lock().unwrap().resolve)();
+        self.rendered_images.remove(&request).unwrap()
+    }
+
+    fn delete_font(&mut self, _key: FontKey) {}
+
+    fn delete_font_instance(&mut self, _key: FontInstanceKey) {}
+}
diff --git a/gfx/wrench/src/cgfont_to_data.rs b/gfx/wrench/src/cgfont_to_data.rs
new file mode 100644
index 000000000000..5c63bef69f05
--- /dev/null
+++ b/gfx/wrench/src/cgfont_to_data.rs
@@ -0,0 +1,124 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use byteorder::{BigEndian, ByteOrder, ReadBytesExt, WriteBytesExt};
+use core_foundation::data::CFData;
+use core_graphics::font::CGFont;
+use std;
+use std::io::Cursor;
+use std::io::Read;
+use std::io::Write;
+
+
+fn calc_table_checksum<D: Read>(mut data: D) -> u32 {
+    let mut sum: u32 = 0;
+
+    while let Ok(x) = data.read_u32::<BigEndian>() {
+        sum = sum.wrapping_add(x);
+    }
+    // read the remaining bytes
+    let mut buf: [u8; 4] = [0; 4];
+    data.read(&mut buf).unwrap();
+    // if there was nothing left in buf we'll just read a 0
+    // which won't affect the checksum
+    sum = sum.wrapping_add(BigEndian::read_u32(&buf));
+    sum
+}
+
+fn max_pow_2_less_than(a: u16) -> u16 {
+    let x = 1;
+    let mut shift = 0;
+    while (x << (shift + 1)) < a {
+        shift += 1;
+    }
+    shift
+}
+
+struct TableRecord {
+    tag: u32,
+    checksum: u32,
+    offset: u32,
+    length: u32,
+    data: CFData
+}
+
+const CFF_TAG: u32 = 0x43464620; // 'CFF '
+const HEAD_TAG: u32 = 0x68656164; // 'head'
+const OTTO_TAG: u32 = 0x4f54544f; // 'OTTO'
+const TRUE_TAG: u32 = 0x00010000;
+
+pub fn font_to_data(font: CGFont) -> Result<Vec<u8>, std::io::Error> {
+    // We'll reconstruct a TTF font from the tables we can get from the CGFont
+    let mut cff = false;
+    let tags = font.copy_table_tags();
+    let count = tags.len() as u16;
+
+    let mut records = Vec::new();
+    let mut offset: u32 = 0;
+    offset += 4 * 3;
+    offset += 4 * 4 * (count as u32);
+    for tag in tags.iter() {
+        if tag == CFF_TAG {
+            cff = true;
+        }
+        let data = font.copy_table_for_tag(tag).unwrap();
+        let length = data.len() as u32;
+        let checksum;
+        if tag == HEAD_TAG {
+            // we need to skip the checksum field
+            checksum = calc_table_checksum(&data.bytes()[0..2])
+                .wrapping_add(calc_table_checksum(&data.bytes()[3..]))
+        } else {
+            checksum = calc_table_checksum(data.bytes());
+        }
+        records.push(TableRecord { tag, offset, data, length, checksum } );
+        offset += length;
+        // 32 bit align the tables
+        offset = (offset + 3) & !3;
+    }
+
+    let mut buf = Vec::new();
+    if cff {
+        buf.write_u32::<BigEndian>(OTTO_TAG)?;
+    } else {
+        buf.write_u32::<BigEndian>(TRUE_TAG)?;
+    }
+
+    buf.write_u16::<BigEndian>(count)?;
+    buf.write_u16::<BigEndian>((1 << max_pow_2_less_than(count)) * 16)?;
+    buf.write_u16::<BigEndian>(max_pow_2_less_than(count))?;
+    buf.write_u16::<BigEndian>(count * 16 - ((1 << max_pow_2_less_than(count)) * 16))?;
+
+    // write table record entries
+    for r in &records {
+        buf.write_u32::<BigEndian>(r.tag)?;
+        buf.write_u32::<BigEndian>(r.checksum)?;
+        buf.write_u32::<BigEndian>(r.offset)?;
+        buf.write_u32::<BigEndian>(r.length)?;
+    }
+
+    // write tables
+    let mut check_sum_adjustment_offset = 0;
+    for r in &records {
+        if r.tag == 0x68656164 {
+            check_sum_adjustment_offset = buf.len() + 2 * 4;
+        }
+        buf.write(r.data.bytes())?;
+        // 32 bit align the tables
+        while buf.len() & 3 != 0 {
+            buf.push(0);
+        }
+    }
+
+    let mut c = Cursor::new(buf);
+    c.set_position(check_sum_adjustment_offset as u64);
+    // clear the checksumAdjust field before checksumming the whole font
+    c.write_u32::<BigEndian>(0)?;
+    let sum = 0xb1b0afba_u32.wrapping_sub(calc_table_checksum(&c.get_mut()[..]));
+    // set checkSumAdjust to the computed checksum
+    c.set_position(check_sum_adjustment_offset as u64);
+    c.write_u32::<BigEndian>(sum)?;
+
+    Ok(c.into_inner())
+}
diff --git a/gfx/wrench/src/json_frame_writer.rs b/gfx/wrench/src/json_frame_writer.rs
new file mode 100644
index 000000000000..1ecacf3a438c
--- /dev/null
+++ b/gfx/wrench/src/json_frame_writer.rs
@@ -0,0 +1,317 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// the json code is largely unfinished; allow these to silence a bunch of warnings
+#![allow(unused_variables)]
+#![allow(dead_code)]
+
+use app_units::Au;
+use image::{save_buffer, ColorType};
+use premultiply::unpremultiply;
+use serde_json;
+use std::collections::HashMap;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::{fmt, fs};
+use super::CURRENT_FRAME_NUMBER;
+use time;
+use webrender;
+use webrender::api::*;
+use webrender::api::channel::Payload;
+
+enum CachedFont {
+    Native(NativeFontHandle),
+    Raw(Option<Vec<u8>>, u32, Option<PathBuf>),
+}
+
+struct CachedFontInstance {
+    font_key: FontKey,
+    glyph_size: Au,
+}
+
+struct CachedImage {
+    width: u32,
+    height: u32,
+    stride: u32,
+    format: ImageFormat,
+    bytes: Option<Vec<u8>>,
+    path: Option<PathBuf>,
+}
+
+pub struct JsonFrameWriter {
+    frame_base: PathBuf,
+    rsrc_base: PathBuf,
+    rsrc_prefix: String,
+    next_rsrc_num: u32,
+    images: HashMap<ImageKey, CachedImage>,
+    fonts: HashMap<FontKey, CachedFont>,
+    font_instances: HashMap<FontInstanceKey, CachedFontInstance>,
+
+    last_frame_written: u32,
+
+    dl_descriptor: Option<BuiltDisplayListDescriptor>,
+}
+
+impl JsonFrameWriter {
+    pub fn new(path: &Path) -> Self {
+        let mut rsrc_base = path.to_owned();
+        rsrc_base.push("res");
+        fs::create_dir_all(&rsrc_base).ok();
+
+        let rsrc_prefix = format!("{}", time::get_time().sec);
+
+        JsonFrameWriter {
+            frame_base: path.to_owned(),
+            rsrc_base,
+            rsrc_prefix,
+            next_rsrc_num: 1,
+            images: HashMap::new(),
+            fonts: HashMap::new(),
+            font_instances: HashMap::new(),
+
+            dl_descriptor: None,
+
+            last_frame_written: u32::max_value(),
+        }
+    }
+
+    pub fn begin_write_display_list(
+        &mut self,
+        _: &Epoch,
+        _: &PipelineId,
+        _: &Option<ColorF>,
+        _: &LayoutSize,
+        display_list: &BuiltDisplayListDescriptor,
+    ) {
+        unsafe {
+            if CURRENT_FRAME_NUMBER == self.last_frame_written {
+                return;
+            }
+            self.last_frame_written = CURRENT_FRAME_NUMBER;
+        }
+
+        self.dl_descriptor = Some(display_list.clone());
+    }
+
+    pub fn finish_write_display_list(&mut self, frame: u32, data: &[u8]) {
+        let payload = Payload::from_data(data);
+        let dl_desc = self.dl_descriptor.take().unwrap();
+
+        let dl = BuiltDisplayList::from_data(payload.display_list_data, dl_desc);
+
+        let mut frame_file_name = self.frame_base.clone();
+        let current_shown_frame = unsafe { CURRENT_FRAME_NUMBER };
+        frame_file_name.push(format!("frame-{}.json", current_shown_frame));
+
+        let mut file = fs::File::create(&frame_file_name).unwrap();
+
+        let s = serde_json::to_string_pretty(&dl).unwrap();
+        file.write_all(&s.into_bytes()).unwrap();
+        file.write_all(b"\n").unwrap();
+    }
+
+    fn update_resources(&mut self, updates: &ResourceUpdates) {
+        for update in &updates.updates {
+            match *update {
+                ResourceUpdate::AddImage(ref img) => {
+                    let stride = img.descriptor.stride.unwrap_or(
+                        img.descriptor.width * img.descriptor.format.bytes_per_pixel(),
+                    );
+                    let bytes = match img.data {
+                        ImageData::Raw(ref v) => (**v).clone(),
+                        ImageData::External(_) | ImageData::Blob(_) => {
+                            return;
+                        }
+                    };
+                    self.images.insert(
+                        img.key,
+                        CachedImage {
+                            width: img.descriptor.width,
+                            height: img.descriptor.height,
+                            stride,
+                            format: img.descriptor.format,
+                            bytes: Some(bytes),
+                            path: None,
+                        },
+                    );
+                }
+                ResourceUpdate::UpdateImage(ref img) => {
+                    if let Some(ref mut data) = self.images.get_mut(&img.key) {
+                        assert_eq!(data.width, img.descriptor.width);
+                        assert_eq!(data.height, img.descriptor.height);
+                        assert_eq!(data.format, img.descriptor.format);
+
+                        if let ImageData::Raw(ref bytes) = img.data {
+                            data.path = None;
+                            data.bytes = Some((**bytes).clone());
+                        } else {
+                            // Other existing image types only make sense within the gecko integration.
+                            println!(
+                                "Wrench only supports updating buffer images ({}).",
+                                "ignoring update commands"
+                            );
+                        }
+                    }
+                }
+                ResourceUpdate::DeleteImage(img) => {
+                    self.images.remove(&img);
+                }
+                ResourceUpdate::AddFont(ref font) => match font {
+                    &AddFont::Raw(key, ref bytes, index) => {
+                        self.fonts
+                            .insert(key, CachedFont::Raw(Some(bytes.clone()), index, None));
+                    }
+                    &AddFont::Native(key, ref handle) => {
+                        self.fonts.insert(key, CachedFont::Native(handle.clone()));
+                    }
+                },
+                ResourceUpdate::DeleteFont(_) => {}
+                ResourceUpdate::AddFontInstance(ref instance) => {
+                    self.font_instances.insert(
+                        instance.key,
+                        CachedFontInstance {
+                            font_key: instance.font_key,
+                            glyph_size: instance.glyph_size,
+                        },
+                    );
+                }
+                ResourceUpdate::DeleteFontInstance(_) => {}
+            }
+        }
+    }
+
+    fn next_rsrc_paths(
+        prefix: &str,
+        counter: &mut u32,
+        base_path: &Path,
+        base: &str,
+        ext: &str,
+    ) -> (PathBuf, PathBuf) {
+        let mut path_file = base_path.to_owned();
+        let mut path = PathBuf::from("res");
+
+        let fstr = format!("{}-{}-{}.{}", prefix, base, counter, ext);
+        path_file.push(&fstr);
+        path.push(&fstr);
+
+        *counter += 1;
+
+        (path_file, path)
+    }
+
+    fn path_for_image(&mut self, key: ImageKey) -> Option<PathBuf> {
+        if let Some(ref mut data) = self.images.get_mut(&key) {
+            if data.path.is_some() {
+                return data.path.clone();
+            }
+        } else {
+            return None;
+        };
+
+        // Remove the data to munge it
+        let mut data = self.images.remove(&key).unwrap();
+        let mut bytes = data.bytes.take().unwrap();
+        let (path_file, path) = Self::next_rsrc_paths(
+            &self.rsrc_prefix,
+            &mut self.next_rsrc_num,
+            &self.rsrc_base,
+            "img",
+            "png",
+        );
+
+        let ok = match data.format {
+            ImageFormat::BGRA8 => if data.stride == data.width * 4 {
+                unpremultiply(bytes.as_mut_slice());
+                save_buffer(
+                    &path_file,
+                    &bytes,
+                    data.width,
+                    data.height,
+                    ColorType::RGBA(8),
+                ).unwrap();
+                true
+            } else {
+                false
+            },
+            ImageFormat::R8 => if data.stride == data.width {
+                save_buffer(
+                    &path_file,
+                    &bytes,
+                    data.width,
+                    data.height,
+                    ColorType::Gray(8),
+                ).unwrap();
+                true
+            } else {
+                false
+            },
+            _ => false,
+        };
+
+        if !ok {
+            println!(
+                "Failed to write image with format {:?}, dimensions {}x{}, stride {}",
+                data.format,
+                data.width,
+                data.height,
+                data.stride
+            );
+            return None;
+        }
+
+        data.path = Some(path.clone());
+        // put it back
+        self.images.insert(key, data);
+        Some(path)
+    }
+}
+
+impl fmt::Debug for JsonFrameWriter {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "JsonFrameWriter")
+    }
+}
+
+impl webrender::ApiRecordingReceiver for JsonFrameWriter {
+    fn write_msg(&mut self, _: u32, msg: &ApiMsg) {
+        match *msg {
+            ApiMsg::UpdateResources(ref updates) => self.update_resources(updates),
+
+            ApiMsg::UpdateDocument(_, ref doc_msgs) => {
+                for doc_msg in doc_msgs {
+                    match *doc_msg {
+                        DocumentMsg::UpdateResources(ref resources) => {
+                            self.update_resources(resources);
+                        }
+                        DocumentMsg::SetDisplayList {
+                            ref epoch,
+                            ref pipeline_id,
+                            ref background,
+                            ref viewport_size,
+                            ref list_descriptor,
+                            ..
+                        } => {
+                            self.begin_write_display_list(
+                                epoch,
+                                pipeline_id,
+                                background,
+                                viewport_size,
+                                list_descriptor,
+                            );
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            ApiMsg::CloneApi(..) => {}
+            _ => {}
+        }
+    }
+
+    fn write_payload(&mut self, frame: u32, data: &[u8]) {
+        if self.dl_descriptor.is_some() {
+            self.finish_write_display_list(frame, data);
+        }
+    }
+}
diff --git a/gfx/wrench/src/main.rs b/gfx/wrench/src/main.rs
new file mode 100644
index 000000000000..1126b4b24cee
--- /dev/null
+++ b/gfx/wrench/src/main.rs
@@ -0,0 +1,597 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+extern crate app_units;
+extern crate base64;
+extern crate bincode;
+extern crate byteorder;
+#[macro_use]
+extern crate clap;
+#[cfg(target_os = "macos")]
+extern crate core_foundation;
+#[cfg(target_os = "macos")]
+extern crate core_graphics;
+extern crate crossbeam;
+#[cfg(target_os = "windows")]
+extern crate dwrote;
+#[cfg(feature = "logging")]
+extern crate env_logger;
+extern crate euclid;
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+extern crate font_loader;
+extern crate gleam;
+extern crate glutin;
+extern crate image;
+#[macro_use]
+extern crate lazy_static;
+#[macro_use]
+extern crate log;
+#[cfg(feature = "headless")]
+extern crate osmesa_sys;
+extern crate ron;
+#[macro_use]
+extern crate serde;
+extern crate serde_json;
+extern crate time;
+extern crate webrender;
+extern crate yaml_rust;
+
+mod binary_frame_reader;
+mod blob;
+mod json_frame_writer;
+mod parse_function;
+mod perf;
+mod png;
+mod premultiply;
+mod rawtest;
+mod reftest;
+mod ron_frame_writer;
+mod scene;
+mod wrench;
+mod yaml_frame_reader;
+mod yaml_frame_writer;
+mod yaml_helper;
+#[cfg(target_os = "macos")]
+mod cgfont_to_data;
+
+use binary_frame_reader::BinaryFrameReader;
+use gleam::gl;
+use glutin::{ElementState, VirtualKeyCode, WindowProxy};
+use perf::PerfHarness;
+use png::save_flipped;
+use rawtest::RawtestHarness;
+use reftest::{ReftestHarness, ReftestOptions};
+#[cfg(feature = "headless")]
+use std::ffi::CString;
+#[cfg(feature = "headless")]
+use std::mem;
+use std::os::raw::c_void;
+use std::path::{Path, PathBuf};
+use std::ptr;
+use std::rc::Rc;
+use std::sync::mpsc::{channel, Sender, Receiver};
+use webrender::DebugFlags;
+use webrender::api::*;
+use wrench::{Wrench, WrenchThing};
+use yaml_frame_reader::YamlFrameReader;
+
+lazy_static! {
+    static ref PLATFORM_DEFAULT_FACE_NAME: String = String::from("Arial");
+    static ref WHITE_COLOR: ColorF = ColorF::new(1.0, 1.0, 1.0, 1.0);
+    static ref BLACK_COLOR: ColorF = ColorF::new(0.0, 0.0, 0.0, 1.0);
+}
+
+pub static mut CURRENT_FRAME_NUMBER: u32 = 0;
+
+#[cfg(feature = "headless")]
+pub struct HeadlessContext {
+    width: u32,
+    height: u32,
+    _context: osmesa_sys::OSMesaContext,
+    _buffer: Vec<u32>,
+}
+
+#[cfg(not(feature = "headless"))]
+pub struct HeadlessContext {
+    width: u32,
+    height: u32,
+}
+
+impl HeadlessContext {
+    #[cfg(feature = "headless")]
+    fn new(width: u32, height: u32) -> HeadlessContext {
+        let mut attribs = Vec::new();
+
+        attribs.push(osmesa_sys::OSMESA_PROFILE);
+        attribs.push(osmesa_sys::OSMESA_CORE_PROFILE);
+        attribs.push(osmesa_sys::OSMESA_CONTEXT_MAJOR_VERSION);
+        attribs.push(3);
+        attribs.push(osmesa_sys::OSMESA_CONTEXT_MINOR_VERSION);
+        attribs.push(3);
+        attribs.push(osmesa_sys::OSMESA_DEPTH_BITS);
+        attribs.push(24);
+        attribs.push(0);
+
+        let context =
+            unsafe { osmesa_sys::OSMesaCreateContextAttribs(attribs.as_ptr(), ptr::null_mut()) };
+
+        assert!(!context.is_null());
+
+        let mut buffer = vec![0; (width * height) as usize];
+
+        unsafe {
+            let ret = osmesa_sys::OSMesaMakeCurrent(
+                context,
+                buffer.as_mut_ptr() as *mut _,
+                gl::UNSIGNED_BYTE,
+                width as i32,
+                height as i32,
+            );
+            assert!(ret != 0);
+        };
+
+        HeadlessContext {
+            width,
+            height,
+            _context: context,
+            _buffer: buffer,
+        }
+    }
+
+    #[cfg(not(feature = "headless"))]
+    fn new(width: u32, height: u32) -> HeadlessContext {
+        HeadlessContext { width, height }
+    }
+
+    #[cfg(feature = "headless")]
+    fn get_proc_address(s: &str) -> *const c_void {
+        let c_str = CString::new(s).expect("Unable to create CString");
+        unsafe { mem::transmute(osmesa_sys::OSMesaGetProcAddress(c_str.as_ptr())) }
+    }
+
+    #[cfg(not(feature = "headless"))]
+    fn get_proc_address(_: &str) -> *const c_void {
+        ptr::null() as *const _
+    }
+}
+
+pub enum WindowWrapper {
+    Window(glutin::Window, Rc<gl::Gl>),
+    Headless(HeadlessContext, Rc<gl::Gl>),
+}
+
+pub struct HeadlessEventIterater;
+
+impl WindowWrapper {
+    fn swap_buffers(&self) {
+        match *self {
+            WindowWrapper::Window(ref window, _) => window.swap_buffers().unwrap(),
+            WindowWrapper::Headless(..) => {}
+        }
+    }
+
+    fn get_inner_size(&self) -> DeviceUintSize {
+        let (w, h) = match *self {
+            WindowWrapper::Window(ref window, _) => window.get_inner_size_pixels().unwrap(),
+            WindowWrapper::Headless(ref context, _) => (context.width, context.height),
+        };
+        DeviceUintSize::new(w, h)
+    }
+
+    fn hidpi_factor(&self) -> f32 {
+        match *self {
+            WindowWrapper::Window(ref window, _) => window.hidpi_factor(),
+            WindowWrapper::Headless(..) => 1.0,
+        }
+    }
+
+    fn resize(&mut self, size: DeviceUintSize) {
+        match *self {
+            WindowWrapper::Window(ref mut window, _) => window.set_inner_size(size.width, size.height),
+            WindowWrapper::Headless(_, _) => unimplemented!(), // requites Glutin update
+        }
+    }
+
+    fn create_window_proxy(&mut self) -> Option<WindowProxy> {
+        match *self {
+            WindowWrapper::Window(ref window, _) => Some(window.create_window_proxy()),
+            WindowWrapper::Headless(..) => None,
+        }
+    }
+
+    fn set_title(&mut self, title: &str) {
+        match *self {
+            WindowWrapper::Window(ref window, _) => window.set_title(title),
+            WindowWrapper::Headless(..) => (),
+        }
+    }
+
+    pub fn gl(&self) -> &gl::Gl {
+        match *self {
+            WindowWrapper::Window(_, ref gl) | WindowWrapper::Headless(_, ref gl) => &**gl,
+        }
+    }
+
+    pub fn clone_gl(&self) -> Rc<gl::Gl> {
+        match *self {
+            WindowWrapper::Window(_, ref gl) | WindowWrapper::Headless(_, ref gl) => gl.clone(),
+        }
+    }
+}
+
+fn make_window(
+    size: DeviceUintSize,
+    dp_ratio: Option<f32>,
+    vsync: bool,
+    headless: bool,
+) -> WindowWrapper {
+    let wrapper = if headless {
+        let gl = match gl::GlType::default() {
+            gl::GlType::Gl => unsafe {
+                gl::GlFns::load_with(|symbol| {
+                    HeadlessContext::get_proc_address(symbol) as *const _
+                })
+            },
+            gl::GlType::Gles => unsafe {
+                gl::GlesFns::load_with(|symbol| {
+                    HeadlessContext::get_proc_address(symbol) as *const _
+                })
+            },
+        };
+        WindowWrapper::Headless(HeadlessContext::new(size.width, size.height), gl)
+    } else {
+        let mut builder = glutin::WindowBuilder::new()
+            .with_gl(glutin::GlRequest::GlThenGles {
+                opengl_version: (3, 2),
+                opengles_version: (3, 0),
+            })
+            .with_dimensions(size.width, size.height);
+        builder.opengl.vsync = vsync;
+        let window = builder.build().unwrap();
+        unsafe {
+            window
+                .make_current()
+                .expect("unable to make context current!");
+        }
+
+        let gl = match window.get_api() {
+            glutin::Api::OpenGl => unsafe {
+                gl::GlFns::load_with(|symbol| window.get_proc_address(symbol) as *const _)
+            },
+            glutin::Api::OpenGlEs => unsafe {
+                gl::GlesFns::load_with(|symbol| window.get_proc_address(symbol) as *const _)
+            },
+            glutin::Api::WebGl => unimplemented!(),
+        };
+        WindowWrapper::Window(window, gl)
+    };
+
+    wrapper.gl().clear_color(0.3, 0.0, 0.0, 1.0);
+
+    let gl_version = wrapper.gl().get_string(gl::VERSION);
+    let gl_renderer = wrapper.gl().get_string(gl::RENDERER);
+
+    let dp_ratio = dp_ratio.unwrap_or(wrapper.hidpi_factor());
+    println!("OpenGL version {}, {}", gl_version, gl_renderer);
+    println!(
+        "hidpi factor: {} (native {})",
+        dp_ratio,
+        wrapper.hidpi_factor()
+    );
+
+    wrapper
+}
+
+struct Notifier {
+    tx: Sender<()>,
+}
+
+// setup a notifier so we can wait for frames to be finished
+impl RenderNotifier for Notifier {
+    fn clone(&self) -> Box<RenderNotifier> {
+        Box::new(Notifier {
+            tx: self.tx.clone(),
+        })
+    }
+
+    fn wake_up(&self) {
+        self.tx.send(()).unwrap();
+    }
+
+    fn new_document_ready(&self, _: DocumentId, scrolled: bool, _composite_needed: bool) {
+        if !scrolled {
+            self.wake_up();
+        }
+    }
+}
+
+fn create_notifier() -> (Box<RenderNotifier>, Receiver<()>) {
+    let (tx, rx) = channel();
+    (Box::new(Notifier { tx: tx }), rx)
+}
+
+fn main() {
+    #[cfg(feature = "logging")]
+    env_logger::init().unwrap();
+
+    let args_yaml = load_yaml!("args.yaml");
+    let args = clap::App::from_yaml(args_yaml)
+        .setting(clap::AppSettings::ArgRequiredElseHelp)
+        .get_matches();
+
+    // handle some global arguments
+    let res_path = args.value_of("shaders").map(|s| PathBuf::from(s));
+    let dp_ratio = args.value_of("dp_ratio").map(|v| v.parse::<f32>().unwrap());
+    let save_type = args.value_of("save").map(|s| match s {
+        "yaml" => wrench::SaveType::Yaml,
+        "json" => wrench::SaveType::Json,
+        "ron" => wrench::SaveType::Ron,
+        "binary" => wrench::SaveType::Binary,
+        _ => panic!("Save type must be json, ron, yaml, or binary")
+    });
+    let size = args.value_of("size")
+        .map(|s| if s == "720p" {
+            DeviceUintSize::new(1280, 720)
+        } else if s == "1080p" {
+            DeviceUintSize::new(1920, 1080)
+        } else if s == "4k" {
+            DeviceUintSize::new(3840, 2160)
+        } else {
+            let x = s.find('x').expect(
+                "Size must be specified exactly as 720p, 1080p, 4k, or width x height",
+            );
+            let w = s[0 .. x].parse::<u32>().expect("Invalid size width");
+            let h = s[x + 1 ..].parse::<u32>().expect("Invalid size height");
+            DeviceUintSize::new(w, h)
+        })
+        .unwrap_or(DeviceUintSize::new(1920, 1080));
+    let is_headless = args.is_present("headless");
+    let zoom_factor = args.value_of("zoom").map(|z| z.parse::<f32>().unwrap());
+    let mut window = make_window(size, dp_ratio, args.is_present("vsync"), is_headless);
+    let dp_ratio = dp_ratio.unwrap_or(window.hidpi_factor());
+    let dim = window.get_inner_size();
+
+    let needs_frame_notifier = ["perf", "reftest", "png", "rawtest"]
+        .iter()
+        .any(|s| args.subcommand_matches(s).is_some());
+    let (notifier, rx) = if needs_frame_notifier {
+        let (notifier, rx) = create_notifier();
+        (Some(notifier), Some(rx))
+    } else {
+        (None, None)
+    };
+
+    let mut wrench = Wrench::new(
+        &mut window,
+        res_path,
+        dp_ratio,
+        save_type,
+        dim,
+        args.is_present("rebuild"),
+        args.is_present("no_subpixel_aa"),
+        args.is_present("debug"),
+        args.is_present("verbose"),
+        args.is_present("no_scissor"),
+        args.is_present("no_batch"),
+        args.is_present("precache"),
+        args.is_present("slow_subpixel"),
+        zoom_factor.unwrap_or(1.0),
+        notifier,
+    );
+
+    let mut thing = if let Some(subargs) = args.subcommand_matches("show") {
+        Box::new(YamlFrameReader::new_from_args(subargs)) as Box<WrenchThing>
+    } else if let Some(subargs) = args.subcommand_matches("replay") {
+        Box::new(BinaryFrameReader::new_from_args(subargs)) as Box<WrenchThing>
+    } else if let Some(subargs) = args.subcommand_matches("png") {
+        let surface = match subargs.value_of("surface") {
+            Some("screen") | None => png::ReadSurface::Screen,
+            Some("gpu-cache") => png::ReadSurface::GpuCache,
+            _ => panic!("Unknown surface argument value")
+        };
+        let reader = YamlFrameReader::new_from_args(subargs);
+        png::png(&mut wrench, surface, &mut window, reader, rx.unwrap());
+        wrench.renderer.deinit();
+        return;
+    } else if let Some(subargs) = args.subcommand_matches("reftest") {
+        let dim = window.get_inner_size();
+        let harness = ReftestHarness::new(&mut wrench, &mut window, rx.unwrap());
+        let base_manifest = Path::new("reftests/reftest.list");
+        let specific_reftest = subargs.value_of("REFTEST").map(|x| Path::new(x));
+        let mut reftest_options = ReftestOptions::default();
+        if let Some(allow_max_diff) = subargs.value_of("fuzz_tolerance") {
+            reftest_options.allow_max_difference = allow_max_diff.parse().unwrap_or(1);
+            reftest_options.allow_num_differences = dim.width as usize * dim.height as usize;
+        }
+        harness.run(base_manifest, specific_reftest, &reftest_options);
+        return;
+    } else if let Some(_) = args.subcommand_matches("rawtest") {
+        {
+            let harness = RawtestHarness::new(&mut wrench, &mut window, rx.unwrap());
+            harness.run();
+        }
+        wrench.renderer.deinit();
+        return;
+    } else if let Some(subargs) = args.subcommand_matches("perf") {
+        // Perf mode wants to benchmark the total cost of drawing
+        // a new displaty list each frame.
+        wrench.rebuild_display_lists = true;
+        let harness = PerfHarness::new(&mut wrench, &mut window, rx.unwrap());
+        let base_manifest = Path::new("benchmarks/benchmarks.list");
+        let filename = subargs.value_of("filename").unwrap();
+        harness.run(base_manifest, filename);
+        return;
+    } else if let Some(subargs) = args.subcommand_matches("compare_perf") {
+        let first_filename = subargs.value_of("first_filename").unwrap();
+        let second_filename = subargs.value_of("second_filename").unwrap();
+        perf::compare(first_filename, second_filename);
+        return;
+    } else if let Some(subargs) = args.subcommand_matches("load") {
+        let path = PathBuf::from(subargs.value_of("path").unwrap());
+        let mut documents = wrench.api.load_capture(path);
+        println!("loaded {:?}", documents.iter().map(|cd| cd.document_id).collect::<Vec<_>>());
+        let captured = documents.swap_remove(0);
+        window.resize(captured.window_size);
+        wrench.document_id = captured.document_id;
+        Box::new(captured) as Box<WrenchThing>
+    } else {
+        panic!("Should never have gotten here! {:?}", args);
+    };
+
+    let mut show_help = false;
+    let mut do_loop = false;
+    let mut cpu_profile_index = 0;
+
+    let dim = window.get_inner_size();
+    wrench.update(dim);
+    thing.do_frame(&mut wrench);
+
+    'outer: loop {
+        if let Some(window_title) = wrench.take_title() {
+            window.set_title(&window_title);
+        }
+
+        let mut events = Vec::new();
+
+        match window {
+            WindowWrapper::Headless(..) => {
+                events.push(glutin::Event::Awakened);
+            }
+            WindowWrapper::Window(ref window, _) => {
+                events.push(window.wait_events().next().unwrap());
+                events.extend(window.poll_events());
+            }
+        }
+
+        let mut do_frame = false;
+        let mut do_render = false;
+
+        for event in events {
+            match event {
+                glutin::Event::Closed => {
+                    break 'outer;
+                }
+
+                glutin::Event::Refresh |
+                glutin::Event::Awakened |
+                glutin::Event::Focused(..) |
+                glutin::Event::MouseMoved(..) => {
+                    do_render = true;
+                }
+
+                glutin::Event::KeyboardInput(ElementState::Pressed, _scan_code, Some(vk)) => match vk {
+                    VirtualKeyCode::Escape => {
+                        break 'outer;
+                    }
+                    VirtualKeyCode::P => {
+                        wrench.renderer.toggle_debug_flags(DebugFlags::PROFILER_DBG);
+                        do_render = true;
+                    }
+                    VirtualKeyCode::O => {
+                        wrench.renderer.toggle_debug_flags(DebugFlags::RENDER_TARGET_DBG);
+                        do_render = true;
+                    }
+                    VirtualKeyCode::I => {
+                        wrench.renderer.toggle_debug_flags(DebugFlags::TEXTURE_CACHE_DBG);
+                        do_render = true;
+                    }
+                    VirtualKeyCode::B => {
+                        wrench.renderer.toggle_debug_flags(DebugFlags::ALPHA_PRIM_DBG);
+                        do_render = true;
+                    }
+                    VirtualKeyCode::S => {
+                        wrench.renderer.toggle_debug_flags(DebugFlags::COMPACT_PROFILER);
+                        do_render = true;
+                    }
+                    VirtualKeyCode::Q => {
+                        wrench.renderer.toggle_debug_flags(
+                            DebugFlags::GPU_TIME_QUERIES | DebugFlags::GPU_SAMPLE_QUERIES
+                        );
+                        do_render = true;
+                    }
+                    VirtualKeyCode::R => {
+                        wrench.set_page_zoom(ZoomFactor::new(1.0));
+                        do_frame = true;
+                    }
+                    VirtualKeyCode::M => {
+                        wrench.api.notify_memory_pressure();
+                        do_render = true;
+                    }
+                    VirtualKeyCode::L => {
+                        do_loop = !do_loop;
+                        do_render = true;
+                    }
+                    VirtualKeyCode::Left => {
+                        thing.prev_frame();
+                        do_frame = true;
+                    }
+                    VirtualKeyCode::Right => {
+                        thing.next_frame();
+                        do_frame = true;
+                    }
+                    VirtualKeyCode::H => {
+                        show_help = !show_help;
+                        do_render = true;
+                    }
+                    VirtualKeyCode::T => {
+                        let file_name = format!("profile-{}.json", cpu_profile_index);
+                        wrench.renderer.save_cpu_profile(&file_name);
+                        cpu_profile_index += 1;
+                    }
+                    VirtualKeyCode::C => {
+                        let path = PathBuf::from("../captures/wrench");
+                        wrench.api.save_capture(path, CaptureBits::all());
+                    }
+                    VirtualKeyCode::Up => {
+                        let current_zoom = wrench.get_page_zoom();
+                        let new_zoom_factor = ZoomFactor::new(current_zoom.get() + 0.1);
+
+                        wrench.set_page_zoom(new_zoom_factor);
+                        do_frame = true;
+                    }
+                    VirtualKeyCode::Down => {
+                        let current_zoom = wrench.get_page_zoom();
+                        let new_zoom_factor = ZoomFactor::new((current_zoom.get() - 0.1).max(0.1));
+
+                        wrench.set_page_zoom(new_zoom_factor);
+                        do_frame = true;
+                    }
+                    _ => {}
+                }
+                _ => {}
+            }
+        }
+
+        let dim = window.get_inner_size();
+        wrench.update(dim);
+
+        if do_frame {
+            let frame_num = thing.do_frame(&mut wrench);
+            unsafe {
+                CURRENT_FRAME_NUMBER = frame_num;
+            }
+        }
+
+        if do_render {
+            if show_help {
+                wrench.show_onscreen_help();
+            }
+
+            wrench.render();
+            window.swap_buffers();
+
+            if do_loop {
+                thing.next_frame();
+            }
+        }
+    }
+
+    if is_headless {
+        let rect = DeviceUintRect::new(DeviceUintPoint::zero(), size);
+        let pixels = wrench.renderer.read_pixels_rgba8(rect);
+        save_flipped("screenshot.png", pixels, size);
+    }
+
+    wrench.renderer.deinit();
+}
diff --git a/gfx/wrench/src/parse_function.rs b/gfx/wrench/src/parse_function.rs
new file mode 100644
index 000000000000..2b2328b855cf
--- /dev/null
+++ b/gfx/wrench/src/parse_function.rs
@@ -0,0 +1,121 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use std::str::CharIndices;
+
+// support arguments like '4', 'ab', '4.0'
+fn acceptable_arg_character(c: char) -> bool {
+    c.is_alphanumeric() || c == '.' || c == '-'
+}
+
+// A crapy parser for parsing strings like "translate(1, 3)"
+pub fn parse_function(s: &str) -> (&str, Vec<&str>, &str) {
+    // XXX: This it not particular easy to read. Sorry.
+    struct Parser<'a> {
+        itr: CharIndices<'a>,
+        start: usize,
+        o: Option<(usize, char)>,
+    }
+    impl<'a> Parser<'a> {
+        fn skip_whitespace(&mut self) {
+            while let Some(k) = self.o {
+                if !k.1.is_whitespace() {
+                    break;
+                }
+                self.start = k.0 + k.1.len_utf8();
+                self.o = self.itr.next();
+            }
+        }
+    }
+    let mut c = s.char_indices();
+    let o = c.next();
+    let mut p = Parser {
+        itr: c,
+        start: 0,
+        o: o,
+    };
+
+    p.skip_whitespace();
+
+    let mut end = p.start;
+    while let Some(k) = p.o {
+        if !k.1.is_alphabetic() && k.1 != '_' && k.1 != '-' {
+            break;
+        }
+        end = k.0 + k.1.len_utf8();
+        p.o = p.itr.next();
+    }
+
+    let name = &s[p.start .. end];
+    let mut args = Vec::new();
+
+    p.skip_whitespace();
+
+    if let Some(k) = p.o {
+        if k.1 != '(' {
+            return (name, args, &s[p.start ..]);
+        }
+        p.start = k.0 + k.1.len_utf8();
+        p.o = p.itr.next();
+    }
+
+    loop {
+        p.skip_whitespace();
+
+        let mut end = p.start;
+        let mut bracket_count: i32 = 0;
+        while let Some(k) = p.o {
+            let prev_bracket_count = bracket_count;
+            if k.1 == '[' {
+                bracket_count = bracket_count + 1;
+            } else if k.1 == ']' {
+                bracket_count = bracket_count - 1;
+            }
+
+            if bracket_count < 0 {
+                println!("Unexpected closing bracket");
+                break;
+            }
+
+            let not_in_bracket = bracket_count == 0 && prev_bracket_count == 0;
+            if !acceptable_arg_character(k.1) && not_in_bracket {
+                break;
+            }
+            end = k.0 + k.1.len_utf8();
+            p.o = p.itr.next();
+        }
+
+        args.push(&s[p.start .. end]);
+
+        p.skip_whitespace();
+
+        if let Some(k) = p.o {
+            p.start = k.0 + k.1.len_utf8();
+            p.o = p.itr.next();
+            // unless we find a comma we're done
+            if k.1 != ',' {
+                if k.1 != ')' {
+                    println!("Unexpected closing character: {}", k.1);
+                }
+                break;
+            }
+        } else {
+            break;
+        }
+    }
+    (name, args, &s[p.start ..])
+}
+
+#[test]
+fn test() {
+    assert_eq!(parse_function("rotate(40)").0, "rotate");
+    assert_eq!(parse_function("  rotate(40)").0, "rotate");
+    assert_eq!(parse_function("  rotate  (40)").0, "rotate");
+    assert_eq!(parse_function("  rotate  (  40 )").1[0], "40");
+    assert_eq!(parse_function("rotate(-40.0)").1[0], "-40.0");
+    assert_eq!(parse_function("drop-shadow(0, [1, 2, 3, 4], 5)").1[0], "0");
+    assert_eq!(parse_function("drop-shadow(0, [1, 2, 3, 4], 5)").1[1], "[1, 2, 3, 4]");
+    assert_eq!(parse_function("drop-shadow(0, [1, 2, 3, 4], 5)").1[2], "5");
+    assert_eq!(parse_function("drop-shadow(0, [1, 2, [3, 4]], 5)").1[1], "[1, 2, [3, 4]]");
+}
diff --git a/gfx/wrench/src/perf.rs b/gfx/wrench/src/perf.rs
new file mode 100644
index 000000000000..3d5621b515c4
--- /dev/null
+++ b/gfx/wrench/src/perf.rs
@@ -0,0 +1,279 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use WindowWrapper;
+use serde_json;
+use std::collections::{HashMap, HashSet};
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::io::{Read, Write};
+use std::path::{Path, PathBuf};
+use std::sync::mpsc::Receiver;
+use wrench::{Wrench, WrenchThing};
+use yaml_frame_reader::YamlFrameReader;
+
+const COLOR_DEFAULT: &str = "\x1b[0m";
+const COLOR_RED: &str = "\x1b[31m";
+const COLOR_GREEN: &str = "\x1b[32m";
+const COLOR_MAGENTA: &str = "\x1b[95m";
+
+const MIN_SAMPLE_COUNT: usize = 50;
+const SAMPLE_EXCLUDE_COUNT: usize = 10;
+
+pub struct Benchmark {
+    pub test: PathBuf,
+}
+
+pub struct BenchmarkManifest {
+    pub benchmarks: Vec<Benchmark>,
+}
+
+impl BenchmarkManifest {
+    pub fn new(manifest: &Path) -> BenchmarkManifest {
+        let dir = manifest.parent().unwrap();
+        let f =
+            File::open(manifest).expect(&format!("couldn't open manifest: {}", manifest.display()));
+        let file = BufReader::new(&f);
+
+        let mut benchmarks = Vec::new();
+
+        for line in file.lines() {
+            let l = line.unwrap();
+
+            // strip the comments
+            let s = &l[0 .. l.find('#').unwrap_or(l.len())];
+            let s = s.trim();
+            if s.is_empty() {
+                continue;
+            }
+
+            let mut items = s.split_whitespace();
+
+            match items.next() {
+                Some("include") => {
+                    let include = dir.join(items.next().unwrap());
+
+                    benchmarks.append(&mut BenchmarkManifest::new(include.as_path()).benchmarks);
+                }
+                Some(name) => {
+                    let test = dir.join(name);
+                    benchmarks.push(Benchmark { test });
+                }
+                _ => panic!(),
+            };
+        }
+
+        BenchmarkManifest {
+            benchmarks: benchmarks,
+        }
+    }
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+struct TestProfile {
+    name: String,
+    backend_time_ns: u64,
+    composite_time_ns: u64,
+    paint_time_ns: u64,
+    draw_calls: usize,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Profile {
+    tests: Vec<TestProfile>,
+}
+
+impl Profile {
+    fn new() -> Profile {
+        Profile { tests: Vec::new() }
+    }
+
+    fn add(&mut self, profile: TestProfile) {
+        self.tests.push(profile);
+    }
+
+    fn save(&self, filename: &str) {
+        let mut file = File::create(&filename).unwrap();
+        let s = serde_json::to_string_pretty(self).unwrap();
+        file.write_all(&s.into_bytes()).unwrap();
+        file.write_all(b"\n").unwrap();
+    }
+
+    fn load(filename: &str) -> Profile {
+        let mut file = File::open(&filename).unwrap();
+        let mut string = String::new();
+        file.read_to_string(&mut string).unwrap();
+        serde_json::from_str(&string).expect("Unable to load profile!")
+    }
+
+    fn build_set_and_map_of_tests(&self) -> (HashSet<String>, HashMap<String, TestProfile>) {
+        let mut hash_set = HashSet::new();
+        let mut hash_map = HashMap::new();
+
+        for test in &self.tests {
+            hash_set.insert(test.name.clone());
+            hash_map.insert(test.name.clone(), test.clone());
+        }
+
+        (hash_set, hash_map)
+    }
+}
+
+pub struct PerfHarness<'a> {
+    wrench: &'a mut Wrench,
+    window: &'a mut WindowWrapper,
+    rx: Receiver<()>,
+}
+
+impl<'a> PerfHarness<'a> {
+    pub fn new(wrench: &'a mut Wrench, window: &'a mut WindowWrapper, rx: Receiver<()>) -> Self {
+        PerfHarness { wrench, window, rx }
+    }
+
+    pub fn run(mut self, base_manifest: &Path, filename: &str) {
+        let manifest = BenchmarkManifest::new(base_manifest);
+
+        let mut profile = Profile::new();
+
+        for t in manifest.benchmarks {
+            let stats = self.render_yaml(t.test.as_path());
+            profile.add(stats);
+        }
+
+        profile.save(filename);
+    }
+
+    fn render_yaml(&mut self, filename: &Path) -> TestProfile {
+        let mut reader = YamlFrameReader::new(filename);
+
+        // Loop until we get a reasonable number of CPU and GPU
+        // frame profiles. Then take the mean.
+        let mut cpu_frame_profiles = Vec::new();
+        let mut gpu_frame_profiles = Vec::new();
+
+        while cpu_frame_profiles.len() < MIN_SAMPLE_COUNT ||
+            gpu_frame_profiles.len() < MIN_SAMPLE_COUNT
+        {
+            reader.do_frame(self.wrench);
+            self.rx.recv().unwrap();
+            self.wrench.render();
+            self.window.swap_buffers();
+            let (cpu_profiles, gpu_profiles) = self.wrench.get_frame_profiles();
+            cpu_frame_profiles.extend(cpu_profiles);
+            gpu_frame_profiles.extend(gpu_profiles);
+        }
+
+        // Ensure the draw calls match in every sample.
+        let draw_calls = cpu_frame_profiles[0].draw_calls;
+        assert!(
+            cpu_frame_profiles
+                .iter()
+                .all(|s| s.draw_calls == draw_calls)
+        );
+
+        let composite_time_ns = extract_sample(&mut cpu_frame_profiles, |a| a.composite_time_ns);
+        let paint_time_ns = extract_sample(&mut gpu_frame_profiles, |a| a.paint_time_ns);
+        let backend_time_ns = extract_sample(&mut cpu_frame_profiles, |a| a.backend_time_ns);
+
+        TestProfile {
+            name: filename.to_str().unwrap().to_string(),
+            composite_time_ns,
+            paint_time_ns,
+            backend_time_ns,
+            draw_calls,
+        }
+    }
+}
+
+fn extract_sample<F, T>(profiles: &mut [T], f: F) -> u64
+where
+    F: Fn(&T) -> u64,
+{
+    let mut samples: Vec<u64> = profiles.iter().map(f).collect();
+    samples.sort();
+    let useful_samples = &samples[SAMPLE_EXCLUDE_COUNT .. samples.len() - SAMPLE_EXCLUDE_COUNT];
+    let total_time: u64 = useful_samples.iter().sum();
+    total_time / useful_samples.len() as u64
+}
+
+fn select_color(base: f32, value: f32) -> &'static str {
+    let tolerance = base * 0.1;
+    if (value - base).abs() < tolerance {
+        COLOR_DEFAULT
+    } else if value > base {
+        COLOR_RED
+    } else {
+        COLOR_GREEN
+    }
+}
+
+pub fn compare(first_filename: &str, second_filename: &str) {
+    let profile0 = Profile::load(first_filename);
+    let profile1 = Profile::load(second_filename);
+
+    let (set0, map0) = profile0.build_set_and_map_of_tests();
+    let (set1, map1) = profile1.build_set_and_map_of_tests();
+
+    print!("+------------------------------------------------");
+    println!("+--------------+------------------+------------------+");
+    print!("|  Test name                                     ");
+    println!("| Draw Calls   | Composite (ms)   | Paint (ms)       |");
+    print!("+------------------------------------------------");
+    println!("+--------------+------------------+------------------+");
+
+    for test_name in set0.symmetric_difference(&set1) {
+        println!(
+            "| {}{:47}{}|{:14}|{:18}|{:18}|",
+            COLOR_MAGENTA,
+            test_name,
+            COLOR_DEFAULT,
+            " -",
+            " -",
+            " -"
+        );
+    }
+
+    for test_name in set0.intersection(&set1) {
+        let test0 = &map0[test_name];
+        let test1 = &map1[test_name];
+
+        let composite_time0 = test0.composite_time_ns as f32 / 1000000.0;
+        let composite_time1 = test1.composite_time_ns as f32 / 1000000.0;
+
+        let paint_time0 = test0.paint_time_ns as f32 / 1000000.0;
+        let paint_time1 = test1.paint_time_ns as f32 / 1000000.0;
+
+        let draw_calls_color = if test0.draw_calls == test1.draw_calls {
+            COLOR_DEFAULT
+        } else if test0.draw_calls > test1.draw_calls {
+            COLOR_GREEN
+        } else {
+            COLOR_RED
+        };
+
+        let composite_time_color = select_color(composite_time0, composite_time1);
+        let paint_time_color = select_color(paint_time0, paint_time1);
+
+        let draw_call_string = format!(" {} -> {}", test0.draw_calls, test1.draw_calls);
+        let composite_time_string = format!(" {:.2} -> {:.2}", composite_time0, composite_time1);
+        let paint_time_string = format!(" {:.2} -> {:.2}", paint_time0, paint_time1);
+
+        println!(
+            "| {:47}|{}{:14}{}|{}{:18}{}|{}{:18}{}|",
+            test_name,
+            draw_calls_color,
+            draw_call_string,
+            COLOR_DEFAULT,
+            composite_time_color,
+            composite_time_string,
+            COLOR_DEFAULT,
+            paint_time_color,
+            paint_time_string,
+            COLOR_DEFAULT
+        );
+    }
+
+    print!("+------------------------------------------------");
+    println!("+--------------+------------------+------------------+");
+}
diff --git a/gfx/wrench/src/png.rs b/gfx/wrench/src/png.rs
new file mode 100644
index 000000000000..725678fe98a9
--- /dev/null
+++ b/gfx/wrench/src/png.rs
@@ -0,0 +1,113 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use WindowWrapper;
+use image::png::PNGEncoder;
+use image::{self, ColorType, GenericImage};
+use std::fs::File;
+use std::path::Path;
+use std::sync::mpsc::Receiver;
+use webrender::api::*;
+use wrench::{Wrench, WrenchThing};
+use yaml_frame_reader::YamlFrameReader;
+
+pub enum ReadSurface {
+    Screen,
+    GpuCache,
+}
+
+pub struct SaveSettings {
+    pub flip_vertical: bool,
+    pub try_crop: bool,
+}
+
+pub fn save<P: Clone + AsRef<Path>>(
+    path: P,
+    orig_pixels: Vec<u8>,
+    mut size: DeviceUintSize,
+    settings: SaveSettings
+) {
+    let mut buffer = image::RgbaImage::from_raw(
+        size.width,
+        size.height,
+        orig_pixels,
+    ).expect("bug: unable to construct image buffer");
+
+    if settings.flip_vertical {
+        // flip image vertically (texture is upside down)
+        buffer = image::imageops::flip_vertical(&buffer);
+    }
+
+    if settings.try_crop {
+        if let Ok(existing_image) = image::open(path.clone()) {
+            let old_dims = existing_image.dimensions();
+            println!("Crop from {:?} to {:?}", size, old_dims);
+            size.width = old_dims.0;
+            size.height = old_dims.1;
+            buffer = image::imageops::crop(
+                &mut buffer,
+                0,
+                0,
+                size.width,
+                size.height
+            ).to_image();
+        }
+    }
+
+    let encoder = PNGEncoder::new(File::create(path).unwrap());
+    encoder
+        .encode(&buffer, size.width, size.height, ColorType::RGBA(8))
+        .expect("Unable to encode PNG!");
+}
+
+pub fn save_flipped<P: Clone + AsRef<Path>>(
+    path: P,
+    orig_pixels: Vec<u8>,
+    size: DeviceUintSize,
+) {
+    save(path, orig_pixels, size, SaveSettings {
+        flip_vertical: true,
+        try_crop: true,
+    })
+}
+
+pub fn png(
+    wrench: &mut Wrench,
+    surface: ReadSurface,
+    window: &mut WindowWrapper,
+    mut reader: YamlFrameReader,
+    rx: Receiver<()>,
+) {
+    reader.do_frame(wrench);
+
+    // wait for the frame
+    rx.recv().unwrap();
+    wrench.render();
+
+    let (device_size, data, settings) = match surface {
+        ReadSurface::Screen => {
+            let dim = window.get_inner_size();
+            let rect = DeviceUintRect::new(DeviceUintPoint::zero(), dim);
+            let data = wrench.renderer
+                .read_pixels_rgba8(rect);
+            (rect.size, data, SaveSettings {
+                flip_vertical: true,
+                try_crop: true,
+            })
+        }
+        ReadSurface::GpuCache => {
+            let (size, data) = wrench.renderer
+                .read_gpu_cache();
+            (size, data, SaveSettings {
+                flip_vertical: false,
+                try_crop: false,
+            })
+        }
+    };
+
+    let mut out_path = reader.yaml_path().clone();
+    out_path.set_extension("png");
+
+    save(out_path, data, device_size, settings);
+}
diff --git a/gfx/wrench/src/premultiply.rs b/gfx/wrench/src/premultiply.rs
new file mode 100644
index 000000000000..ecdcce85e1c4
--- /dev/null
+++ b/gfx/wrench/src/premultiply.rs
@@ -0,0 +1,55 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// These are slow. Gecko's gfx/2d/Swizzle.cpp has better versions
+pub fn premultiply(data: &mut [u8]) {
+    for pixel in data.chunks_mut(4) {
+        let a = pixel[3] as u32;
+        let b = pixel[2] as u32;
+        let g = pixel[1] as u32;
+        let r = pixel[0] as u32;
+
+        pixel[3] = a as u8;
+        pixel[2] = ((r * a + 128) / 255) as u8;
+        pixel[1] = ((g * a + 128) / 255) as u8;
+        pixel[0] = ((b * a + 128) / 255) as u8;
+    }
+}
+
+pub fn unpremultiply(data: &mut [u8]) {
+    for pixel in data.chunks_mut(4) {
+        let a = pixel[3] as u32;
+        let mut b = pixel[2] as u32;
+        let mut g = pixel[1] as u32;
+        let mut r = pixel[0] as u32;
+
+        if a > 0 {
+            r = r * 255 / a;
+            g = g * 255 / a;
+            b = b * 255 / a;
+        }
+
+        pixel[3] = a as u8;
+        pixel[2] = r as u8;
+        pixel[1] = g as u8;
+        pixel[0] = b as u8;
+    }
+}
+
+#[test]
+fn it_works() {
+    let mut f = [0xff, 0xff, 0xff, 0x80, 0x00, 0xff, 0x00, 0x80];
+    premultiply(&mut f);
+    println!("{:?}", f);
+    assert!(
+        f[0] == 0x80 && f[1] == 0x80 && f[2] == 0x80 && f[3] == 0x80 && f[4] == 0x00 &&
+            f[5] == 0x80 && f[6] == 0x00 && f[7] == 0x80
+    );
+    unpremultiply(&mut f);
+    println!("{:?}", f);
+    assert!(
+        f[0] == 0xff && f[1] == 0xff && f[2] == 0xff && f[3] == 0x80 && f[4] == 0x00 &&
+            f[5] == 0xff && f[6] == 0x00 && f[7] == 0x80
+    );
+}
diff --git a/gfx/wrench/src/rawtest.rs b/gfx/wrench/src/rawtest.rs
new file mode 100644
index 000000000000..7933f2575b0b
--- /dev/null
+++ b/gfx/wrench/src/rawtest.rs
@@ -0,0 +1,567 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use WindowWrapper;
+use blob;
+use euclid::{TypedRect, TypedSize2D, TypedPoint2D};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicIsize, Ordering};
+use std::sync::mpsc::Receiver;
+use webrender::api::*;
+use wrench::Wrench;
+
+pub struct RawtestHarness<'a> {
+    wrench: &'a mut Wrench,
+    rx: Receiver<()>,
+    window: &'a mut WindowWrapper,
+}
+
+fn point<T: Copy, U>(x: T, y: T) -> TypedPoint2D<T, U> {
+    TypedPoint2D::new(x, y)
+}
+
+fn size<T: Copy, U>(x: T, y: T) -> TypedSize2D<T, U> {
+    TypedSize2D::new(x, y)
+}
+
+fn rect<T: Copy, U>(x: T, y: T, width: T, height: T) -> TypedRect<T, U> {
+    TypedRect::new(point(x, y), size(width, height))
+}
+
+impl<'a> RawtestHarness<'a> {
+    pub fn new(wrench: &'a mut Wrench, window: &'a mut WindowWrapper, rx: Receiver<()>) -> Self {
+        RawtestHarness {
+            wrench,
+            rx,
+            window,
+        }
+    }
+
+    pub fn run(mut self) {
+        self.test_retained_blob_images_test();
+        self.test_blob_update_test();
+        self.test_blob_update_epoch_test();
+        self.test_tile_decomposition();
+        self.test_save_restore();
+        self.test_capture();
+    }
+
+    fn render_and_get_pixels(&mut self, window_rect: DeviceUintRect) -> Vec<u8> {
+        self.rx.recv().unwrap();
+        self.wrench.render();
+        self.wrench.renderer.read_pixels_rgba8(window_rect)
+    }
+
+    fn submit_dl(
+        &mut self,
+        epoch: &mut Epoch,
+        layout_size: LayoutSize,
+        builder: DisplayListBuilder,
+        resources: Option<ResourceUpdates>
+    ) {
+        let mut txn = Transaction::new();
+        let root_background_color = Some(ColorF::new(1.0, 1.0, 1.0, 1.0));
+        if let Some(resources) = resources {
+            txn.update_resources(resources);
+        }
+        txn.set_display_list(
+            *epoch,
+            root_background_color,
+            layout_size,
+            builder.finalize(),
+            false,
+        );
+        epoch.0 += 1;
+
+        txn.generate_frame();
+        self.wrench.api.send_transaction(self.wrench.document_id, txn);
+    }
+
+    fn test_tile_decomposition(&mut self) {
+        println!("\ttile decomposition...");
+        // This exposes a crash in tile decomposition
+        let layout_size = LayoutSize::new(800., 800.);
+        let mut resources = ResourceUpdates::new();
+
+        let blob_img = self.wrench.api.generate_image_key();
+        resources.add_image(
+            blob_img,
+            ImageDescriptor::new(151, 56, ImageFormat::BGRA8, true),
+            ImageData::new_blob_image(blob::serialize_blob(ColorU::new(50, 50, 150, 255))),
+            Some(128),
+        );
+
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+
+        let info = LayoutPrimitiveInfo::new(rect(448.899994, 74.0, 151.000031, 56.));
+
+        // setup some malicious image size parameters
+        builder.push_image(
+            &info,
+            size(151., 56.0),
+            size(151.0, 56.0),
+            ImageRendering::Auto,
+            AlphaType::PremultipliedAlpha,
+            blob_img,
+        );
+
+        let mut epoch = Epoch(0);
+
+        self.submit_dl(&mut epoch, layout_size, builder, Some(resources));
+
+        self.rx.recv().unwrap();
+        self.wrench.render();
+
+        // Leaving a tiled blob image in the resource cache
+        // confuses the `test_capture`. TODO: remove this
+        resources = ResourceUpdates::new();
+        resources.delete_image(blob_img);
+        self.wrench.api.update_resources(resources);
+    }
+
+    fn test_retained_blob_images_test(&mut self) {
+        println!("\tretained blob images test...");
+        let blob_img;
+        let window_size = self.window.get_inner_size();
+
+        let test_size = DeviceUintSize::new(400, 400);
+
+        let window_rect = DeviceUintRect::new(
+            DeviceUintPoint::new(0, window_size.height - test_size.height),
+            test_size,
+        );
+        let layout_size = LayoutSize::new(400., 400.);
+        let mut resources = ResourceUpdates::new();
+        {
+            let api = &self.wrench.api;
+
+            blob_img = api.generate_image_key();
+            resources.add_image(
+                blob_img,
+                ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+                ImageData::new_blob_image(blob::serialize_blob(ColorU::new(50, 50, 150, 255))),
+                None,
+            );
+        }
+
+        // draw the blob the first time
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+        let info = LayoutPrimitiveInfo::new(rect(0.0, 60.0, 200.0, 200.0));
+
+        builder.push_image(
+            &info,
+            size(200.0, 200.0),
+            size(0.0, 0.0),
+            ImageRendering::Auto,
+            AlphaType::PremultipliedAlpha,
+            blob_img,
+        );
+
+        let mut epoch = Epoch(0);
+
+        self.submit_dl(&mut epoch, layout_size, builder, Some(resources));
+
+        // draw the blob image a second time at a different location
+
+        // make a new display list that refers to the first image
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+        let info = LayoutPrimitiveInfo::new(rect(1.0, 60.0, 200.0, 200.0));
+        builder.push_image(
+            &info,
+            size(200.0, 200.0),
+            size(0.0, 0.0),
+            ImageRendering::Auto,
+            AlphaType::PremultipliedAlpha,
+            blob_img,
+        );
+
+        self.submit_dl(&mut epoch, layout_size, builder, None);
+
+        let called = Arc::new(AtomicIsize::new(0));
+        let called_inner = Arc::clone(&called);
+
+        self.wrench.callbacks.lock().unwrap().request = Box::new(move |_| {
+            called_inner.fetch_add(1, Ordering::SeqCst);
+        });
+
+        let pixels_first = self.render_and_get_pixels(window_rect);
+        assert!(called.load(Ordering::SeqCst) == 1);
+
+        let pixels_second = self.render_and_get_pixels(window_rect);
+
+        // make sure we only requested once
+        assert!(called.load(Ordering::SeqCst) == 1);
+
+        // use png;
+        // png::save_flipped("out1.png", &pixels_first, window_rect.size);
+        // png::save_flipped("out2.png", &pixels_second, window_rect.size);
+
+        assert!(pixels_first != pixels_second);
+    }
+
+    fn test_blob_update_epoch_test(&mut self) {
+        println!("\tblob update epoch test...");
+        let (blob_img, blob_img2);
+        let window_size = self.window.get_inner_size();
+
+        let test_size = DeviceUintSize::new(400, 400);
+
+        let window_rect = DeviceUintRect::new(
+            point(0, window_size.height - test_size.height),
+            test_size,
+        );
+        let layout_size = LayoutSize::new(400., 400.);
+        let mut resources = ResourceUpdates::new();
+        let (blob_img, blob_img2) = {
+            let api = &self.wrench.api;
+
+            blob_img = api.generate_image_key();
+            resources.add_image(
+                blob_img,
+                ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+                ImageData::new_blob_image(blob::serialize_blob(ColorU::new(50, 50, 150, 255))),
+                None,
+            );
+            blob_img2 = api.generate_image_key();
+            resources.add_image(
+                blob_img2,
+                ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+                ImageData::new_blob_image(blob::serialize_blob(ColorU::new(80, 50, 150, 255))),
+                None,
+            );
+            (blob_img, blob_img2)
+        };
+
+        // setup some counters to count how many times each image is requested
+        let img1_requested = Arc::new(AtomicIsize::new(0));
+        let img1_requested_inner = Arc::clone(&img1_requested);
+        let img2_requested = Arc::new(AtomicIsize::new(0));
+        let img2_requested_inner = Arc::clone(&img2_requested);
+
+        // track the number of times that the second image has been requested
+        self.wrench.callbacks.lock().unwrap().request = Box::new(move |&desc| {
+            if desc.key == blob_img {
+                img1_requested_inner.fetch_add(1, Ordering::SeqCst);
+            }
+            if desc.key == blob_img2 {
+                img2_requested_inner.fetch_add(1, Ordering::SeqCst);
+            }
+        });
+
+        // create two blob images and draw them
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+        let info = LayoutPrimitiveInfo::new(rect(0.0, 60.0, 200.0, 200.0));
+        let info2 = LayoutPrimitiveInfo::new(rect(200.0, 60.0, 200.0, 200.0));
+        let push_images = |builder: &mut DisplayListBuilder| {
+            builder.push_image(
+                &info,
+                size(200.0, 200.0),
+                size(0.0, 0.0),
+                ImageRendering::Auto,
+                AlphaType::PremultipliedAlpha,
+                blob_img,
+            );
+            builder.push_image(
+                &info2,
+                size(200.0, 200.0),
+                size(0.0, 0.0),
+                ImageRendering::Auto,
+                AlphaType::PremultipliedAlpha,
+                blob_img2,
+            );
+        };
+
+        push_images(&mut builder);
+
+        let mut epoch = Epoch(0);
+
+        self.submit_dl(&mut epoch, layout_size, builder, Some(resources));
+        let _pixels_first = self.render_and_get_pixels(window_rect);
+
+
+        // update and redraw both images
+        let mut resources = ResourceUpdates::new();
+        resources.update_image(
+            blob_img,
+            ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+            ImageData::new_blob_image(blob::serialize_blob(ColorU::new(50, 50, 150, 255))),
+            Some(rect(100, 100, 100, 100)),
+        );
+        resources.update_image(
+            blob_img2,
+            ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+            ImageData::new_blob_image(blob::serialize_blob(ColorU::new(59, 50, 150, 255))),
+            Some(rect(100, 100, 100, 100)),
+        );
+
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+        push_images(&mut builder);
+        self.submit_dl(&mut epoch, layout_size, builder, Some(resources));
+        let _pixels_second = self.render_and_get_pixels(window_rect);
+
+
+        // only update the first image
+        let mut resources = ResourceUpdates::new();
+        resources.update_image(
+            blob_img,
+            ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+            ImageData::new_blob_image(blob::serialize_blob(ColorU::new(50, 150, 150, 255))),
+            Some(rect(200, 200, 100, 100)),
+        );
+
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+        push_images(&mut builder);
+        self.submit_dl(&mut epoch, layout_size, builder, Some(resources));
+        let _pixels_third = self.render_and_get_pixels(window_rect);
+
+        // the first image should be requested 3 times
+        assert_eq!(img1_requested.load(Ordering::SeqCst), 3);
+        // the second image should've been requested twice
+        assert_eq!(img2_requested.load(Ordering::SeqCst), 2);
+    }
+
+    fn test_blob_update_test(&mut self) {
+        println!("\tblob update test...");
+        let window_size = self.window.get_inner_size();
+
+        let test_size = DeviceUintSize::new(400, 400);
+
+        let window_rect = DeviceUintRect::new(
+            point(0, window_size.height - test_size.height),
+            test_size,
+        );
+        let layout_size = LayoutSize::new(400., 400.);
+        let mut resources = ResourceUpdates::new();
+
+        let blob_img = {
+            let img = self.wrench.api.generate_image_key();
+            resources.add_image(
+                img,
+                ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+                ImageData::new_blob_image(blob::serialize_blob(ColorU::new(50, 50, 150, 255))),
+                None,
+            );
+            img
+        };
+
+        // draw the blobs the first time
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+        let info = LayoutPrimitiveInfo::new(rect(0.0, 60.0, 200.0, 200.0));
+
+        builder.push_image(
+            &info,
+            size(200.0, 200.0),
+            size(0.0, 0.0),
+            ImageRendering::Auto,
+            AlphaType::PremultipliedAlpha,
+            blob_img,
+        );
+
+        let mut epoch = Epoch(0);
+
+        self.submit_dl(&mut epoch, layout_size, builder, Some(resources));
+        let pixels_first = self.render_and_get_pixels(window_rect);
+
+        // draw the blob image a second time after updating it with the same color
+        let mut resources = ResourceUpdates::new();
+        resources.update_image(
+            blob_img,
+            ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+            ImageData::new_blob_image(blob::serialize_blob(ColorU::new(50, 50, 150, 255))),
+            Some(rect(100, 100, 100, 100)),
+        );
+
+        // make a new display list that refers to the first image
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+        let info = LayoutPrimitiveInfo::new(rect(0.0, 60.0, 200.0, 200.0));
+        builder.push_image(
+            &info,
+            size(200.0, 200.0),
+            size(0.0, 0.0),
+            ImageRendering::Auto,
+            AlphaType::PremultipliedAlpha,
+            blob_img,
+        );
+
+        self.submit_dl(&mut epoch, layout_size, builder, Some(resources));
+        let pixels_second = self.render_and_get_pixels(window_rect);
+
+        // draw the blob image a third time after updating it with a different color
+        let mut resources = ResourceUpdates::new();
+        resources.update_image(
+            blob_img,
+            ImageDescriptor::new(500, 500, ImageFormat::BGRA8, true),
+            ImageData::new_blob_image(blob::serialize_blob(ColorU::new(50, 150, 150, 255))),
+            Some(rect(200, 200, 100, 100)),
+        );
+
+        // make a new display list that refers to the first image
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+        let info = LayoutPrimitiveInfo::new(rect(0.0, 60.0, 200.0, 200.0));
+        builder.push_image(
+            &info,
+            size(200.0, 200.0),
+            size(0.0, 0.0),
+            ImageRendering::Auto,
+            AlphaType::PremultipliedAlpha,
+            blob_img,
+        );
+
+        self.submit_dl(&mut epoch, layout_size, builder, Some(resources));
+        let pixels_third = self.render_and_get_pixels(window_rect);
+
+        assert!(pixels_first == pixels_second);
+        assert!(pixels_first != pixels_third);
+    }
+
+    // Ensures that content doing a save-restore produces the same results as not
+    fn test_save_restore(&mut self) {
+        println!("\tsave/restore...");
+        let window_size = self.window.get_inner_size();
+
+        let test_size = DeviceUintSize::new(400, 400);
+
+        let window_rect = DeviceUintRect::new(
+            DeviceUintPoint::new(0, window_size.height - test_size.height),
+            test_size,
+        );
+        let layout_size = LayoutSize::new(400., 400.);
+
+        let mut do_test = |should_try_and_fail| {
+            let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+
+            let clip = builder.define_clip(None, rect(110., 120., 200., 200.),
+                                           None::<ComplexClipRegion>, None);
+            builder.push_clip_id(clip);
+            builder.push_rect(&PrimitiveInfo::new(rect(100., 100., 100., 100.)),
+                              ColorF::new(0.0, 0.0, 1.0, 1.0));
+
+            if should_try_and_fail {
+                builder.save();
+                let clip = builder.define_clip(None, rect(80., 80., 90., 90.),
+                                               None::<ComplexClipRegion>, None);
+                builder.push_clip_id(clip);
+                builder.push_rect(&PrimitiveInfo::new(rect(110., 110., 50., 50.)),
+                              ColorF::new(0.0, 1.0, 0.0, 1.0));
+                builder.push_shadow(&PrimitiveInfo::new(rect(100., 100., 100., 100.)),
+                    Shadow {
+                        offset: LayoutVector2D::new(1.0, 1.0),
+                        blur_radius: 1.0,
+                        color: ColorF::new(0.0, 0.0, 0.0, 1.0),
+                    });
+                builder.push_line(&PrimitiveInfo::new(rect(110., 110., 50., 2.)),
+                                  0.0, LineOrientation::Horizontal,
+                                  &ColorF::new(0.0, 0.0, 0.0, 1.0), LineStyle::Solid);
+                builder.restore();
+            }
+
+            {
+                builder.save();
+                let clip = builder.define_clip(None, rect(80., 80., 100., 100.),
+                                               None::<ComplexClipRegion>, None);
+                builder.push_clip_id(clip);
+                builder.push_rect(&PrimitiveInfo::new(rect(150., 150., 100., 100.)),
+                                  ColorF::new(0.0, 0.0, 1.0, 1.0));
+
+                builder.pop_clip_id();
+                builder.clear_save();
+            }
+
+            builder.pop_clip_id();
+
+            self.submit_dl(&mut Epoch(0), layout_size, builder, None);
+
+            self.render_and_get_pixels(window_rect)
+        };
+
+        let first = do_test(false);
+        let second = do_test(true);
+
+        assert_eq!(first, second);
+    }
+
+    fn test_capture(&mut self) {
+        println!("\tcapture...");
+        let path = "../captures/test";
+        let layout_size = LayoutSize::new(400., 400.);
+        let dim = self.window.get_inner_size();
+        let window_rect = DeviceUintRect::new(
+            point(0, dim.height - layout_size.height as u32),
+            size(layout_size.width as u32, layout_size.height as u32),
+        );
+
+        // 1. render some scene
+
+        let mut resources = ResourceUpdates::new();
+        let image = self.wrench.api.generate_image_key();
+        resources.add_image(
+            image,
+            ImageDescriptor::new(1, 1, ImageFormat::BGRA8, true),
+            ImageData::new(vec![0xFF, 0, 0, 0xFF]),
+            None,
+        );
+
+        let mut builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+
+        builder.push_image(
+            &LayoutPrimitiveInfo::new(rect(300.0, 70.0, 150.0, 50.0)),
+            size(150.0, 50.0),
+            size(0.0, 0.0),
+            ImageRendering::Auto,
+            AlphaType::PremultipliedAlpha,
+            image,
+        );
+
+        let mut txn = Transaction::new();
+
+        txn.set_display_list(
+            Epoch(0),
+            Some(ColorF::new(1.0, 1.0, 1.0, 1.0)),
+            layout_size,
+            builder.finalize(),
+            false,
+        );
+        txn.generate_frame();
+
+        self.wrench.api.send_transaction(self.wrench.document_id, txn);
+
+        let pixels0 = self.render_and_get_pixels(window_rect);
+
+        // 2. capture it
+
+        self.wrench.api.save_capture(path.into(), CaptureBits::all());
+        self.rx.recv().unwrap();
+
+        // 3. set a different scene
+
+        builder = DisplayListBuilder::new(self.wrench.root_pipeline_id, layout_size);
+
+        let mut txn = Transaction::new();
+        txn.set_display_list(
+            Epoch(1),
+            Some(ColorF::new(1.0, 0.0, 0.0, 1.0)),
+            layout_size,
+            builder.finalize(),
+            false,
+        );
+        self.wrench.api.send_transaction(self.wrench.document_id, txn);
+
+        // 4. load the first one
+
+        let mut documents = self.wrench.api.load_capture(path.into());
+        let captured = documents.swap_remove(0);
+
+        // 5. render the built frame and compare
+        let pixels1 = self.render_and_get_pixels(window_rect);
+        assert!(pixels0 == pixels1);
+
+        // 6. rebuild the scene and compare again
+        let mut txn = Transaction::new();
+        txn.set_root_pipeline(captured.root_pipeline_id.unwrap());
+        txn.generate_frame();
+        self.wrench.api.send_transaction(captured.document_id, txn);
+        let pixels2 = self.render_and_get_pixels(window_rect);
+        assert!(pixels0 == pixels2);
+    }
+}
diff --git a/gfx/wrench/src/reftest.rs b/gfx/wrench/src/reftest.rs
new file mode 100644
index 000000000000..791a5212ba0b
--- /dev/null
+++ b/gfx/wrench/src/reftest.rs
@@ -0,0 +1,495 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use WindowWrapper;
+use base64;
+use image::load as load_piston_image;
+use image::png::PNGEncoder;
+use image::{ColorType, ImageFormat};
+use parse_function::parse_function;
+use png::save_flipped;
+use std::cmp;
+use std::fmt::{Display, Error, Formatter};
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::path::{Path, PathBuf};
+use std::sync::mpsc::Receiver;
+use webrender::RendererStats;
+use webrender::api::*;
+use wrench::{Wrench, WrenchThing};
+use yaml_frame_reader::YamlFrameReader;
+
+#[cfg(target_os = "windows")]
+const PLATFORM: &str = "win";
+#[cfg(target_os = "linux")]
+const PLATFORM: &str = "linux";
+#[cfg(target_os = "macos")]
+const PLATFORM: &str = "mac";
+#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
+const PLATFORM: &str = "other";
+
+const OPTION_DISABLE_SUBPX: &str = "disable-subpixel";
+const OPTION_DISABLE_AA: &str = "disable-aa";
+const OPTION_DISABLE_DUAL_SOURCE_BLENDING: &str = "disable-dual-source-blending";
+
+pub struct ReftestOptions {
+    // These override values that are lower.
+    pub allow_max_difference: usize,
+    pub allow_num_differences: usize,
+}
+
+impl ReftestOptions {
+    pub fn default() -> Self {
+        ReftestOptions {
+            allow_max_difference: 0,
+            allow_num_differences: 0,
+        }
+    }
+}
+
+pub enum ReftestOp {
+    Equal,
+    NotEqual,
+}
+
+impl Display for ReftestOp {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(
+            f,
+            "{}",
+            match *self {
+                ReftestOp::Equal => "==".to_owned(),
+                ReftestOp::NotEqual => "!=".to_owned(),
+            }
+        )
+    }
+}
+
+pub struct Reftest {
+    op: ReftestOp,
+    test: PathBuf,
+    reference: PathBuf,
+    font_render_mode: Option<FontRenderMode>,
+    max_difference: usize,
+    num_differences: usize,
+    expected_draw_calls: Option<usize>,
+    expected_alpha_targets: Option<usize>,
+    expected_color_targets: Option<usize>,
+    disable_dual_source_blending: bool,
+    zoom_factor: f32,
+}
+
+impl Display for Reftest {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(
+            f,
+            "{} {} {}",
+            self.test.display(),
+            self.op,
+            self.reference.display()
+        )
+    }
+}
+
+struct ReftestImage {
+    data: Vec<u8>,
+    size: DeviceUintSize,
+}
+enum ReftestImageComparison {
+    Equal,
+    NotEqual {
+        max_difference: usize,
+        count_different: usize,
+    },
+}
+
+impl ReftestImage {
+    fn compare(&self, other: &ReftestImage) -> ReftestImageComparison {
+        assert_eq!(self.size, other.size);
+        assert_eq!(self.data.len(), other.data.len());
+        assert_eq!(self.data.len() % 4, 0);
+
+        let mut count = 0;
+        let mut max = 0;
+
+        for (a, b) in self.data.chunks(4).zip(other.data.chunks(4)) {
+            if a != b {
+                let pixel_max = a.iter()
+                    .zip(b.iter())
+                    .map(|(x, y)| (*x as isize - *y as isize).abs() as usize)
+                    .max()
+                    .unwrap();
+
+                count += 1;
+                max = cmp::max(max, pixel_max);
+            }
+        }
+
+        if count != 0 {
+            ReftestImageComparison::NotEqual {
+                max_difference: max,
+                count_different: count,
+            }
+        } else {
+            ReftestImageComparison::Equal
+        }
+    }
+
+    fn create_data_uri(mut self) -> String {
+        let width = self.size.width;
+        let height = self.size.height;
+
+        // flip image vertically (texture is upside down)
+        let orig_pixels = self.data.clone();
+        let stride = width as usize * 4;
+        for y in 0 .. height as usize {
+            let dst_start = y * stride;
+            let src_start = (height as usize - y - 1) * stride;
+            let src_slice = &orig_pixels[src_start .. src_start + stride];
+            (&mut self.data[dst_start .. dst_start + stride])
+                .clone_from_slice(&src_slice[.. stride]);
+        }
+
+        let mut png: Vec<u8> = vec![];
+        {
+            let encoder = PNGEncoder::new(&mut png);
+            encoder
+                .encode(&self.data[..], width, height, ColorType::RGBA(8))
+                .expect("Unable to encode PNG!");
+        }
+        let png_base64 = base64::encode(&png);
+        format!("data:image/png;base64,{}", png_base64)
+    }
+}
+
+struct ReftestManifest {
+    reftests: Vec<Reftest>,
+}
+impl ReftestManifest {
+    fn new(manifest: &Path, options: &ReftestOptions) -> ReftestManifest {
+        let dir = manifest.parent().unwrap();
+        let f =
+            File::open(manifest).expect(&format!("couldn't open manifest: {}", manifest.display()));
+        let file = BufReader::new(&f);
+
+        let mut reftests = Vec::new();
+
+        for line in file.lines() {
+            let l = line.unwrap();
+
+            // strip the comments
+            let s = &l[0 .. l.find('#').unwrap_or(l.len())];
+            let s = s.trim();
+            if s.is_empty() {
+                continue;
+            }
+
+            let tokens: Vec<&str> = s.split_whitespace().collect();
+
+            let mut max_difference = 0;
+            let mut max_count = 0;
+            let mut op = ReftestOp::Equal;
+            let mut font_render_mode = None;
+            let mut expected_color_targets = None;
+            let mut expected_alpha_targets = None;
+            let mut expected_draw_calls = None;
+            let mut disable_dual_source_blending = false;
+            let mut zoom_factor = 1.0;
+
+            for (i, token) in tokens.iter().enumerate() {
+                match *token {
+                    "include" => {
+                        assert!(i == 0, "include must be by itself");
+                        let include = dir.join(tokens[1]);
+
+                        reftests.append(
+                            &mut ReftestManifest::new(include.as_path(), options).reftests,
+                        );
+
+                        break;
+                    }
+                    platform if platform.starts_with("platform") => {
+                        let (_, args, _) = parse_function(platform);
+                        if !args.iter().any(|arg| arg == &PLATFORM) {
+                            // Skip due to platform not matching
+                            break;
+                        }
+                    }
+                    function if function.starts_with("zoom") => {
+                        let (_, args, _) = parse_function(function);
+                        zoom_factor = args[0].parse().unwrap();
+                    }
+                    function if function.starts_with("fuzzy") => {
+                        let (_, args, _) = parse_function(function);
+                        max_difference = args[0].parse().unwrap();
+                        max_count = args[1].parse().unwrap();
+                    }
+                    function if function.starts_with("draw_calls") => {
+                        let (_, args, _) = parse_function(function);
+                        expected_draw_calls = Some(args[0].parse().unwrap());
+                    }
+                    function if function.starts_with("alpha_targets") => {
+                        let (_, args, _) = parse_function(function);
+                        expected_alpha_targets = Some(args[0].parse().unwrap());
+                    }
+                    function if function.starts_with("color_targets") => {
+                        let (_, args, _) = parse_function(function);
+                        expected_color_targets = Some(args[0].parse().unwrap());
+                    }
+                    options if options.starts_with("options") => {
+                        let (_, args, _) = parse_function(options);
+                        if args.iter().any(|arg| arg == &OPTION_DISABLE_SUBPX) {
+                            font_render_mode = Some(FontRenderMode::Alpha);
+                        }
+                        if args.iter().any(|arg| arg == &OPTION_DISABLE_AA) {
+                            font_render_mode = Some(FontRenderMode::Mono);
+                        }
+                        if args.iter().any(|arg| arg == &OPTION_DISABLE_DUAL_SOURCE_BLENDING) {
+                            disable_dual_source_blending = true;
+                        }
+                    }
+                    "==" => {
+                        op = ReftestOp::Equal;
+                    }
+                    "!=" => {
+                        op = ReftestOp::NotEqual;
+                    }
+                    _ => {
+                        reftests.push(Reftest {
+                            op,
+                            test: dir.join(tokens[i + 0]),
+                            reference: dir.join(tokens[i + 1]),
+                            font_render_mode,
+                            max_difference: cmp::max(max_difference, options.allow_max_difference),
+                            num_differences: cmp::max(max_count, options.allow_num_differences),
+                            expected_draw_calls,
+                            expected_alpha_targets,
+                            expected_color_targets,
+                            disable_dual_source_blending,
+                            zoom_factor,
+                        });
+
+                        break;
+                    }
+                }
+            }
+        }
+
+        ReftestManifest { reftests: reftests }
+    }
+
+    fn find(&self, prefix: &Path) -> Vec<&Reftest> {
+        self.reftests
+            .iter()
+            .filter(|x| {
+                x.test.starts_with(prefix) || x.reference.starts_with(prefix)
+            })
+            .collect()
+    }
+}
+
+pub struct ReftestHarness<'a> {
+    wrench: &'a mut Wrench,
+    window: &'a mut WindowWrapper,
+    rx: Receiver<()>,
+}
+impl<'a> ReftestHarness<'a> {
+    pub fn new(wrench: &'a mut Wrench, window: &'a mut WindowWrapper, rx: Receiver<()>) -> Self {
+        ReftestHarness { wrench, window, rx }
+    }
+
+    pub fn run(mut self, base_manifest: &Path, reftests: Option<&Path>, options: &ReftestOptions) {
+        let manifest = ReftestManifest::new(base_manifest, options);
+        let reftests = manifest.find(reftests.unwrap_or(&PathBuf::new()));
+
+        let mut total_passing = 0;
+        let mut failing = Vec::new();
+
+        for t in reftests {
+            if self.run_reftest(t) {
+                total_passing += 1;
+            } else {
+                failing.push(t);
+            }
+        }
+
+        println!(
+            "REFTEST INFO | {} passing, {} failing",
+            total_passing,
+            failing.len()
+        );
+
+        if !failing.is_empty() {
+            println!("\nReftests with unexpected results:");
+
+            for reftest in &failing {
+                println!("\t{}", reftest);
+            }
+        }
+
+        // panic here so that we fail CI
+        assert!(failing.is_empty());
+    }
+
+    fn run_reftest(&mut self, t: &Reftest) -> bool {
+        println!("REFTEST {}", t);
+
+        self.wrench.set_page_zoom(ZoomFactor::new(t.zoom_factor));
+
+        if t.disable_dual_source_blending {
+            self.wrench
+                .api
+                .send_debug_cmd(
+                    DebugCommand::EnableDualSourceBlending(false)
+                );
+        }
+
+        let window_size = self.window.get_inner_size();
+        let reference = match t.reference.extension().unwrap().to_str().unwrap() {
+            "yaml" => {
+                let (reference, _) = self.render_yaml(
+                    t.reference.as_path(),
+                    window_size,
+                    t.font_render_mode,
+                );
+                reference
+            }
+            "png" => {
+                self.load_image(t.reference.as_path(), ImageFormat::PNG)
+            }
+            other => panic!("Unknown reftest extension: {}", other),
+        };
+
+        // the reference can be smaller than the window size,
+        // in which case we only compare the intersection
+        let (test, stats) = self.render_yaml(
+            t.test.as_path(),
+            reference.size,
+            t.font_render_mode,
+        );
+
+        if t.disable_dual_source_blending {
+            self.wrench
+                .api
+                .send_debug_cmd(
+                    DebugCommand::EnableDualSourceBlending(true)
+                );
+        }
+
+        let comparison = test.compare(&reference);
+
+        if let Some(expected_draw_calls) = t.expected_draw_calls {
+            if expected_draw_calls != stats.total_draw_calls {
+                println!("REFTEST TEST-UNEXPECTED-FAIL | {}/{} | expected_draw_calls",
+                    stats.total_draw_calls,
+                    expected_draw_calls
+                );
+                println!("REFTEST TEST-END | {}", t);
+                return false;
+            }
+        }
+        if let Some(expected_alpha_targets) = t.expected_alpha_targets {
+            if expected_alpha_targets != stats.alpha_target_count {
+                println!("REFTEST TEST-UNEXPECTED-FAIL | {}/{} | alpha_target_count",
+                    stats.alpha_target_count,
+                    expected_alpha_targets
+                );
+                println!("REFTEST TEST-END | {}", t);
+                return false;
+            }
+        }
+        if let Some(expected_color_targets) = t.expected_color_targets {
+            if expected_color_targets != stats.color_target_count {
+                println!("REFTEST TEST-UNEXPECTED-FAIL | {}/{} | color_target_count",
+                    stats.color_target_count,
+                    expected_color_targets
+                );
+                println!("REFTEST TEST-END | {}", t);
+                return false;
+            }
+        }
+
+        match (&t.op, comparison) {
+            (&ReftestOp::Equal, ReftestImageComparison::Equal) => true,
+            (
+                &ReftestOp::Equal,
+                ReftestImageComparison::NotEqual {
+                    max_difference,
+                    count_different,
+                },
+            ) => if max_difference > t.max_difference || count_different > t.num_differences {
+                println!(
+                    "{} | {} | {}: {}, {}: {}",
+                    "REFTEST TEST-UNEXPECTED-FAIL",
+                    t,
+                    "image comparison, max difference",
+                    max_difference,
+                    "number of differing pixels",
+                    count_different
+                );
+                println!("REFTEST   IMAGE 1 (TEST): {}", test.create_data_uri());
+                println!(
+                    "REFTEST   IMAGE 2 (REFERENCE): {}",
+                    reference.create_data_uri()
+                );
+                println!("REFTEST TEST-END | {}", t);
+
+                false
+            } else {
+                true
+            },
+            (&ReftestOp::NotEqual, ReftestImageComparison::Equal) => {
+                println!("REFTEST TEST-UNEXPECTED-FAIL | {} | image comparison", t);
+                println!("REFTEST TEST-END | {}", t);
+
+                false
+            }
+            (&ReftestOp::NotEqual, ReftestImageComparison::NotEqual { .. }) => true,
+        }
+    }
+
+    fn load_image(&mut self, filename: &Path, format: ImageFormat) -> ReftestImage {
+        let file = BufReader::new(File::open(filename).unwrap());
+        let img_raw = load_piston_image(file, format).unwrap();
+        let img = img_raw.flipv().to_rgba();
+        let size = img.dimensions();
+        ReftestImage {
+            data: img.into_raw(),
+            size: DeviceUintSize::new(size.0, size.1),
+        }
+    }
+
+    fn render_yaml(
+        &mut self,
+        filename: &Path,
+        size: DeviceUintSize,
+        font_render_mode: Option<FontRenderMode>,
+    ) -> (ReftestImage, RendererStats) {
+        let mut reader = YamlFrameReader::new(filename);
+        reader.set_font_render_mode(font_render_mode);
+        reader.do_frame(self.wrench);
+
+        // wait for the frame
+        self.rx.recv().unwrap();
+        let stats = self.wrench.render();
+
+        let window_size = self.window.get_inner_size();
+        assert!(size.width <= window_size.width && size.height <= window_size.height);
+
+        // taking the bottom left sub-rectangle
+        let rect = DeviceUintRect::new(DeviceUintPoint::new(0, window_size.height - size.height), size);
+        let pixels = self.wrench.renderer.read_pixels_rgba8(rect);
+        self.window.swap_buffers();
+
+        let write_debug_images = false;
+        if write_debug_images {
+            let debug_path = filename.with_extension("yaml.png");
+            save_flipped(debug_path, pixels.clone(), size);
+        }
+
+        reader.deinit(self.wrench);
+
+        (ReftestImage { data: pixels, size }, stats)
+    }
+}
diff --git a/gfx/wrench/src/ron_frame_writer.rs b/gfx/wrench/src/ron_frame_writer.rs
new file mode 100644
index 000000000000..51b683102b1c
--- /dev/null
+++ b/gfx/wrench/src/ron_frame_writer.rs
@@ -0,0 +1,195 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use ron;
+use std::collections::HashMap;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::{fmt, fs};
+use super::CURRENT_FRAME_NUMBER;
+use webrender;
+use webrender::api::*;
+use webrender::api::channel::Payload;
+
+enum CachedFont {
+    Native(NativeFontHandle),
+    Raw(Option<Vec<u8>>, u32, Option<PathBuf>),
+}
+
+struct CachedImage {
+    width: u32,
+    height: u32,
+    format: ImageFormat,
+    bytes: Option<Vec<u8>>,
+    path: Option<PathBuf>,
+}
+
+pub struct RonFrameWriter {
+    frame_base: PathBuf,
+    images: HashMap<ImageKey, CachedImage>,
+    fonts: HashMap<FontKey, CachedFont>,
+
+    last_frame_written: u32,
+
+    dl_descriptor: Option<BuiltDisplayListDescriptor>,
+}
+
+impl RonFrameWriter {
+    pub fn new(path: &Path) -> Self {
+        let mut rsrc_base = path.to_owned();
+        rsrc_base.push("res");
+        fs::create_dir_all(&rsrc_base).ok();
+
+        RonFrameWriter {
+            frame_base: path.to_owned(),
+            images: HashMap::new(),
+            fonts: HashMap::new(),
+
+            dl_descriptor: None,
+
+            last_frame_written: u32::max_value(),
+        }
+    }
+
+    pub fn begin_write_display_list(
+        &mut self,
+        _: &Epoch,
+        _: &PipelineId,
+        _: &Option<ColorF>,
+        _: &LayoutSize,
+        display_list: &BuiltDisplayListDescriptor,
+    ) {
+        unsafe {
+            if CURRENT_FRAME_NUMBER == self.last_frame_written {
+                return;
+            }
+            self.last_frame_written = CURRENT_FRAME_NUMBER;
+        }
+
+        self.dl_descriptor = Some(display_list.clone());
+    }
+
+    pub fn finish_write_display_list(&mut self, _frame: u32, data: &[u8]) {
+        let payload = Payload::from_data(data);
+        let dl_desc = self.dl_descriptor.take().unwrap();
+
+        let dl = BuiltDisplayList::from_data(payload.display_list_data, dl_desc);
+
+        let mut frame_file_name = self.frame_base.clone();
+        let current_shown_frame = unsafe { CURRENT_FRAME_NUMBER };
+        frame_file_name.push(format!("frame-{}.ron", current_shown_frame));
+
+        let mut file = fs::File::create(&frame_file_name).unwrap();
+
+        let s = ron::ser::to_string_pretty(&dl, Default::default()).unwrap();
+        file.write_all(&s.into_bytes()).unwrap();
+        file.write_all(b"\n").unwrap();
+    }
+
+    fn update_resources(&mut self, updates: &ResourceUpdates) {
+        for update in &updates.updates {
+            match *update {
+                ResourceUpdate::AddImage(ref img) => {
+                    let bytes = match img.data {
+                        ImageData::Raw(ref v) => (**v).clone(),
+                        ImageData::External(_) | ImageData::Blob(_) => {
+                            return;
+                        }
+                    };
+                    self.images.insert(
+                        img.key,
+                        CachedImage {
+                            width: img.descriptor.width,
+                            height: img.descriptor.height,
+                            format: img.descriptor.format,
+                            bytes: Some(bytes),
+                            path: None,
+                        },
+                    );
+                }
+                ResourceUpdate::UpdateImage(ref img) => {
+                    if let Some(ref mut data) = self.images.get_mut(&img.key) {
+                        assert_eq!(data.width, img.descriptor.width);
+                        assert_eq!(data.height, img.descriptor.height);
+                        assert_eq!(data.format, img.descriptor.format);
+
+                        if let ImageData::Raw(ref bytes) = img.data {
+                            data.path = None;
+                            data.bytes = Some((**bytes).clone());
+                        } else {
+                            // Other existing image types only make sense within the gecko integration.
+                            println!(
+                                "Wrench only supports updating buffer images ({}).",
+                                "ignoring update commands"
+                            );
+                        }
+                    }
+                }
+                ResourceUpdate::DeleteImage(img) => {
+                    self.images.remove(&img);
+                }
+                ResourceUpdate::AddFont(ref font) => match font {
+                    &AddFont::Raw(key, ref bytes, index) => {
+                        self.fonts
+                            .insert(key, CachedFont::Raw(Some(bytes.clone()), index, None));
+                    }
+                    &AddFont::Native(key, ref handle) => {
+                        self.fonts.insert(key, CachedFont::Native(handle.clone()));
+                    }
+                },
+                ResourceUpdate::DeleteFont(_) => {}
+                ResourceUpdate::AddFontInstance(_) => {}
+                ResourceUpdate::DeleteFontInstance(_) => {}
+            }
+        }
+    }
+}
+
+impl fmt::Debug for RonFrameWriter {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "RonFrameWriter")
+    }
+}
+
+impl webrender::ApiRecordingReceiver for RonFrameWriter {
+    fn write_msg(&mut self, _: u32, msg: &ApiMsg) {
+        match *msg {
+            ApiMsg::UpdateResources(ref updates) => self.update_resources(updates),
+            ApiMsg::UpdateDocument(_, ref doc_msgs) => {
+                for doc_msg in doc_msgs {
+                    match *doc_msg {
+                        DocumentMsg::UpdateResources(ref resources) => {
+                            self.update_resources(resources);
+                        }
+                        DocumentMsg::SetDisplayList {
+                            ref epoch,
+                            ref pipeline_id,
+                            ref background,
+                            ref viewport_size,
+                            ref list_descriptor,
+                            ..
+                        } => {
+                            self.begin_write_display_list(
+                                epoch,
+                                pipeline_id,
+                                background,
+                                viewport_size,
+                                list_descriptor,
+                            );
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            ApiMsg::CloneApi(..) => {}
+            _ => {}
+        }
+    }
+
+    fn write_payload(&mut self, frame: u32, data: &[u8]) {
+        if self.dl_descriptor.is_some() {
+            self.finish_write_display_list(frame, data);
+        }
+    }
+}
diff --git a/gfx/wrench/src/scene.rs b/gfx/wrench/src/scene.rs
new file mode 100644
index 000000000000..4e6d5753a2c1
--- /dev/null
+++ b/gfx/wrench/src/scene.rs
@@ -0,0 +1,134 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use std::collections::HashMap;
+use webrender::api::{BuiltDisplayList, ColorF, Epoch};
+use webrender::api::{LayerSize, PipelineId};
+use webrender::api::{PropertyBinding, PropertyBindingId, LayoutTransform, DynamicProperties};
+
+/// Stores a map of the animated property bindings for the current display list. These
+/// can be used to animate the transform and/or opacity of a display list without
+/// re-submitting the display list itself.
+#[derive(Default)]
+pub struct SceneProperties {
+    transform_properties: HashMap<PropertyBindingId, LayoutTransform>,
+    float_properties: HashMap<PropertyBindingId, f32>,
+}
+
+impl SceneProperties {
+    /// Set the current property list for this display list.
+    pub fn set_properties(&mut self, properties: &DynamicProperties) {
+        self.transform_properties.clear();
+        self.float_properties.clear();
+
+        for property in &properties.transforms {
+            self.transform_properties
+                .insert(property.key.id, property.value);
+        }
+
+        for property in &properties.floats {
+            self.float_properties
+                .insert(property.key.id, property.value);
+        }
+    }
+
+    /// Get the current value for a transform property.
+    pub fn resolve_layout_transform(
+        &self,
+        property: &Option<PropertyBinding<LayoutTransform>>,
+    ) -> LayoutTransform {
+        let property = match *property {
+            Some(property) => property,
+            None => return LayoutTransform::identity(),
+        };
+
+        match property {
+            PropertyBinding::Value(matrix) => matrix,
+            PropertyBinding::Binding(ref key) => self.transform_properties
+                .get(&key.id)
+                .cloned()
+                .unwrap_or_else(|| {
+                    println!("Property binding {:?} has an invalid value.", key);
+                    LayoutTransform::identity()
+                }),
+        }
+    }
+
+    /// Get the current value for a float property.
+    pub fn resolve_float(&self, property: &PropertyBinding<f32>, default_value: f32) -> f32 {
+        match *property {
+            PropertyBinding::Value(value) => value,
+            PropertyBinding::Binding(ref key) => self.float_properties
+                .get(&key.id)
+                .cloned()
+                .unwrap_or_else(|| {
+                    println!("Property binding {:?} has an invalid value.", key);
+                    default_value
+                }),
+        }
+    }
+}
+
+/// A representation of the layout within the display port for a given document or iframe.
+#[derive(Debug)]
+pub struct ScenePipeline {
+    pub epoch: Epoch,
+    pub viewport_size: LayerSize,
+    pub background_color: Option<ColorF>,
+}
+
+/// A complete representation of the layout bundling visible pipelines together.
+pub struct Scene {
+    pub properties: SceneProperties,
+    pub root_pipeline_id: Option<PipelineId>,
+    pub pipeline_map: HashMap<PipelineId, ScenePipeline>,
+    pub display_lists: HashMap<PipelineId, BuiltDisplayList>,
+}
+
+impl Scene {
+    pub fn new() -> Scene {
+        Scene {
+            properties: SceneProperties::default(),
+            root_pipeline_id: None,
+            pipeline_map: HashMap::default(),
+            display_lists: HashMap::default(),
+        }
+    }
+
+    pub fn set_root_pipeline_id(&mut self, pipeline_id: PipelineId) {
+        self.root_pipeline_id = Some(pipeline_id);
+    }
+
+    pub fn remove_pipeline(&mut self, pipeline_id: &PipelineId) {
+        if self.root_pipeline_id == Some(*pipeline_id) {
+            self.root_pipeline_id = None;
+        }
+        self.pipeline_map.remove(pipeline_id);
+        self.display_lists.remove(pipeline_id);
+    }
+
+    pub fn begin_display_list(
+        &mut self,
+        pipeline_id: &PipelineId,
+        epoch: &Epoch,
+        background_color: &Option<ColorF>,
+        viewport_size: &LayerSize,
+    ) {
+        let new_pipeline = ScenePipeline {
+            epoch: epoch.clone(),
+            viewport_size: viewport_size.clone(),
+            background_color: background_color.clone(),
+        };
+
+        self.pipeline_map.insert(pipeline_id.clone(), new_pipeline);
+    }
+
+    pub fn finish_display_list(
+        &mut self,
+        pipeline_id: PipelineId,
+        built_display_list: BuiltDisplayList,
+    ) {
+        self.display_lists.insert(pipeline_id, built_display_list);
+    }
+}
diff --git a/gfx/wrench/src/wrench.rs b/gfx/wrench/src/wrench.rs
new file mode 100644
index 000000000000..09ca3b7e93d9
--- /dev/null
+++ b/gfx/wrench/src/wrench.rs
@@ -0,0 +1,595 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+
+use app_units::Au;
+use blob;
+use crossbeam::sync::chase_lev;
+#[cfg(windows)]
+use dwrote;
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+use font_loader::system_fonts;
+use glutin::WindowProxy;
+use json_frame_writer::JsonFrameWriter;
+use ron_frame_writer::RonFrameWriter;
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
+use time;
+use webrender;
+use webrender::api::*;
+use webrender::{DebugFlags, RendererStats};
+use yaml_frame_writer::YamlFrameWriterReceiver;
+use {WindowWrapper, BLACK_COLOR, WHITE_COLOR};
+
+// TODO(gw): This descriptor matches what we currently support for fonts
+//           but is quite a mess. We should at least document and
+//           use better types for things like the style and stretch.
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub enum FontDescriptor {
+    Path { path: PathBuf, font_index: u32 },
+    Family { name: String },
+    Properties {
+        family: String,
+        weight: u32,
+        style: u32,
+        stretch: u32,
+    },
+}
+
+pub enum SaveType {
+    Yaml,
+    Json,
+    Ron,
+    Binary,
+}
+
+struct NotifierData {
+    window_proxy: Option<WindowProxy>,
+    frames_notified: u32,
+    timing_receiver: chase_lev::Stealer<time::SteadyTime>,
+    verbose: bool,
+}
+
+impl NotifierData {
+    fn new(
+        window_proxy: Option<WindowProxy>,
+        timing_receiver: chase_lev::Stealer<time::SteadyTime>,
+        verbose: bool,
+    ) -> Self {
+        NotifierData {
+            window_proxy,
+            frames_notified: 0,
+            timing_receiver,
+            verbose,
+        }
+    }
+}
+
+struct Notifier(Arc<Mutex<NotifierData>>);
+
+impl RenderNotifier for Notifier {
+    fn clone(&self) -> Box<RenderNotifier> {
+        Box::new(Notifier(self.0.clone()))
+    }
+
+    fn wake_up(&self) {
+        let mut data = self.0.lock();
+        let data = data.as_mut().unwrap();
+        match data.timing_receiver.steal() {
+            chase_lev::Steal::Data(last_timing) => {
+                data.frames_notified += 1;
+                if data.verbose && data.frames_notified == 600 {
+                    let elapsed = time::SteadyTime::now() - last_timing;
+                    println!(
+                        "frame latency (consider queue depth here): {:3.6} ms",
+                        elapsed.num_microseconds().unwrap() as f64 / 1000.
+                    );
+                    data.frames_notified = 0;
+                }
+            }
+            _ => {
+                println!("Notified of frame, but no frame was ready?");
+            }
+        }
+        if let Some(ref window_proxy) = data.window_proxy {
+            #[cfg(not(target_os = "android"))]
+            window_proxy.wakeup_event_loop();
+        }
+    }
+
+    fn new_document_ready(&self, _: DocumentId, scrolled: bool, _composite_needed: bool) {
+        if scrolled {
+            let data = self.0.lock();
+            if let Some(ref window_proxy) = data.unwrap().window_proxy {
+                #[cfg(not(target_os = "android"))]
+                window_proxy.wakeup_event_loop();
+            }
+        } else {
+            self.wake_up();
+        }
+    }
+}
+
+pub trait WrenchThing {
+    fn next_frame(&mut self);
+    fn prev_frame(&mut self);
+    fn do_frame(&mut self, &mut Wrench) -> u32;
+    fn queue_frames(&self) -> u32 {
+        0
+    }
+}
+
+impl WrenchThing for CapturedDocument {
+    fn next_frame(&mut self) {}
+    fn prev_frame(&mut self) {}
+    fn do_frame(&mut self, wrench: &mut Wrench) -> u32 {
+        match self.root_pipeline_id.take() {
+            Some(root_pipeline_id) => {
+                // skip the first frame - to not overwrite the loaded one
+                let mut txn = Transaction::new();
+                txn.set_root_pipeline(root_pipeline_id);
+                wrench.api.send_transaction(self.document_id, txn);
+            }
+            None => {
+                wrench.refresh();
+            }
+        }
+        0
+    }
+}
+
+pub struct Wrench {
+    window_size: DeviceUintSize,
+    device_pixel_ratio: f32,
+    page_zoom_factor: ZoomFactor,
+
+    pub renderer: webrender::Renderer,
+    pub api: RenderApi,
+    pub document_id: DocumentId,
+    pub root_pipeline_id: PipelineId,
+
+    window_title_to_set: Option<String>,
+
+    graphics_api: webrender::GraphicsApiInfo,
+
+    pub rebuild_display_lists: bool,
+    pub verbose: bool,
+
+    pub frame_start_sender: chase_lev::Worker<time::SteadyTime>,
+
+    pub callbacks: Arc<Mutex<blob::BlobCallbacks>>,
+}
+
+impl Wrench {
+    pub fn new(
+        window: &mut WindowWrapper,
+        shader_override_path: Option<PathBuf>,
+        dp_ratio: f32,
+        save_type: Option<SaveType>,
+        size: DeviceUintSize,
+        do_rebuild: bool,
+        no_subpixel_aa: bool,
+        debug: bool,
+        verbose: bool,
+        no_scissor: bool,
+        no_batch: bool,
+        precache_shaders: bool,
+        disable_dual_source_blending: bool,
+        zoom_factor: f32,
+        notifier: Option<Box<RenderNotifier>>,
+    ) -> Self {
+        println!("Shader override path: {:?}", shader_override_path);
+
+        let recorder = save_type.map(|save_type| match save_type {
+            SaveType::Yaml => Box::new(
+                YamlFrameWriterReceiver::new(&PathBuf::from("yaml_frames")),
+            ) as Box<webrender::ApiRecordingReceiver>,
+            SaveType::Json => Box::new(JsonFrameWriter::new(&PathBuf::from("json_frames"))) as
+                Box<webrender::ApiRecordingReceiver>,
+            SaveType::Ron => Box::new(RonFrameWriter::new(&PathBuf::from("ron_frames"))) as
+                Box<webrender::ApiRecordingReceiver>,
+            SaveType::Binary => Box::new(webrender::BinaryRecorder::new(
+                &PathBuf::from("wr-record.bin"),
+            )) as Box<webrender::ApiRecordingReceiver>,
+        });
+
+        let mut debug_flags = DebugFlags::default();
+        debug_flags.set(DebugFlags::DISABLE_BATCHING, no_batch);
+        let callbacks = Arc::new(Mutex::new(blob::BlobCallbacks::new()));
+
+        let opts = webrender::RendererOptions {
+            device_pixel_ratio: dp_ratio,
+            resource_override_path: shader_override_path,
+            recorder,
+            enable_subpixel_aa: !no_subpixel_aa,
+            debug,
+            debug_flags,
+            enable_clear_scissor: !no_scissor,
+            max_recorded_profiles: 16,
+            precache_shaders,
+            blob_image_renderer: Some(Box::new(blob::CheckerboardRenderer::new(callbacks.clone()))),
+            disable_dual_source_blending,
+            ..Default::default()
+        };
+
+        let proxy = window.create_window_proxy();
+        // put an Awakened event into the queue to kick off the first frame
+        if let Some(ref wp) = proxy {
+            #[cfg(not(target_os = "android"))]
+            wp.wakeup_event_loop();
+        }
+
+        let (timing_sender, timing_receiver) = chase_lev::deque();
+        let notifier = notifier.unwrap_or_else(|| {
+            let data = Arc::new(Mutex::new(NotifierData::new(proxy, timing_receiver, verbose)));
+            Box::new(Notifier(data))
+        });
+
+        let (renderer, sender) = webrender::Renderer::new(window.clone_gl(), notifier, opts).unwrap();
+        let api = sender.create_api();
+        let document_id = api.add_document(size, 0);
+
+        let graphics_api = renderer.get_graphics_api_info();
+        let zoom_factor = ZoomFactor::new(zoom_factor);
+
+        let mut wrench = Wrench {
+            window_size: size,
+
+            renderer,
+            api,
+            document_id,
+            window_title_to_set: None,
+
+            rebuild_display_lists: do_rebuild,
+            verbose,
+            device_pixel_ratio: dp_ratio,
+            page_zoom_factor: zoom_factor,
+
+            root_pipeline_id: PipelineId(0, 0),
+
+            graphics_api,
+            frame_start_sender: timing_sender,
+
+            callbacks,
+        };
+
+        wrench.set_page_zoom(zoom_factor);
+        wrench.set_title("start");
+        let mut txn = Transaction::new();
+        txn.set_root_pipeline(wrench.root_pipeline_id);
+        wrench.api.send_transaction(wrench.document_id, txn);
+
+        wrench
+    }
+
+    pub fn get_page_zoom(&self) -> ZoomFactor {
+        self.page_zoom_factor
+    }
+
+    pub fn set_page_zoom(&mut self, zoom_factor: ZoomFactor) {
+        self.page_zoom_factor = zoom_factor;
+        let mut txn = Transaction::new();
+        txn.set_page_zoom(self.page_zoom_factor);
+        self.api.send_transaction(self.document_id, txn);
+        self.set_title("");
+    }
+
+    pub fn layout_simple_ascii(
+        &mut self,
+        font_key: FontKey,
+        instance_key: FontInstanceKey,
+        render_mode: Option<FontRenderMode>,
+        text: &str,
+        size: Au,
+        origin: LayerPoint,
+        flags: FontInstanceFlags,
+    ) -> (Vec<u32>, Vec<LayerPoint>, LayoutRect) {
+        // Map the string codepoints to glyph indices in this font.
+        // Just drop any glyph that isn't present in this font.
+        let indices: Vec<u32> = self.api
+            .get_glyph_indices(font_key, text)
+            .iter()
+            .filter_map(|idx| *idx)
+            .collect();
+
+        let render_mode = render_mode.unwrap_or(<FontInstanceOptions as Default>::default().render_mode);
+        let subpx_dir = SubpixelDirection::Horizontal.limit_by(render_mode);
+
+        // Retrieve the metrics for each glyph.
+        let mut keys = Vec::new();
+        for glyph_index in &indices {
+            keys.push(GlyphKey::new(
+                *glyph_index,
+                LayerPoint::zero(),
+                render_mode,
+                subpx_dir,
+            ));
+        }
+        let metrics = self.api.get_glyph_dimensions(instance_key, keys);
+
+        let mut bounding_rect = LayoutRect::zero();
+        let mut positions = Vec::new();
+
+        let mut cursor = origin;
+        let direction = if flags.contains(FontInstanceFlags::TRANSPOSE) {
+            LayerVector2D::new(
+                0.0,
+                if flags.contains(FontInstanceFlags::FLIP_Y) { -1.0 } else { 1.0 },
+            )
+        } else {
+            LayerVector2D::new(
+                if flags.contains(FontInstanceFlags::FLIP_X) { -1.0 } else { 1.0 },
+                0.0,
+            )
+        };
+        for metric in metrics {
+            positions.push(cursor);
+
+            match metric {
+                Some(metric) => {
+                    let glyph_rect = LayoutRect::new(
+                        LayoutPoint::new(cursor.x + metric.left as f32, cursor.y - metric.top as f32),
+                        LayoutSize::new(metric.width as f32, metric.height as f32)
+                    );
+                    bounding_rect = bounding_rect.union(&glyph_rect);
+                    cursor += direction * metric.advance;
+                }
+                None => {
+                    // Extract the advances from the metrics. The get_glyph_dimensions API
+                    // has a limitation that it can't currently get dimensions for non-renderable
+                    // glyphs (e.g. spaces), so just use a rough estimate in that case.
+                    let space_advance = size.to_f32_px() / 3.0;
+                    cursor += direction * space_advance;
+                }
+            }
+        }
+
+        // The platform font implementations don't always handle
+        // the exact dimensions used when subpixel AA is enabled
+        // on glyphs. As a workaround, inflate the bounds by
+        // 2 pixels on either side, to give a slightly less
+        // tight fitting bounding rect.
+        let bounding_rect = bounding_rect.inflate(2.0, 2.0);
+
+        (indices, positions, bounding_rect)
+    }
+
+    pub fn set_title(&mut self, extra: &str) {
+        self.window_title_to_set = Some(format!(
+            "Wrench: {} ({}x zoom={}) - {} - {}",
+            extra,
+            self.device_pixel_ratio,
+            self.page_zoom_factor.get(),
+            self.graphics_api.renderer,
+            self.graphics_api.version
+        ));
+    }
+
+    pub fn take_title(&mut self) -> Option<String> {
+        self.window_title_to_set.take()
+    }
+
+    pub fn should_rebuild_display_lists(&self) -> bool {
+        self.rebuild_display_lists
+    }
+
+    pub fn window_size_f32(&self) -> LayoutSize {
+        LayoutSize::new(
+            self.window_size.width as f32,
+            self.window_size.height as f32,
+        )
+    }
+
+    #[cfg(target_os = "windows")]
+    pub fn font_key_from_native_handle(&mut self, descriptor: &NativeFontHandle) -> FontKey {
+        let key = self.api.generate_font_key();
+        let mut resources = ResourceUpdates::new();
+        resources.add_native_font(key, descriptor.clone());
+        self.api.update_resources(resources);
+        key
+    }
+
+    #[cfg(target_os = "windows")]
+    pub fn font_key_from_name(&mut self, font_name: &str) -> FontKey {
+        let system_fc = dwrote::FontCollection::system();
+        let family = system_fc.get_font_family_by_name(font_name).unwrap();
+        let font = family.get_first_matching_font(
+            dwrote::FontWeight::Regular,
+            dwrote::FontStretch::Normal,
+            dwrote::FontStyle::Normal,
+        );
+        let descriptor = font.to_descriptor();
+        self.font_key_from_native_handle(&descriptor)
+    }
+
+    #[cfg(target_os = "windows")]
+    pub fn font_key_from_properties(
+        &mut self,
+        family: &str,
+        weight: u32,
+        style: u32,
+        stretch: u32,
+    ) -> FontKey {
+        let weight = dwrote::FontWeight::from_u32(weight);
+        let style = dwrote::FontStyle::from_u32(style);
+        let stretch = dwrote::FontStretch::from_u32(stretch);
+
+        let desc = dwrote::FontDescriptor {
+            family_name: family.to_owned(),
+            weight,
+            style,
+            stretch,
+        };
+        self.font_key_from_native_handle(&desc)
+    }
+
+    #[cfg(any(target_os = "linux", target_os = "macos"))]
+    pub fn font_key_from_properties(
+        &mut self,
+        family: &str,
+        _weight: u32,
+        _style: u32,
+        _stretch: u32,
+    ) -> FontKey {
+        let property = system_fonts::FontPropertyBuilder::new()
+            .family(family)
+            .build();
+        let (font, index) = system_fonts::get(&property).unwrap();
+        self.font_key_from_bytes(font, index as u32)
+    }
+
+    #[cfg(unix)]
+    pub fn font_key_from_name(&mut self, font_name: &str) -> FontKey {
+        let property = system_fonts::FontPropertyBuilder::new()
+            .family(font_name)
+            .build();
+        let (font, index) = system_fonts::get(&property).unwrap();
+        self.font_key_from_bytes(font, index as u32)
+    }
+
+    #[cfg(target_os = "android")]
+    pub fn font_key_from_name(&mut self, font_name: &str) -> FontKey {
+        unimplemented!()
+    }
+
+    pub fn font_key_from_bytes(&mut self, bytes: Vec<u8>, index: u32) -> FontKey {
+        let key = self.api.generate_font_key();
+        let mut update = ResourceUpdates::new();
+        update.add_raw_font(key, bytes, index);
+        self.api.update_resources(update);
+        key
+    }
+
+    pub fn add_font_instance(&mut self,
+        font_key: FontKey,
+        size: Au,
+        flags: FontInstanceFlags,
+        render_mode: Option<FontRenderMode>,
+    ) -> FontInstanceKey {
+        let key = self.api.generate_font_instance_key();
+        let mut update = ResourceUpdates::new();
+        let mut options: FontInstanceOptions = Default::default();
+        options.flags |= flags;
+        if let Some(render_mode) = render_mode {
+            options.render_mode = render_mode;
+        }
+        update.add_font_instance(key, font_key, size, Some(options), None, Vec::new());
+        self.api.update_resources(update);
+        key
+    }
+
+    #[allow(dead_code)]
+    pub fn delete_font_instance(&mut self, key: FontInstanceKey) {
+        let mut update = ResourceUpdates::new();
+        update.delete_font_instance(key);
+        self.api.update_resources(update);
+    }
+
+    pub fn update(&mut self, dim: DeviceUintSize) {
+        if dim != self.window_size {
+            self.window_size = dim;
+        }
+    }
+
+    pub fn begin_frame(&mut self) {
+        self.frame_start_sender.push(time::SteadyTime::now());
+    }
+
+    pub fn send_lists(
+        &mut self,
+        frame_number: u32,
+        display_lists: Vec<(PipelineId, LayerSize, BuiltDisplayList)>,
+        scroll_offsets: &HashMap<ClipId, LayerPoint>,
+    ) {
+        let root_background_color = Some(ColorF::new(1.0, 1.0, 1.0, 1.0));
+
+        let mut txn = Transaction::new();
+        for display_list in display_lists {
+            txn.set_display_list(
+                Epoch(frame_number),
+                root_background_color,
+                self.window_size_f32(),
+                display_list,
+                false,
+            );
+        }
+        // TODO(nical) - Need to separate the set_display_list from the scrolling
+        // operations into separate transactions for mysterious -but probably related
+        // to the other comment below- reasons.
+        self.api.send_transaction(self.document_id, txn);
+
+        let mut txn = Transaction::new();
+        for (id, offset) in scroll_offsets {
+            txn.scroll_node_with_id(
+                *offset,
+                *id,
+                ScrollClamping::NoClamping,
+            );
+        }
+        // TODO(nical) - Wrench does not notify frames when there was scrolling
+        // in the transaction (See RenderNotifier implementations). If we don't
+        // generate a frame after scrolling, wrench just stops and some tests
+        // will time out.
+        // I suppose this was to avoid taking the snapshot after scrolling if
+        // there was other updates coming in a subsequent messages but it's very
+        // error-prone with transactions.
+        // For now just send two transactions to avoid the deadlock, but we should
+        // figure this out.
+        self.api.send_transaction(self.document_id, txn);
+
+        let mut txn = Transaction::new();
+        txn.generate_frame();
+        self.api.send_transaction(self.document_id, txn);
+    }
+
+    pub fn get_frame_profiles(
+        &mut self,
+    ) -> (Vec<webrender::CpuProfile>, Vec<webrender::GpuProfile>) {
+        self.renderer.get_frame_profiles()
+    }
+
+    pub fn render(&mut self) -> RendererStats {
+        self.renderer.update();
+        self.renderer
+            .render(self.window_size)
+            .expect("errors encountered during render!")
+    }
+
+    pub fn refresh(&mut self) {
+        self.begin_frame();
+        let mut txn = Transaction::new();
+        txn.generate_frame();
+        self.api.send_transaction(self.document_id, txn);
+    }
+
+    pub fn show_onscreen_help(&mut self) {
+        let help_lines = [
+            "Esc - Quit",
+            "H - Toggle help",
+            "R - Toggle recreating display items each frame",
+            "P - Toggle profiler",
+            "O - Toggle showing intermediate targets",
+            "I - Toggle showing texture caches",
+            "B - Toggle showing alpha primitive rects",
+            "S - Toggle compact profiler",
+            "Q - Toggle GPU queries for time and samples",
+            "M - Trigger memory pressure event",
+            "T - Save CPU profile to a file",
+            "C - Save a capture to captures/wrench/",
+        ];
+
+        let color_and_offset = [(*BLACK_COLOR, 2.0), (*WHITE_COLOR, 0.0)];
+        let dr = self.renderer.debug_renderer();
+
+        for ref co in &color_and_offset {
+            let x = self.device_pixel_ratio * (15.0 + co.1);
+            let mut y = self.device_pixel_ratio * (15.0 + co.1 + dr.line_height());
+            for ref line in &help_lines {
+                dr.add_text(x, y, line, co.0.into());
+                y += self.device_pixel_ratio * dr.line_height();
+            }
+        }
+    }
+}
diff --git a/gfx/wrench/src/yaml_frame_reader.rs b/gfx/wrench/src/yaml_frame_reader.rs
new file mode 100644
index 000000000000..a3637b9f3e0d
--- /dev/null
+++ b/gfx/wrench/src/yaml_frame_reader.rs
@@ -0,0 +1,1510 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use app_units::Au;
+use clap;
+use euclid::SideOffsets2D;
+use image;
+use image::GenericImage;
+use parse_function::parse_function;
+use premultiply::premultiply;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::Read;
+use std::path::{Path, PathBuf};
+use webrender::api::*;
+use wrench::{FontDescriptor, Wrench, WrenchThing};
+use yaml_helper::{StringEnum, YamlHelper};
+use yaml_rust::{Yaml, YamlLoader};
+use {BLACK_COLOR, PLATFORM_DEFAULT_FACE_NAME, WHITE_COLOR};
+
+fn rsrc_path(item: &Yaml, aux_dir: &PathBuf) -> PathBuf {
+    let filename = item.as_str().unwrap();
+    let mut file = aux_dir.clone();
+    file.push(filename);
+    file
+}
+
+impl FontDescriptor {
+    fn from_yaml(item: &Yaml, aux_dir: &PathBuf) -> FontDescriptor {
+        if !item["family"].is_badvalue() {
+            FontDescriptor::Properties {
+                family: item["family"].as_str().unwrap().to_owned(),
+                weight: item["weight"].as_i64().unwrap_or(400) as u32,
+                style: item["style"].as_i64().unwrap_or(0) as u32,
+                stretch: item["stretch"].as_i64().unwrap_or(5) as u32,
+            }
+        } else if !item["font"].is_badvalue() {
+            let path = rsrc_path(&item["font"], aux_dir);
+            FontDescriptor::Path {
+                path,
+                font_index: item["font-index"].as_i64().unwrap_or(0) as u32,
+            }
+        } else {
+            FontDescriptor::Family {
+                name: PLATFORM_DEFAULT_FACE_NAME.clone(),
+            }
+        }
+    }
+}
+
+fn broadcast<T: Clone>(base_vals: &[T], num_items: usize) -> Vec<T> {
+    if base_vals.len() == num_items {
+        return base_vals.to_vec();
+    }
+
+    assert_eq!(
+        num_items % base_vals.len(),
+        0,
+        "Cannot broadcast {} elements into {}",
+        base_vals.len(),
+        num_items
+    );
+
+    let mut vals = vec![];
+    loop {
+        if vals.len() == num_items {
+            break;
+        }
+        vals.extend_from_slice(base_vals);
+    }
+    vals
+}
+
+fn generate_checkerboard_image(
+    border: u32,
+    tile_size: u32,
+    tile_count: u32
+) -> (ImageDescriptor, ImageData) {
+    let size = 2 * border + tile_size * tile_count;
+    let mut pixels = Vec::new();
+
+    for y in 0 .. size {
+        for x in 0 .. size {
+            if y < border || y >= (size-border) ||
+               x < border || x >= (size-border) {
+                pixels.push(0);
+                pixels.push(0);
+                pixels.push(0xff);
+                pixels.push(0xff);
+            } else {
+                let xon = ((x - border) % (2 * tile_size)) < tile_size;
+                let yon = ((y - border) % (2 * tile_size)) < tile_size;
+                let value = if xon ^ yon { 0xff } else { 0x7f };
+                pixels.push(value);
+                pixels.push(value);
+                pixels.push(value);
+                pixels.push(0xff);
+            }
+        }
+    }
+
+    (
+        ImageDescriptor::new(size, size, ImageFormat::BGRA8, true),
+        ImageData::new(pixels),
+    )
+}
+
+fn generate_xy_gradient_image(w: u32, h: u32) -> (ImageDescriptor, ImageData) {
+    let mut pixels = Vec::with_capacity((w * h * 4) as usize);
+    for y in 0 .. h {
+        for x in 0 .. w {
+            let grid = if x % 100 < 3 || y % 100 < 3 { 0.9 } else { 1.0 };
+            pixels.push((y as f32 / h as f32 * 255.0 * grid) as u8);
+            pixels.push(0);
+            pixels.push((x as f32 / w as f32 * 255.0 * grid) as u8);
+            pixels.push(255);
+        }
+    }
+
+    (
+        ImageDescriptor::new(w, h, ImageFormat::BGRA8, true),
+        ImageData::new(pixels),
+    )
+}
+
+fn generate_solid_color_image(
+    r: u8,
+    g: u8,
+    b: u8,
+    a: u8,
+    w: u32,
+    h: u32,
+) -> (ImageDescriptor, ImageData) {
+    let buf_size = (w * h * 4) as usize;
+    let mut pixels = Vec::with_capacity(buf_size);
+    // Unsafely filling the buffer is horrible. Unfortunately doing this idiomatically
+    // is terribly slow in debug builds to the point that reftests/image/very-big.yaml
+    // takes more than 20 seconds to run on a recent laptop.
+    unsafe {
+        pixels.set_len(buf_size);
+        let color: u32 = ::std::mem::transmute([b, g, r, a]);
+        let mut ptr: *mut u32 = ::std::mem::transmute(&mut pixels[0]);
+        let end = ptr.offset((w * h) as isize);
+        while ptr < end {
+            *ptr = color;
+            ptr = ptr.offset(1);
+        }
+    }
+
+    (
+        ImageDescriptor::new(w, h, ImageFormat::BGRA8, a == 255),
+        ImageData::new(pixels),
+    )
+}
+
+
+
+fn is_image_opaque(format: ImageFormat, bytes: &[u8]) -> bool {
+    match format {
+        ImageFormat::BGRA8 => {
+            let mut is_opaque = true;
+            for i in 0 .. (bytes.len() / 4) {
+                if bytes[i * 4 + 3] != 255 {
+                    is_opaque = false;
+                    break;
+                }
+            }
+            is_opaque
+        }
+        ImageFormat::RG8 => true,
+        ImageFormat::R8 => false,
+        ImageFormat::RGBAF32 => unreachable!(),
+    }
+}
+
+pub struct YamlFrameReader {
+    frame_built: bool,
+    yaml_path: PathBuf,
+    aux_dir: PathBuf,
+    frame_count: u32,
+
+    display_lists: Vec<(PipelineId, LayoutSize, BuiltDisplayList)>,
+    queue_depth: u32,
+
+    include_only: Vec<String>,
+
+    watch_source: bool,
+    list_resources: bool,
+
+    /// A HashMap of offsets which specify what scroll offsets particular
+    /// scroll layers should be initialized with.
+    scroll_offsets: HashMap<ClipId, LayerPoint>,
+
+    image_map: HashMap<(PathBuf, Option<i64>), (ImageKey, LayoutSize)>,
+
+    fonts: HashMap<FontDescriptor, FontKey>,
+    font_instances: HashMap<(FontKey, Au, FontInstanceFlags), FontInstanceKey>,
+    font_render_mode: Option<FontRenderMode>,
+
+    /// A HashMap that allows specifying a numeric id for clip and clip chains in YAML
+    /// and having each of those ids correspond to a unique ClipId.
+    clip_id_map: HashMap<u64, ClipId>,
+}
+
+impl YamlFrameReader {
+    pub fn new(yaml_path: &Path) -> YamlFrameReader {
+        YamlFrameReader {
+            watch_source: false,
+            list_resources: false,
+            frame_built: false,
+            yaml_path: yaml_path.to_owned(),
+            aux_dir: yaml_path.parent().unwrap().to_owned(),
+            frame_count: 0,
+            display_lists: Vec::new(),
+            queue_depth: 1,
+            include_only: vec![],
+            scroll_offsets: HashMap::new(),
+            fonts: HashMap::new(),
+            font_instances: HashMap::new(),
+            font_render_mode: None,
+            image_map: HashMap::new(),
+            clip_id_map: HashMap::new(),
+        }
+    }
+
+    pub fn deinit(mut self, wrench: &mut Wrench) {
+        let mut updates = ResourceUpdates::new();
+
+        for (_, font_instance) in self.font_instances.drain() {
+            updates.delete_font_instance(font_instance);
+        }
+
+        for (_, font) in self.fonts.drain() {
+            updates.delete_font(font);
+        }
+
+        wrench.api.update_resources(updates);
+    }
+
+    pub fn yaml_path(&self) -> &PathBuf {
+        &self.yaml_path
+    }
+
+    pub fn new_from_args(args: &clap::ArgMatches) -> YamlFrameReader {
+        let yaml_file = args.value_of("INPUT").map(|s| PathBuf::from(s)).unwrap();
+
+        let mut y = YamlFrameReader::new(&yaml_file);
+        y.list_resources = args.is_present("list-resources");
+        y.watch_source = args.is_present("watch");
+        y.queue_depth = args.value_of("queue")
+            .map(|s| s.parse::<u32>().unwrap())
+            .unwrap_or(1);
+        y.include_only = args.values_of("include")
+            .map(|v| v.map(|s| s.to_owned()).collect())
+            .unwrap_or(vec![]);
+        y
+    }
+
+    pub fn reset(&mut self) {
+        self.scroll_offsets.clear();
+        self.display_lists.clear();
+    }
+
+    pub fn build(&mut self, wrench: &mut Wrench) {
+        let mut file = File::open(&self.yaml_path).unwrap();
+        let mut src = String::new();
+        file.read_to_string(&mut src).unwrap();
+
+        let mut yaml_doc = YamlLoader::load_from_str(&src).expect("Failed to parse YAML file");
+        assert_eq!(yaml_doc.len(), 1);
+
+        self.reset();
+
+        let yaml = yaml_doc.pop().unwrap();
+        if let Some(pipelines) = yaml["pipelines"].as_vec() {
+            for pipeline in pipelines {
+                self.build_pipeline(wrench, pipeline["id"].as_pipeline_id().unwrap(), pipeline);
+            }
+        }
+
+        assert!(!yaml["root"].is_badvalue(), "Missing root stacking context");
+        let root_pipeline_id = wrench.root_pipeline_id;
+        self.build_pipeline(wrench, root_pipeline_id, &yaml["root"]);
+    }
+
+    pub fn build_pipeline(
+        &mut self,
+        wrench: &mut Wrench,
+        pipeline_id: PipelineId,
+        yaml: &Yaml
+    ) {
+        // Don't allow referencing clips between pipelines for now.
+        self.clip_id_map.clear();
+
+        let content_size = self.get_root_size_from_yaml(wrench, yaml);
+        let mut builder = DisplayListBuilder::new(pipeline_id, content_size);
+        let mut info = LayoutPrimitiveInfo::new(LayoutRect::zero());
+        self.add_stacking_context_from_yaml(&mut builder, wrench, yaml, true, &mut info);
+        self.display_lists.push(builder.finalize());
+    }
+
+    fn to_complex_clip_region(&mut self, item: &Yaml) -> ComplexClipRegion {
+        let rect = item["rect"]
+            .as_rect()
+            .expect("Complex clip entry must have rect");
+        let radius = item["radius"]
+            .as_border_radius()
+            .unwrap_or(BorderRadius::zero());
+        let mode = item["clip-mode"]
+            .as_clip_mode()
+            .unwrap_or(ClipMode::Clip);
+        ComplexClipRegion::new(rect, radius, mode)
+    }
+
+    fn to_complex_clip_regions(&mut self, item: &Yaml) -> Vec<ComplexClipRegion> {
+        match *item {
+            Yaml::Array(ref array) => array
+                .iter()
+                .map(|entry| self.to_complex_clip_region(entry))
+                .collect(),
+            Yaml::BadValue => vec![],
+            _ => {
+                println!("Unable to parse complex clip region {:?}", item);
+                vec![]
+            }
+        }
+    }
+
+    fn to_sticky_offset_bounds(&mut self, item: &Yaml) -> StickyOffsetBounds {
+        match *item {
+            Yaml::Array(ref array) => StickyOffsetBounds::new(
+                array[0].as_f32().unwrap_or(0.0),
+                array[1].as_f32().unwrap_or(0.0),
+            ),
+            _ => StickyOffsetBounds::new(0.0, 0.0),
+        }
+    }
+
+    pub fn to_clip_id(&self, id: u64, pipeline_id: PipelineId) -> ClipId {
+        if id == 0 {
+            return ClipId::root_scroll_node(pipeline_id);
+        }
+        self.clip_id_map[&id]
+    }
+
+    fn to_clip_and_scroll_info(
+        &self,
+        item: &Yaml,
+        pipeline_id: PipelineId
+    ) -> Option<ClipAndScrollInfo> {
+        match *item {
+            Yaml::Integer(value) => {
+                Some(ClipAndScrollInfo::simple(self.to_clip_id(value as u64, pipeline_id)))
+            }
+            Yaml::Array(ref array) if array.len() == 2 => {
+                let id_ints = (array[0].as_i64(), array[1].as_i64());
+                if let (Some(scroll_node_numeric_id), Some(clip_node_numeric_id)) = id_ints {
+                    Some(ClipAndScrollInfo::new(
+                        self.to_clip_id(scroll_node_numeric_id as u64, pipeline_id),
+                        self.to_clip_id(clip_node_numeric_id as u64, pipeline_id)
+                    ))
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
+
+    pub fn add_or_get_image(
+            &mut self,
+            file: &Path,
+            tiling: Option<i64>,
+            wrench: &mut Wrench,
+    ) -> (ImageKey, LayoutSize) {
+        let key = (file.to_owned(), tiling);
+        if let Some(k) = self.image_map.get(&key) {
+            return *k;
+        }
+
+        if self.list_resources { println!("{}", file.to_string_lossy()); }
+        let (descriptor, image_data) = match image::open(file) {
+            Ok(image) => {
+                let image_dims = image.dimensions();
+                let (format, bytes) = match image {
+                    image::ImageLuma8(_) => {
+                        (ImageFormat::R8, image.raw_pixels())
+                    }
+                    image::ImageRgba8(_) => {
+                        let mut pixels = image.raw_pixels();
+                        premultiply(pixels.as_mut_slice());
+                        (ImageFormat::BGRA8, pixels)
+                    }
+                    image::ImageRgb8(_) => {
+                        let bytes = image.raw_pixels();
+                        let mut pixels = Vec::new();
+                        for bgr in bytes.chunks(3) {
+                            pixels.extend_from_slice(&[
+                                bgr[2],
+                                bgr[1],
+                                bgr[0],
+                                0xff
+                            ]);
+                        }
+                        (ImageFormat::BGRA8, pixels)
+                    }
+                    _ => panic!("We don't support whatever your crazy image type is, come on"),
+                };
+                let descriptor = ImageDescriptor::new(
+                    image_dims.0,
+                    image_dims.1,
+                    format,
+                    is_image_opaque(format, &bytes[..]),
+                );
+                let data = ImageData::new(bytes);
+                (descriptor, data)
+            }
+            _ => {
+                // This is a hack but it is convenient when generating test cases and avoids
+                // bloating the repository.
+                match parse_function(
+                    file.components()
+                        .last()
+                        .unwrap()
+                        .as_os_str()
+                        .to_str()
+                        .unwrap(),
+                ) {
+                    ("xy-gradient", args, _) => generate_xy_gradient_image(
+                        args.get(0).unwrap_or(&"1000").parse::<u32>().unwrap(),
+                        args.get(1).unwrap_or(&"1000").parse::<u32>().unwrap(),
+                    ),
+                    ("solid-color", args, _) => generate_solid_color_image(
+                        args.get(0).unwrap_or(&"255").parse::<u8>().unwrap(),
+                        args.get(1).unwrap_or(&"255").parse::<u8>().unwrap(),
+                        args.get(2).unwrap_or(&"255").parse::<u8>().unwrap(),
+                        args.get(3).unwrap_or(&"255").parse::<u8>().unwrap(),
+                        args.get(4).unwrap_or(&"1000").parse::<u32>().unwrap(),
+                        args.get(5).unwrap_or(&"1000").parse::<u32>().unwrap(),
+                    ),
+                    ("checkerboard", args, _) => generate_checkerboard_image(
+                        args.get(0).unwrap_or(&"4").parse::<u32>().unwrap(),
+                        args.get(1).unwrap_or(&"32").parse::<u32>().unwrap(),
+                        args.get(2).unwrap_or(&"8").parse::<u32>().unwrap(),
+                    ),
+                    _ => {
+                        panic!("Failed to load image {:?}", file.to_str());
+                    }
+                }
+            }
+        };
+        let tiling = tiling.map(|tile_size| tile_size as u16);
+        let image_key = wrench.api.generate_image_key();
+        let mut resources = ResourceUpdates::new();
+        resources.add_image(image_key, descriptor, image_data, tiling);
+        wrench.api.update_resources(resources);
+        let val = (
+            image_key,
+            LayoutSize::new(descriptor.width as f32, descriptor.height as f32),
+        );
+        self.image_map.insert(key, val);
+        val
+    }
+
+    fn get_or_create_font(&mut self, desc: FontDescriptor, wrench: &mut Wrench) -> FontKey {
+        let list_resources = self.list_resources;
+        *self.fonts
+            .entry(desc.clone())
+            .or_insert_with(|| match desc {
+                FontDescriptor::Path {
+                    ref path,
+                    font_index,
+                } => {
+                    if list_resources { println!("{}", path.to_string_lossy()); }
+                    let mut file = File::open(path).expect("Couldn't open font file");
+                    let mut bytes = vec![];
+                    file.read_to_end(&mut bytes)
+                        .expect("failed to read font file");
+                    wrench.font_key_from_bytes(bytes, font_index)
+                }
+                FontDescriptor::Family { ref name } => wrench.font_key_from_name(name),
+                FontDescriptor::Properties {
+                    ref family,
+                    weight,
+                    style,
+                    stretch,
+                } => wrench.font_key_from_properties(family, weight, style, stretch),
+            })
+    }
+
+    pub fn set_font_render_mode(&mut self, render_mode: Option<FontRenderMode>) {
+        self.font_render_mode = render_mode;
+    }
+
+    fn get_or_create_font_instance(
+        &mut self,
+        font_key: FontKey,
+        size: Au,
+        flags: FontInstanceFlags,
+        wrench: &mut Wrench,
+    ) -> FontInstanceKey {
+        let font_render_mode = self.font_render_mode;
+
+        *self.font_instances
+            .entry((font_key, size, flags))
+            .or_insert_with(|| {
+                wrench.add_font_instance(
+                    font_key,
+                    size,
+                    flags,
+                    font_render_mode,
+                )
+            })
+    }
+
+    fn to_image_mask(&mut self, item: &Yaml, wrench: &mut Wrench) -> Option<ImageMask> {
+        if item.as_hash().is_none() {
+            return None;
+        }
+
+        let file = match item["image"].as_str() {
+            Some(filename) => {
+                let mut file = self.aux_dir.clone();
+                file.push(filename);
+                file
+            }
+            None => {
+                warn!("No image provided for the image-mask!");
+                return None;
+            }
+        };
+
+        let (image_key, image_dims) =
+            self.add_or_get_image(&file, None, wrench);
+        let image_rect = item["rect"]
+            .as_rect()
+            .unwrap_or(LayoutRect::new(LayoutPoint::zero(), image_dims));
+        let image_repeat = item["repeat"].as_bool().expect("Expected boolean");
+        Some(ImageMask {
+            image: image_key,
+            rect: image_rect,
+            repeat: image_repeat,
+        })
+    }
+
+    fn to_gradient(&mut self, dl: &mut DisplayListBuilder, item: &Yaml) -> Gradient {
+        let start = item["start"].as_point().expect("gradient must have start");
+        let end = item["end"].as_point().expect("gradient must have end");
+        let stops = item["stops"]
+            .as_vec()
+            .expect("gradient must have stops")
+            .chunks(2)
+            .map(|chunk| {
+                GradientStop {
+                    offset: chunk[0]
+                        .as_force_f32()
+                        .expect("gradient stop offset is not f32"),
+                    color: chunk[1]
+                        .as_colorf()
+                        .expect("gradient stop color is not color"),
+                }
+            })
+            .collect::<Vec<_>>();
+        let extend_mode = if item["repeat"].as_bool().unwrap_or(false) {
+            ExtendMode::Repeat
+        } else {
+            ExtendMode::Clamp
+        };
+
+        dl.create_gradient(start, end, stops, extend_mode)
+    }
+
+    fn to_radial_gradient(&mut self, dl: &mut DisplayListBuilder, item: &Yaml) -> RadialGradient {
+        if item["start-center"].is_badvalue() {
+            let center = item["center"]
+                .as_point()
+                .expect("radial gradient must have start center");
+            let radius = item["radius"]
+                .as_size()
+                .expect("radial gradient must have start radius");
+            let stops = item["stops"]
+                .as_vec()
+                .expect("radial gradient must have stops")
+                .chunks(2)
+                .map(|chunk| {
+                    GradientStop {
+                        offset: chunk[0]
+                            .as_force_f32()
+                            .expect("gradient stop offset is not f32"),
+                        color: chunk[1]
+                            .as_colorf()
+                            .expect("gradient stop color is not color"),
+                    }
+                })
+                .collect::<Vec<_>>();
+            let extend_mode = if item["repeat"].as_bool().unwrap_or(false) {
+                ExtendMode::Repeat
+            } else {
+                ExtendMode::Clamp
+            };
+
+            dl.create_radial_gradient(center, radius, stops, extend_mode)
+        } else {
+            let start_center = item["start-center"]
+                .as_point()
+                .expect("radial gradient must have start center");
+            let start_radius = item["start-radius"]
+                .as_force_f32()
+                .expect("radial gradient must have start radius");
+            let end_center = item["end-center"]
+                .as_point()
+                .expect("radial gradient must have end center");
+            let end_radius = item["end-radius"]
+                .as_force_f32()
+                .expect("radial gradient must have end radius");
+            let ratio_xy = item["ratio-xy"].as_force_f32().unwrap_or(1.0);
+            let stops = item["stops"]
+                .as_vec()
+                .expect("radial gradient must have stops")
+                .chunks(2)
+                .map(|chunk| {
+                    GradientStop {
+                        offset: chunk[0]
+                            .as_force_f32()
+                            .expect("gradient stop offset is not f32"),
+                        color: chunk[1]
+                            .as_colorf()
+                            .expect("gradient stop color is not color"),
+                    }
+                })
+                .collect::<Vec<_>>();
+            let extend_mode = if item["repeat"].as_bool().unwrap_or(false) {
+                ExtendMode::Repeat
+            } else {
+                ExtendMode::Clamp
+            };
+
+            dl.create_complex_radial_gradient(
+                start_center,
+                start_radius,
+                end_center,
+                end_radius,
+                ratio_xy,
+                stops,
+                extend_mode,
+            )
+        }
+    }
+
+    fn handle_rect(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let bounds_key = if item["type"].is_badvalue() {
+            "rect"
+        } else {
+            "bounds"
+        };
+        info.rect = item[bounds_key]
+            .as_rect()
+            .expect("rect type must have bounds");
+        let color = item["color"].as_colorf().unwrap_or(*WHITE_COLOR);
+        dl.push_rect(&info, color);
+    }
+
+    fn handle_clear_rect(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        info.rect = item["bounds"]
+            .as_rect()
+            .expect("clear-rect type must have bounds");
+        dl.push_clear_rect(&info);
+    }
+
+    fn handle_line(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let color = item["color"].as_colorf().unwrap_or(*BLACK_COLOR);
+        let orientation = item["orientation"]
+            .as_str()
+            .and_then(LineOrientation::from_str)
+            .expect("line must have orientation");
+        let style = item["style"]
+            .as_str()
+            .and_then(LineStyle::from_str)
+            .expect("line must have style");
+
+        let wavy_line_thickness = if let LineStyle::Wavy = style {
+            item["thickness"].as_f32().expect("wavy lines must have a thickness")
+        } else {
+            0.0
+        };
+
+        if item["baseline"].is_badvalue() {
+            let bounds_key = if item["type"].is_badvalue() {
+                "rect"
+            } else {
+                "bounds"
+            };
+
+            info.rect = item[bounds_key]
+                .as_rect()
+                .expect("line type must have bounds");
+        } else {
+            // Legacy line representation
+            let baseline = item["baseline"].as_f32().expect("line must have baseline");
+            let start = item["start"].as_f32().expect("line must have start");
+            let end = item["end"].as_f32().expect("line must have end");
+            let width = item["width"].as_f32().expect("line must have width");
+
+            info.rect = match orientation {
+                LineOrientation::Horizontal => {
+                    LayoutRect::new(LayoutPoint::new(start, baseline),
+                                    LayoutSize::new(end - start, width))
+                }
+                LineOrientation::Vertical => {
+                    LayoutRect::new(LayoutPoint::new(baseline, start),
+                                    LayoutSize::new(width, end - start))
+                }
+            };
+        }
+
+        dl.push_line(
+            &info,
+            wavy_line_thickness,
+            orientation,
+            &color,
+            style,
+        );
+    }
+
+    fn handle_gradient(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let bounds_key = if item["type"].is_badvalue() {
+            "gradient"
+        } else {
+            "bounds"
+        };
+        let bounds = item[bounds_key]
+            .as_rect()
+            .expect("gradient must have bounds");
+        info.rect = bounds;
+        let gradient = self.to_gradient(dl, item);
+        let tile_size = item["tile-size"].as_size().unwrap_or(bounds.size);
+        let tile_spacing = item["tile-spacing"].as_size().unwrap_or(LayoutSize::zero());
+
+        dl.push_gradient(&info, gradient, tile_size, tile_spacing);
+    }
+
+    fn handle_radial_gradient(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let bounds_key = if item["type"].is_badvalue() {
+            "radial-gradient"
+        } else {
+            "bounds"
+        };
+        let bounds = item[bounds_key]
+            .as_rect()
+            .expect("radial gradient must have bounds");
+        info.rect = bounds;
+        let gradient = self.to_radial_gradient(dl, item);
+        let tile_size = item["tile-size"].as_size().unwrap_or(bounds.size);
+        let tile_spacing = item["tile-spacing"].as_size().unwrap_or(LayoutSize::zero());
+
+        dl.push_radial_gradient(&info, gradient, tile_size, tile_spacing);
+    }
+
+    fn handle_border(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        wrench: &mut Wrench,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let bounds_key = if item["type"].is_badvalue() {
+            "border"
+        } else {
+            "bounds"
+        };
+        info.rect = item[bounds_key]
+            .as_rect()
+            .expect("borders must have bounds");
+        let widths = item["width"]
+            .as_vec_f32()
+            .expect("borders must have width(s)");
+        let widths = broadcast(&widths, 4);
+        let widths = BorderWidths {
+            top: widths[0],
+            left: widths[1],
+            bottom: widths[2],
+            right: widths[3],
+        };
+        let border_details = if let Some(border_type) = item["border-type"].as_str() {
+            match border_type {
+                "normal" => {
+                    let colors = item["color"]
+                        .as_vec_colorf()
+                        .expect("borders must have color(s)");
+                    let styles = item["style"]
+                        .as_vec_string()
+                        .expect("borders must have style(s)");
+                    let styles = styles
+                        .iter()
+                        .map(|s| match s.as_str() {
+                            "none" => BorderStyle::None,
+                            "solid" => BorderStyle::Solid,
+                            "double" => BorderStyle::Double,
+                            "dotted" => BorderStyle::Dotted,
+                            "dashed" => BorderStyle::Dashed,
+                            "hidden" => BorderStyle::Hidden,
+                            "ridge" => BorderStyle::Ridge,
+                            "inset" => BorderStyle::Inset,
+                            "outset" => BorderStyle::Outset,
+                            "groove" => BorderStyle::Groove,
+                            s => {
+                                panic!("Unknown border style '{}'", s);
+                            }
+                        })
+                        .collect::<Vec<BorderStyle>>();
+                    let radius = item["radius"]
+                        .as_border_radius()
+                        .unwrap_or(BorderRadius::zero());
+
+                    let colors = broadcast(&colors, 4);
+                    let styles = broadcast(&styles, 4);
+
+                    let top = BorderSide {
+                        color: colors[0],
+                        style: styles[0],
+                    };
+                    let right = BorderSide {
+                        color: colors[1],
+                        style: styles[1],
+                    };
+                    let bottom = BorderSide {
+                        color: colors[2],
+                        style: styles[2],
+                    };
+                    let left = BorderSide {
+                        color: colors[3],
+                        style: styles[3],
+                    };
+                    Some(BorderDetails::Normal(NormalBorder {
+                        top,
+                        left,
+                        bottom,
+                        right,
+                        radius,
+                    }))
+                }
+                "image" => {
+                    let file = rsrc_path(&item["image-source"], &self.aux_dir);
+                    let (image_key, _) = self
+                        .add_or_get_image(&file, None, wrench);
+                    let image_width = item["image-width"]
+                        .as_i64()
+                        .expect("border must have image-width");
+                    let image_height = item["image-height"]
+                        .as_i64()
+                        .expect("border must have image-height");
+                    let fill = item["fill"].as_bool().unwrap_or(false);
+                    let slice = item["slice"].as_vec_u32().expect("border must have slice");
+                    let slice = broadcast(&slice, 4);
+                    let outset = item["outset"]
+                        .as_vec_f32()
+                        .expect("border must have outset");
+                    let outset = broadcast(&outset, 4);
+                    let repeat_horizontal = match item["repeat-horizontal"]
+                        .as_str()
+                        .expect("border must have repeat-horizontal")
+                    {
+                        "stretch" => RepeatMode::Stretch,
+                        "repeat" => RepeatMode::Repeat,
+                        "round" => RepeatMode::Round,
+                        "space" => RepeatMode::Space,
+                        s => panic!("Unknown box border image repeat mode {}", s),
+                    };
+                    let repeat_vertical = match item["repeat-vertical"]
+                        .as_str()
+                        .expect("border must have repeat-vertical")
+                    {
+                        "stretch" => RepeatMode::Stretch,
+                        "repeat" => RepeatMode::Repeat,
+                        "round" => RepeatMode::Round,
+                        "space" => RepeatMode::Space,
+                        s => panic!("Unknown box border image repeat mode {}", s),
+                    };
+                    Some(BorderDetails::Image(ImageBorder {
+                        image_key,
+                        patch: NinePatchDescriptor {
+                            width: image_width as u32,
+                            height: image_height as u32,
+                            slice: SideOffsets2D::new(slice[0], slice[1], slice[2], slice[3]),
+                        },
+                        fill,
+                        outset: SideOffsets2D::new(outset[0], outset[1], outset[2], outset[3]),
+                        repeat_horizontal,
+                        repeat_vertical,
+                    }))
+                }
+                "gradient" => {
+                    let gradient = self.to_gradient(dl, item);
+                    let outset = item["outset"]
+                        .as_vec_f32()
+                        .expect("borders must have outset");
+                    let outset = broadcast(&outset, 4);
+                    Some(BorderDetails::Gradient(GradientBorder {
+                        gradient,
+                        outset: SideOffsets2D::new(outset[0], outset[1], outset[2], outset[3]),
+                    }))
+                }
+                "radial-gradient" => {
+                    let gradient = self.to_radial_gradient(dl, item);
+                    let outset = item["outset"]
+                        .as_vec_f32()
+                        .expect("borders must have outset");
+                    let outset = broadcast(&outset, 4);
+                    Some(BorderDetails::RadialGradient(RadialGradientBorder {
+                        gradient,
+                        outset: SideOffsets2D::new(outset[0], outset[1], outset[2], outset[3]),
+                    }))
+                }
+                _ => {
+                    println!("Unable to parse border {:?}", item);
+                    None
+                }
+            }
+        } else {
+            println!("Unable to parse border {:?}", item);
+            None
+        };
+        if let Some(details) = border_details {
+            dl.push_border(&info, widths, details);
+        }
+    }
+
+    fn handle_box_shadow(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let bounds_key = if item["type"].is_badvalue() {
+            "box-shadow"
+        } else {
+            "bounds"
+        };
+        let bounds = item[bounds_key]
+            .as_rect()
+            .expect("box shadow must have bounds");
+        info.rect = bounds;
+        let box_bounds = item["box-bounds"].as_rect().unwrap_or(bounds);
+        let offset = item["offset"].as_vector().unwrap_or(LayoutVector2D::zero());
+        let color = item["color"]
+            .as_colorf()
+            .unwrap_or(ColorF::new(0.0, 0.0, 0.0, 1.0));
+        let blur_radius = item["blur-radius"].as_force_f32().unwrap_or(0.0);
+        let spread_radius = item["spread-radius"].as_force_f32().unwrap_or(0.0);
+        let border_radius = item["border-radius"]
+            .as_border_radius()
+            .unwrap_or(BorderRadius::zero());
+        let clip_mode = if let Some(mode) = item["clip-mode"].as_str() {
+            match mode {
+                "outset" => BoxShadowClipMode::Outset,
+                "inset" => BoxShadowClipMode::Inset,
+                s => panic!("Unknown box shadow clip mode {}", s),
+            }
+        } else {
+            BoxShadowClipMode::Outset
+        };
+
+        dl.push_box_shadow(
+            &info,
+            box_bounds,
+            offset,
+            color,
+            blur_radius,
+            spread_radius,
+            border_radius,
+            clip_mode,
+        );
+    }
+
+    fn handle_image(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        wrench: &mut Wrench,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let filename = &item[if item["type"].is_badvalue() {
+                                 "image"
+                             } else {
+                                 "src"
+                             }];
+        let tiling = item["tile-size"].as_i64();
+        let file = rsrc_path(filename, &self.aux_dir);
+        let (image_key, image_dims) =
+            self.add_or_get_image(&file, tiling, wrench);
+
+        let bounds_raws = item["bounds"].as_vec_f32().unwrap();
+        info.rect = if bounds_raws.len() == 2 {
+            LayoutRect::new(LayoutPoint::new(bounds_raws[0], bounds_raws[1]), image_dims)
+        } else if bounds_raws.len() == 4 {
+            LayoutRect::new(
+                LayoutPoint::new(bounds_raws[0], bounds_raws[1]),
+                LayoutSize::new(bounds_raws[2], bounds_raws[3]),
+            )
+        } else {
+            panic!(
+                "image expected 2 or 4 values in bounds, got '{:?}'",
+                item["bounds"]
+            );
+        };
+
+        let stretch_size = item["stretch-size"].as_size().unwrap_or(image_dims);
+        let tile_spacing = item["tile-spacing"]
+            .as_size()
+            .unwrap_or(LayoutSize::new(0.0, 0.0));
+        let rendering = match item["rendering"].as_str() {
+            Some("auto") | None => ImageRendering::Auto,
+            Some("crisp-edges") => ImageRendering::CrispEdges,
+            Some("pixelated") => ImageRendering::Pixelated,
+            Some(_) => panic!(
+                "ImageRendering can be auto, crisp-edges, or pixelated -- got {:?}",
+                item
+            ),
+        };
+        let alpha_type = match item["alpha-type"].as_str() {
+            Some("premultiplied-alpha") | None => AlphaType::PremultipliedAlpha,
+            Some("alpha") => AlphaType::Alpha,
+            Some(_) => panic!(
+                "AlphaType can be premultiplied-alpha or alpha -- got {:?}",
+                item
+            ),
+        };
+        dl.push_image(&info, stretch_size, tile_spacing, rendering, alpha_type, image_key);
+    }
+
+    fn handle_text(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        wrench: &mut Wrench,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let size = item["size"].as_pt_to_au().unwrap_or(Au::from_f32_px(16.0));
+        let color = item["color"].as_colorf().unwrap_or(*BLACK_COLOR);
+        let mut flags = FontInstanceFlags::empty();
+        if item["synthetic-italics"].as_bool().unwrap_or(false) {
+            flags |= FontInstanceFlags::SYNTHETIC_ITALICS;
+        }
+        if item["synthetic-bold"].as_bool().unwrap_or(false) {
+            flags |= FontInstanceFlags::SYNTHETIC_BOLD;
+        }
+        if item["embedded-bitmaps"].as_bool().unwrap_or(false) {
+            flags |= FontInstanceFlags::EMBEDDED_BITMAPS;
+        }
+        if item["transpose"].as_bool().unwrap_or(false) {
+            flags |= FontInstanceFlags::TRANSPOSE;
+        }
+        if item["flip-x"].as_bool().unwrap_or(false) {
+            flags |= FontInstanceFlags::FLIP_X;
+        }
+        if item["flip-y"].as_bool().unwrap_or(false) {
+            flags |= FontInstanceFlags::FLIP_Y;
+        }
+
+        assert!(
+            item["blur-radius"].is_badvalue(),
+            "text no longer has a blur radius, use PushShadow and PopAllShadows"
+        );
+
+        let desc = FontDescriptor::from_yaml(item, &self.aux_dir);
+        let font_key = self.get_or_create_font(desc, wrench);
+        let font_instance_key = self.get_or_create_font_instance(font_key,
+                                                                 size,
+                                                                 flags,
+                                                                 wrench);
+
+        assert!(
+            !(item["glyphs"].is_badvalue() && item["text"].is_badvalue()),
+            "text item had neither text nor glyphs!"
+        );
+
+        let (glyphs, rect) = if item["text"].is_badvalue() {
+            // if glyphs are specified, then the glyph positions can have the
+            // origin baked in.
+            let origin = item["origin"]
+                .as_point()
+                .unwrap_or(LayoutPoint::new(0.0, 0.0));
+            let glyph_indices = item["glyphs"].as_vec_u32().unwrap();
+            let glyph_offsets = item["offsets"].as_vec_f32().unwrap();
+            assert_eq!(glyph_offsets.len(), glyph_indices.len() * 2);
+
+            let glyphs = glyph_indices
+                .iter()
+                .enumerate()
+                .map(|k| {
+                    GlyphInstance {
+                        index: *k.1,
+                        point: LayoutPoint::new(
+                            origin.x + glyph_offsets[k.0 * 2],
+                            origin.y + glyph_offsets[k.0 * 2 + 1],
+                        ),
+                    }
+                })
+                .collect::<Vec<_>>();
+            // TODO(gw): We could optionally use the WR API to query glyph dimensions
+            //           here and calculate the bounding region here if we want to.
+            let rect = item["bounds"]
+                .as_rect()
+                .expect("Text items with glyphs require bounds [for now]");
+            (glyphs, rect)
+        } else {
+            let text = item["text"].as_str().unwrap();
+            let origin = item["origin"]
+                .as_point()
+                .expect("origin required for text without glyphs");
+            let (glyph_indices, glyph_positions, bounds) = wrench.layout_simple_ascii(
+                font_key,
+                font_instance_key,
+                self.font_render_mode,
+                text,
+                size,
+                origin,
+                flags,
+            );
+
+            let glyphs = glyph_indices
+                .iter()
+                .zip(glyph_positions)
+                .map(|arg| {
+                    let gi = GlyphInstance {
+                        index: *arg.0 as u32,
+                        point: arg.1,
+                    };
+                    gi
+                })
+                .collect::<Vec<_>>();
+            (glyphs, bounds)
+        };
+        info.rect = rect;
+
+        dl.push_text(&info, &glyphs, font_instance_key, color, None);
+    }
+
+    fn handle_iframe(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        item: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        info.rect = item["bounds"].as_rect().expect("iframe must have bounds");
+        let pipeline_id = item["id"].as_pipeline_id().unwrap();
+        dl.push_iframe(&info, pipeline_id);
+    }
+
+    pub fn get_local_clip_for_item(&mut self, yaml: &Yaml, full_clip: LayoutRect) -> LocalClip {
+        let rect = yaml["clip-rect"].as_rect().unwrap_or(full_clip);
+        let complex_clip = &yaml["complex-clip"];
+        if !complex_clip.is_badvalue() {
+            LocalClip::RoundedRect(rect, self.to_complex_clip_region(complex_clip))
+        } else {
+            LocalClip::from(rect)
+        }
+    }
+
+    pub fn add_display_list_items_from_yaml(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        wrench: &mut Wrench,
+        yaml: &Yaml,
+    ) {
+        let full_clip = LayoutRect::new(LayoutPoint::zero(), wrench.window_size_f32());
+
+        for item in yaml.as_vec().unwrap() {
+            // an explicit type can be skipped with some shorthand
+            let item_type = if !item["rect"].is_badvalue() {
+                "rect"
+            } else if !item["image"].is_badvalue() {
+                "image"
+            } else if !item["text"].is_badvalue() {
+                "text"
+            } else if !item["glyphs"].is_badvalue() {
+                "glyphs"
+            } else if !item["box-shadow"].is_badvalue() {
+                // Note: box_shadow shorthand check has to come before border.
+                "box-shadow"
+            } else if !item["border"].is_badvalue() {
+                "border"
+            } else if !item["gradient"].is_badvalue() {
+                "gradient"
+            } else if !item["radial-gradient"].is_badvalue() {
+                "radial-gradient"
+            } else {
+                item["type"].as_str().unwrap_or("unknown")
+            };
+
+            // We never skip stacking contexts because they are structural elements
+            // of the display list.
+            if item_type != "stacking-context" && self.include_only.contains(&item_type.to_owned())
+            {
+                continue;
+            }
+
+            let clip_scroll_info = self.to_clip_and_scroll_info(
+                &item["clip-and-scroll"],
+                dl.pipeline_id
+            );
+            if let Some(clip_scroll_info) = clip_scroll_info {
+                dl.push_clip_and_scroll_info(clip_scroll_info);
+            }
+            let local_clip = self.get_local_clip_for_item(item, full_clip);
+            let mut info = LayoutPrimitiveInfo::with_clip(LayoutRect::zero(), local_clip);
+            info.is_backface_visible = item["backface-visible"].as_bool().unwrap_or(true);;
+            match item_type {
+                "rect" => self.handle_rect(dl, item, &mut info),
+                "clear-rect" => self.handle_clear_rect(dl, item, &mut info),
+                "line" => self.handle_line(dl, item, &mut info),
+                "image" => self.handle_image(dl, wrench, item, &mut info),
+                "text" | "glyphs" => self.handle_text(dl, wrench, item, &mut info),
+                "scroll-frame" => self.handle_scroll_frame(dl, wrench, item),
+                "sticky-frame" => self.handle_sticky_frame(dl, wrench, item),
+                "clip" => self.handle_clip(dl, wrench, item),
+                "clip-chain" => self.handle_clip_chain(dl, item),
+                "border" => self.handle_border(dl, wrench, item, &mut info),
+                "gradient" => self.handle_gradient(dl, item, &mut info),
+                "radial-gradient" => self.handle_radial_gradient(dl, item, &mut info),
+                "box-shadow" => self.handle_box_shadow(dl, item, &mut info),
+                "iframe" => self.handle_iframe(dl, item, &mut info),
+                "stacking-context" => {
+                    self.add_stacking_context_from_yaml(dl, wrench, item, false, &mut info)
+                }
+                "shadow" => self.handle_push_shadow(dl, item, &mut info),
+                "pop-all-shadows" => self.handle_pop_all_shadows(dl),
+                _ => println!("Skipping unknown item type: {:?}", item),
+            }
+
+            if clip_scroll_info.is_some() {
+                dl.pop_clip_id();
+            }
+        }
+    }
+
+    pub fn handle_scroll_frame(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        wrench: &mut Wrench,
+        yaml: &Yaml,
+    ) {
+        let clip_rect = yaml["bounds"]
+            .as_rect()
+            .expect("scroll frame must have a bounds");
+        let content_size = yaml["content-size"].as_size().unwrap_or(clip_rect.size);
+        let content_rect = LayerRect::new(clip_rect.origin, content_size);
+
+        let numeric_id = yaml["id"].as_i64().map(|id| id as u64);
+        let complex_clips = self.to_complex_clip_regions(&yaml["complex"]);
+        let image_mask = self.to_image_mask(&yaml["image-mask"], wrench);
+
+        let real_id = dl.define_scroll_frame(
+            None,
+            content_rect,
+            clip_rect,
+            complex_clips,
+            image_mask,
+            ScrollSensitivity::Script,
+        );
+        if let Some(numeric_id) = numeric_id {
+            self.clip_id_map.insert(numeric_id, real_id);
+        }
+
+        if let Some(size) = yaml["scroll-offset"].as_point() {
+            self.scroll_offsets.insert(real_id, LayerPoint::new(size.x, size.y));
+        }
+
+        if !yaml["items"].is_badvalue() {
+            dl.push_clip_id(real_id);
+            self.add_display_list_items_from_yaml(dl, wrench, &yaml["items"]);
+            dl.pop_clip_id();
+        }
+    }
+
+    pub fn handle_sticky_frame(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        wrench: &mut Wrench,
+        yaml: &Yaml,
+    ) {
+        let bounds = yaml["bounds"].as_rect().expect("sticky frame must have a bounds");
+        let numeric_id = yaml["id"].as_i64().map(|id| id as u64);
+
+        let real_id = dl.define_sticky_frame(
+            None,
+            bounds,
+            SideOffsets2D::new(
+                yaml["margin-top"].as_f32(),
+                yaml["margin-right"].as_f32(),
+                yaml["margin-bottom"].as_f32(),
+                yaml["margin-left"].as_f32(),
+            ),
+            self.to_sticky_offset_bounds(&yaml["vertical-offset-bounds"]),
+            self.to_sticky_offset_bounds(&yaml["horizontal-offset-bounds"]),
+            yaml["previously-applied-offset"].as_vector().unwrap_or(LayoutVector2D::zero()),
+        );
+
+        if let Some(numeric_id) = numeric_id {
+            self.clip_id_map.insert(numeric_id, real_id);
+        }
+
+        if !yaml["items"].is_badvalue() {
+            dl.push_clip_id(real_id);
+            self.add_display_list_items_from_yaml(dl, wrench, &yaml["items"]);
+            dl.pop_clip_id();
+        }
+    }
+
+    pub fn handle_push_shadow(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        yaml: &Yaml,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let rect = yaml["bounds"]
+            .as_rect()
+            .expect("Text shadows require bounds");
+        info.rect = rect;
+        info.local_clip = LocalClip::from(rect);
+        let blur_radius = yaml["blur-radius"].as_f32().unwrap_or(0.0);
+        let offset = yaml["offset"].as_vector().unwrap_or(LayoutVector2D::zero());
+        let color = yaml["color"].as_colorf().unwrap_or(*BLACK_COLOR);
+
+        dl.push_shadow(
+            &info,
+            Shadow {
+                blur_radius,
+                offset,
+                color,
+            },
+        );
+    }
+
+    pub fn handle_pop_all_shadows(&mut self, dl: &mut DisplayListBuilder) {
+        dl.pop_all_shadows();
+    }
+
+    pub fn handle_clip_chain(&mut self, builder: &mut DisplayListBuilder, yaml: &Yaml) {
+        let numeric_id = yaml["id"].as_i64().expect("clip chains must have an id");
+        let clip_ids: Vec<ClipId> = yaml["clips"]
+            .as_vec_u64()
+            .unwrap_or_else(Vec::new)
+            .iter().map(|id| self.to_clip_id(*id, builder.pipeline_id))
+            .collect();
+
+        let parent = yaml["parent"].as_i64().map(|id|
+            self.to_clip_id(id as u64, builder.pipeline_id)
+        );
+        let parent = match parent {
+            Some(ClipId::ClipChain(clip_chain_id)) => Some(clip_chain_id),
+            Some(_) => panic!("Tried to create a ClipChain with a non-ClipChain parent"),
+            None => None,
+        };
+
+        let real_id = builder.define_clip_chain(parent, clip_ids);
+        self.clip_id_map.insert(numeric_id as u64, ClipId::ClipChain(real_id));
+    }
+
+    pub fn handle_clip(&mut self, dl: &mut DisplayListBuilder, wrench: &mut Wrench, yaml: &Yaml) {
+        let clip_rect = yaml["bounds"].as_rect().expect("clip must have a bounds");
+        let numeric_id = yaml["id"].as_i64();
+        let complex_clips = self.to_complex_clip_regions(&yaml["complex"]);
+        let image_mask = self.to_image_mask(&yaml["image-mask"], wrench);
+
+        let real_id = dl.define_clip(None, clip_rect, complex_clips, image_mask);
+        if let Some(numeric_id) = numeric_id {
+            self.clip_id_map.insert(numeric_id as u64, real_id);
+        }
+
+        if !yaml["items"].is_badvalue() {
+            dl.push_clip_id(real_id);
+            self.add_display_list_items_from_yaml(dl, wrench, &yaml["items"]);
+            dl.pop_clip_id();
+        }
+    }
+
+    pub fn get_root_size_from_yaml(&mut self, wrench: &mut Wrench, yaml: &Yaml) -> LayoutSize {
+        yaml["bounds"]
+            .as_rect()
+            .map(|rect| rect.size)
+            .unwrap_or(wrench.window_size_f32())
+    }
+
+    pub fn add_stacking_context_from_yaml(
+        &mut self,
+        dl: &mut DisplayListBuilder,
+        wrench: &mut Wrench,
+        yaml: &Yaml,
+        is_root: bool,
+        info: &mut LayoutPrimitiveInfo,
+    ) {
+        let default_bounds = LayoutRect::new(LayoutPoint::zero(), wrench.window_size_f32());
+        let bounds = yaml["bounds"].as_rect().unwrap_or(default_bounds);
+
+        // TODO(gw): Add support for specifying the transform origin in yaml.
+        let transform_origin = LayoutPoint::new(
+            bounds.origin.x + bounds.size.width * 0.5,
+            bounds.origin.y + bounds.size.height * 0.5,
+        );
+
+        let transform = yaml["transform"]
+            .as_transform(&transform_origin)
+            .map(|transform| transform.into());
+
+        // TODO(gw): Support perspective-origin.
+        let perspective = match yaml["perspective"].as_f32() {
+            Some(value) if value != 0.0 => Some(LayoutTransform::create_perspective(value as f32)),
+            Some(_) => None,
+            _ => yaml["perspective"].as_matrix4d(),
+        };
+
+        let transform_style = yaml["transform-style"]
+            .as_transform_style()
+            .unwrap_or(TransformStyle::Flat);
+        let mix_blend_mode = yaml["mix-blend-mode"]
+            .as_mix_blend_mode()
+            .unwrap_or(MixBlendMode::Normal);
+        let scroll_policy = yaml["scroll-policy"]
+            .as_scroll_policy()
+            .unwrap_or(ScrollPolicy::Scrollable);
+
+        if is_root {
+            if let Some(size) = yaml["scroll-offset"].as_point() {
+                let id = ClipId::root_scroll_node(dl.pipeline_id);
+                self.scroll_offsets
+                    .insert(id, LayerPoint::new(size.x, size.y));
+            }
+        }
+
+        let filters = yaml["filters"].as_vec_filter_op().unwrap_or(vec![]);
+        info.rect = bounds;
+        info.local_clip = LocalClip::from(bounds);
+
+        dl.push_stacking_context(
+            &info,
+            scroll_policy,
+            transform.into(),
+            transform_style,
+            perspective,
+            mix_blend_mode,
+            filters,
+        );
+
+        if !yaml["items"].is_badvalue() {
+            self.add_display_list_items_from_yaml(dl, wrench, &yaml["items"]);
+        }
+
+        dl.pop_stacking_context();
+    }
+}
+
+impl WrenchThing for YamlFrameReader {
+    fn do_frame(&mut self, wrench: &mut Wrench) -> u32 {
+        if !self.frame_built || self.watch_source {
+            self.build(wrench);
+            self.frame_built = false;
+        }
+
+        self.frame_count += 1;
+
+        if !self.frame_built || wrench.should_rebuild_display_lists() {
+            wrench.begin_frame();
+            wrench.send_lists(
+                self.frame_count,
+                self.display_lists.clone(),
+                &self.scroll_offsets,
+            );
+        } else {
+            wrench.refresh();
+        }
+
+        self.frame_built = true;
+        self.frame_count
+    }
+
+    fn next_frame(&mut self) {}
+
+    fn prev_frame(&mut self) {}
+
+    fn queue_frames(&self) -> u32 {
+        self.queue_depth
+    }
+}
diff --git a/gfx/wrench/src/yaml_frame_writer.rs b/gfx/wrench/src/yaml_frame_writer.rs
new file mode 100644
index 000000000000..bbb047056957
--- /dev/null
+++ b/gfx/wrench/src/yaml_frame_writer.rs
@@ -0,0 +1,1207 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+extern crate yaml_rust;
+
+use app_units::Au;
+use euclid::{TypedPoint2D, TypedRect, TypedSize2D, TypedTransform3D, TypedVector2D};
+use image::{save_buffer, ColorType};
+use premultiply::unpremultiply;
+use scene::{Scene, SceneProperties};
+use std::collections::HashMap;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::{fmt, fs};
+use super::CURRENT_FRAME_NUMBER;
+use time;
+use webrender;
+use webrender::api::*;
+use webrender::api::SpecificDisplayItem::*;
+use webrender::api::channel::Payload;
+use yaml_helper::StringEnum;
+use yaml_rust::{Yaml, YamlEmitter};
+
+type Table = yaml_rust::yaml::Hash;
+
+fn array_elements_are_same<T: PartialEq>(v: &[T]) -> bool {
+    if !v.is_empty() {
+        let first = &v[0];
+        for o in v.iter() {
+            if *first != *o {
+                return false;
+            }
+        }
+    }
+    true
+}
+
+fn new_table() -> Table {
+    Table::new()
+}
+
+fn yaml_node(parent: &mut Table, key: &str, value: Yaml) {
+    parent.insert(Yaml::String(key.to_owned()), value);
+}
+
+fn str_node(parent: &mut Table, key: &str, value: &str) {
+    yaml_node(parent, key, Yaml::String(value.to_owned()));
+}
+
+fn path_node(parent: &mut Table, key: &str, value: &Path) {
+    let pstr = value.to_str().unwrap().to_owned().replace("\\", "/");
+    yaml_node(parent, key, Yaml::String(pstr));
+}
+
+fn enum_node<E: StringEnum>(parent: &mut Table, key: &str, value: E) {
+    yaml_node(parent, key, Yaml::String(value.as_str().to_owned()));
+}
+
+fn color_to_string(value: ColorF) -> String {
+    if value.r == 1.0 && value.g == 1.0 && value.b == 1.0 && value.a == 1.0 {
+        "white".to_owned()
+    } else if value.r == 0.0 && value.g == 0.0 && value.b == 0.0 && value.a == 1.0 {
+        "black".to_owned()
+    } else {
+        format!(
+            "{} {} {} {:.4}",
+            value.r * 255.0,
+            value.g * 255.0,
+            value.b * 255.0,
+            value.a
+        )
+    }
+}
+
+fn color_node(parent: &mut Table, key: &str, value: ColorF) {
+    yaml_node(parent, key, Yaml::String(color_to_string(value)));
+}
+
+fn point_node<U>(parent: &mut Table, key: &str, value: &TypedPoint2D<f32, U>) {
+    f32_vec_node(parent, key, &[value.x, value.y]);
+}
+
+fn vector_node<U>(parent: &mut Table, key: &str, value: &TypedVector2D<f32, U>) {
+    f32_vec_node(parent, key, &[value.x, value.y]);
+}
+
+fn size_node<U>(parent: &mut Table, key: &str, value: &TypedSize2D<f32, U>) {
+    f32_vec_node(parent, key, &[value.width, value.height]);
+}
+
+fn rect_yaml<U>(value: &TypedRect<f32, U>) -> Yaml {
+    f32_vec_yaml(
+        &[
+            value.origin.x,
+            value.origin.y,
+            value.size.width,
+            value.size.height,
+        ],
+        false,
+    )
+}
+
+fn rect_node<U>(parent: &mut Table, key: &str, value: &TypedRect<f32, U>) {
+    yaml_node(parent, key, rect_yaml(value));
+}
+
+fn matrix4d_node<U1, U2>(parent: &mut Table, key: &str, value: &TypedTransform3D<f32, U1, U2>) {
+    f32_vec_node(parent, key, &value.to_row_major_array());
+}
+
+fn u32_node(parent: &mut Table, key: &str, value: u32) {
+    yaml_node(parent, key, Yaml::Integer(value as i64));
+}
+
+fn usize_node(parent: &mut Table, key: &str, value: usize) {
+    yaml_node(parent, key, Yaml::Integer(value as i64));
+}
+
+fn f32_node(parent: &mut Table, key: &str, value: f32) {
+    yaml_node(parent, key, Yaml::Real(value.to_string()));
+}
+
+fn bool_node(parent: &mut Table, key: &str, value: bool) {
+    yaml_node(parent, key, Yaml::Boolean(value));
+}
+
+fn table_node(parent: &mut Table, key: &str, value: Table) {
+    yaml_node(parent, key, Yaml::Hash(value));
+}
+
+fn string_vec_yaml(value: &[String], check_unique: bool) -> Yaml {
+    if !value.is_empty() && check_unique && array_elements_are_same(value) {
+        Yaml::String(value[0].clone())
+    } else {
+        Yaml::Array(value.iter().map(|v| Yaml::String(v.clone())).collect())
+    }
+}
+
+fn u32_vec_yaml(value: &[u32], check_unique: bool) -> Yaml {
+    if !value.is_empty() && check_unique && array_elements_are_same(value) {
+        Yaml::Integer(value[0] as i64)
+    } else {
+        Yaml::Array(value.iter().map(|v| Yaml::Integer(*v as i64)).collect())
+    }
+}
+
+fn u32_vec_node(parent: &mut Table, key: &str, value: &[u32]) {
+    yaml_node(parent, key, u32_vec_yaml(value, false));
+}
+
+fn f32_vec_yaml(value: &[f32], check_unique: bool) -> Yaml {
+    if !value.is_empty() && check_unique && array_elements_are_same(value) {
+        Yaml::Real(value[0].to_string())
+    } else {
+        Yaml::Array(value.iter().map(|v| Yaml::Real(v.to_string())).collect())
+    }
+}
+
+fn f32_vec_node(parent: &mut Table, key: &str, value: &[f32]) {
+    yaml_node(parent, key, f32_vec_yaml(value, false));
+}
+
+fn maybe_radius_yaml(radius: &BorderRadius) -> Option<Yaml> {
+    if let Some(radius) = radius.is_uniform_size() {
+        if radius == LayoutSize::zero() {
+            None
+        } else {
+            Some(f32_vec_yaml(&[radius.width, radius.height], false))
+        }
+    } else {
+        let mut table = new_table();
+        size_node(&mut table, "top-left", &radius.top_left);
+        size_node(&mut table, "top-right", &radius.top_right);
+        size_node(&mut table, "bottom-left", &radius.bottom_left);
+        size_node(&mut table, "bottom-right", &radius.bottom_right);
+        Some(Yaml::Hash(table))
+    }
+}
+
+fn write_sc(parent: &mut Table, sc: &StackingContext, properties: &SceneProperties, filter_iter: AuxIter<FilterOp>) {
+    enum_node(parent, "scroll-policy", sc.scroll_policy);
+
+    matrix4d_node(parent, "transform", &properties.resolve_layout_transform(&sc.transform));
+
+    enum_node(parent, "transform-style", sc.transform_style);
+
+    if let Some(perspective) = sc.perspective {
+        matrix4d_node(parent, "perspective", &perspective);
+    }
+
+    // mix_blend_mode
+    if sc.mix_blend_mode != MixBlendMode::Normal {
+        enum_node(parent, "mix-blend-mode", sc.mix_blend_mode)
+    }
+    // filters
+    let mut filters = vec![];
+    for filter in filter_iter {
+        match filter {
+            FilterOp::Blur(x) => { filters.push(Yaml::String(format!("blur({})", x))) }
+            FilterOp::Brightness(x) => { filters.push(Yaml::String(format!("brightness({})", x))) }
+            FilterOp::Contrast(x) => { filters.push(Yaml::String(format!("contrast({})", x))) }
+            FilterOp::Grayscale(x) => { filters.push(Yaml::String(format!("grayscale({})", x))) }
+            FilterOp::HueRotate(x) => { filters.push(Yaml::String(format!("hue-rotate({})", x))) }
+            FilterOp::Invert(x) => { filters.push(Yaml::String(format!("invert({})", x))) }
+            FilterOp::Opacity(x, _) => {
+                filters.push(Yaml::String(format!("opacity({})",
+                                                  properties.resolve_float(&x, 1.0))))
+            }
+            FilterOp::Saturate(x) => { filters.push(Yaml::String(format!("saturate({})", x))) }
+            FilterOp::Sepia(x) => { filters.push(Yaml::String(format!("sepia({})", x))) }
+            FilterOp::DropShadow(offset, blur, color) => {
+                filters.push(Yaml::String(format!("drop-shadow([{},{}],{},[{}])",
+                                                  offset.x, offset.y,
+                                                  blur,
+                                                  color_to_string(color))))
+            }
+        }
+    }
+
+    yaml_node(parent, "filters", Yaml::Array(filters));
+}
+
+#[cfg(target_os = "windows")]
+fn native_font_handle_to_yaml(
+    _rsrc: &mut ResourceGenerator,
+    handle: &NativeFontHandle,
+    parent: &mut yaml_rust::yaml::Hash,
+    _: &mut Option<PathBuf>,
+) {
+    str_node(parent, "family", &handle.family_name);
+    u32_node(parent, "weight", handle.weight.to_u32());
+    u32_node(parent, "style", handle.style.to_u32());
+    u32_node(parent, "stretch", handle.stretch.to_u32());
+}
+
+#[cfg(target_os = "macos")]
+fn native_font_handle_to_yaml(
+    rsrc: &mut ResourceGenerator,
+    handle: &NativeFontHandle,
+    parent: &mut yaml_rust::yaml::Hash,
+    path_opt: &mut Option<PathBuf>,
+) {
+    let path = match *path_opt {
+        Some(ref path) => { path.clone() },
+        None => {
+            use cgfont_to_data;
+            let bytes = cgfont_to_data::font_to_data(handle.0.clone()).unwrap();
+            let (path_file, path) = rsrc.next_rsrc_paths(
+                "font",
+                "ttf",
+            );
+            let mut file = fs::File::create(&path_file).unwrap();
+            file.write_all(&bytes).unwrap();
+            *path_opt = Some(path.clone());
+            path
+        }
+    };
+
+    path_node(parent, "font", &path);
+}
+
+#[cfg(not(any(target_os = "macos", target_os = "windows")))]
+fn native_font_handle_to_yaml(
+    _rsrc: &mut ResourceGenerator,
+    handle: &NativeFontHandle,
+    parent: &mut yaml_rust::yaml::Hash,
+    _: &mut Option<PathBuf>,
+) {
+    str_node(parent, "font", &handle.pathname);
+}
+
+enum CachedFont {
+    Native(NativeFontHandle, Option<PathBuf>),
+    Raw(Option<Vec<u8>>, u32, Option<PathBuf>),
+}
+
+struct CachedFontInstance {
+    font_key: FontKey,
+    glyph_size: Au,
+}
+
+struct CachedImage {
+    width: u32,
+    height: u32,
+    stride: u32,
+    format: ImageFormat,
+    bytes: Option<Vec<u8>>,
+    path: Option<PathBuf>,
+    tiling: Option<u16>,
+}
+
+struct ResourceGenerator {
+    base: PathBuf,
+    next_num: u32,
+    prefix: String,
+}
+
+impl ResourceGenerator {
+    fn next_rsrc_paths(&mut self, base: &str, ext: &str) -> (PathBuf, PathBuf) {
+        let mut path_file = self.base.to_owned();
+        let mut path = PathBuf::from("res");
+
+        let fstr = format!("{}-{}-{}.{}", self.prefix, base, self.next_num, ext);
+        path_file.push(&fstr);
+        path.push(&fstr);
+
+        self.next_num += 1;
+
+        (path_file, path)
+    }
+}
+
+pub struct YamlFrameWriter {
+    frame_base: PathBuf,
+    rsrc_gen: ResourceGenerator,
+    images: HashMap<ImageKey, CachedImage>,
+    fonts: HashMap<FontKey, CachedFont>,
+    font_instances: HashMap<FontInstanceKey, CachedFontInstance>,
+
+    last_frame_written: u32,
+    pipeline_id: Option<PipelineId>,
+
+    dl_descriptor: Option<BuiltDisplayListDescriptor>,
+}
+
+pub struct YamlFrameWriterReceiver {
+    frame_writer: YamlFrameWriter,
+    scene: Scene,
+}
+
+impl YamlFrameWriterReceiver {
+    pub fn new(path: &Path) -> YamlFrameWriterReceiver {
+        YamlFrameWriterReceiver {
+            frame_writer: YamlFrameWriter::new(path),
+            scene: Scene::new(),
+        }
+    }
+}
+
+impl fmt::Debug for YamlFrameWriterReceiver {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "YamlFrameWriterReceiver")
+    }
+}
+
+impl YamlFrameWriter {
+    pub fn new(path: &Path) -> YamlFrameWriter {
+        let mut rsrc_base = path.to_owned();
+        rsrc_base.push("res");
+        fs::create_dir_all(&rsrc_base).ok();
+
+        let rsrc_prefix = format!("{}", time::get_time().sec);
+
+        YamlFrameWriter {
+            frame_base: path.to_owned(),
+            rsrc_gen: ResourceGenerator {
+                base: rsrc_base,
+                prefix: rsrc_prefix,
+                next_num: 1,
+            },
+            images: HashMap::new(),
+            fonts: HashMap::new(),
+            font_instances: HashMap::new(),
+
+            dl_descriptor: None,
+
+            pipeline_id: None,
+
+            last_frame_written: u32::max_value(),
+        }
+    }
+
+    pub fn begin_write_display_list(
+        &mut self,
+        scene: &mut Scene,
+        epoch: &Epoch,
+        pipeline_id: &PipelineId,
+        background_color: &Option<ColorF>,
+        viewport_size: &LayoutSize,
+        display_list: &BuiltDisplayListDescriptor,
+    ) {
+        unsafe {
+            if CURRENT_FRAME_NUMBER == self.last_frame_written {
+                return;
+            }
+            self.last_frame_written = CURRENT_FRAME_NUMBER;
+        }
+
+        self.dl_descriptor = Some(display_list.clone());
+        self.pipeline_id = Some(pipeline_id.clone());
+
+        scene.begin_display_list(pipeline_id, epoch, background_color, viewport_size);
+    }
+
+    pub fn finish_write_display_list(&mut self, scene: &mut Scene, data: &[u8]) {
+        let dl_desc = self.dl_descriptor.take().unwrap();
+
+        let payload = Payload::from_data(data);
+
+        let dl = BuiltDisplayList::from_data(payload.display_list_data, dl_desc);
+
+        let mut root_dl_table = new_table();
+        {
+            let mut iter = dl.iter();
+            self.write_display_list(&mut root_dl_table, &dl, scene, &mut iter, &mut ClipIdMapper::new());
+        }
+
+        let mut root = new_table();
+        if let Some(root_pipeline_id) = scene.root_pipeline_id {
+            u32_vec_node(
+                &mut root_dl_table,
+                "id",
+                &[root_pipeline_id.0, root_pipeline_id.1],
+            );
+
+            let mut referenced_pipeline_ids = vec![];
+            let mut traversal = dl.iter();
+            while let Some(item) = traversal.next() {
+                if let &SpecificDisplayItem::Iframe(k) = item.item() {
+                    referenced_pipeline_ids.push(k.pipeline_id);
+                }
+            }
+
+            let mut pipelines = vec![];
+            for pipeline_id in referenced_pipeline_ids {
+                if !scene.display_lists.contains_key(&pipeline_id) {
+                    continue;
+                }
+                let mut pipeline = new_table();
+                u32_vec_node(&mut pipeline, "id", &[pipeline_id.0, pipeline_id.1]);
+
+                let dl = scene.display_lists.get(&pipeline_id).unwrap();
+                let mut iter = dl.iter();
+                self.write_display_list(&mut pipeline, &dl, scene, &mut iter, &mut ClipIdMapper::new());
+                pipelines.push(Yaml::Hash(pipeline));
+            }
+
+            table_node(&mut root, "root", root_dl_table);
+
+            root.insert(Yaml::String("pipelines".to_owned()), Yaml::Array(pipelines));
+
+            let mut s = String::new();
+            // FIXME YamlEmitter wants a std::fmt::Write, not a io::Write, so we can't pass a file
+            // directly.  This seems broken.
+            {
+                let mut emitter = YamlEmitter::new(&mut s);
+                emitter.dump(&Yaml::Hash(root)).unwrap();
+            }
+            let sb = s.into_bytes();
+            let mut frame_file_name = self.frame_base.clone();
+            let current_shown_frame = unsafe { CURRENT_FRAME_NUMBER };
+            frame_file_name.push(format!("frame-{}.yaml", current_shown_frame));
+            let mut file = fs::File::create(&frame_file_name).unwrap();
+            file.write_all(&sb).unwrap();
+        }
+
+        scene.finish_display_list(self.pipeline_id.unwrap(), dl);
+    }
+
+    fn update_resources(&mut self, updates: &ResourceUpdates) {
+        for update in &updates.updates {
+            match *update {
+                ResourceUpdate::AddImage(ref img) => {
+                    if let Some(ref data) = self.images.get(&img.key) {
+                          if data.path.is_some() {
+                              return;
+                          }
+                    }
+
+                    let stride = img.descriptor.stride.unwrap_or(
+                        img.descriptor.width * img.descriptor.format.bytes_per_pixel(),
+                    );
+                    let bytes = match img.data {
+                        ImageData::Raw(ref v) => (**v).clone(),
+                        ImageData::External(_) | ImageData::Blob(_) => {
+                            return;
+                        }
+                    };
+                    self.images.insert(
+                        img.key,
+                        CachedImage {
+                            width: img.descriptor.width,
+                            height: img.descriptor.height,
+                            stride,
+                            format: img.descriptor.format,
+                            bytes: Some(bytes),
+                            tiling: img.tiling,
+                            path: None,
+                        },
+                    );
+                }
+                ResourceUpdate::UpdateImage(ref img) => {
+                    if let Some(ref mut data) = self.images.get_mut(&img.key) {
+                        assert_eq!(data.width, img.descriptor.width);
+                        assert_eq!(data.height, img.descriptor.height);
+                        assert_eq!(data.format, img.descriptor.format);
+
+                        if let ImageData::Raw(ref bytes) = img.data {
+                            data.path = None;
+                            data.bytes = Some((**bytes).clone());
+                        } else {
+                            // Other existing image types only make sense within the gecko integration.
+                            println!(
+                                "Wrench only supports updating buffer images ({}).",
+                                "ignoring update command"
+                            );
+                        }
+                    }
+                }
+                ResourceUpdate::DeleteImage(img) => {
+                    self.images.remove(&img);
+                }
+                ResourceUpdate::AddFont(ref font) => match font {
+                    &AddFont::Raw(key, ref bytes, index) => {
+                        self.fonts
+                            .insert(key, CachedFont::Raw(Some(bytes.clone()), index, None));
+                    }
+                    &AddFont::Native(key, ref handle) => {
+                        self.fonts.insert(key, CachedFont::Native(handle.clone(), None));
+                    }
+                },
+                ResourceUpdate::DeleteFont(_) => {}
+                ResourceUpdate::AddFontInstance(ref instance) => {
+                    self.font_instances.insert(
+                        instance.key,
+                        CachedFontInstance {
+                            font_key: instance.font_key,
+                            glyph_size: instance.glyph_size,
+                        },
+                    );
+                }
+                ResourceUpdate::DeleteFontInstance(_) => {}
+            }
+        }
+    }
+
+
+
+    fn path_for_image(&mut self, key: ImageKey) -> Option<PathBuf> {
+        let data = match self.images.get_mut(&key) {
+            Some(data) => data,
+            None => return None,
+        };
+
+        if data.path.is_some() {
+            return data.path.clone();
+        }
+        let mut bytes = data.bytes.take().unwrap();
+        let (path_file, path) = self.rsrc_gen.next_rsrc_paths(
+            "img",
+            "png",
+        );
+
+        assert!(data.stride > 0);
+        let (color_type, bpp) = match data.format {
+            ImageFormat::BGRA8 => (ColorType::RGBA(8), 4),
+            ImageFormat::R8 => (ColorType::Gray(8), 1),
+            _ => {
+                println!(
+                    "Failed to write image with format {:?}, dimensions {}x{}, stride {}",
+                    data.format,
+                    data.width,
+                    data.height,
+                    data.stride
+                );
+                return None;
+            }
+        };
+
+        if data.stride == data.width * bpp {
+            if data.format == ImageFormat::BGRA8 {
+                unpremultiply(bytes.as_mut_slice());
+            }
+            save_buffer(&path_file, &bytes, data.width, data.height, color_type).unwrap();
+        } else {
+            // takes a buffer with a stride and copies it into a new buffer that has stride == width
+            assert!(data.stride > data.width * bpp);
+            let mut tmp: Vec<_> = bytes[..]
+                .chunks(data.stride as usize)
+                .flat_map(|chunk| {
+                    chunk[.. (data.width * bpp) as usize].iter().cloned()
+                })
+                .collect();
+            if data.format == ImageFormat::BGRA8 {
+                unpremultiply(tmp.as_mut_slice());
+            }
+
+            save_buffer(&path_file, &tmp, data.width, data.height, color_type).unwrap();
+        }
+
+        data.path = Some(path.clone());
+        Some(path)
+    }
+
+    fn make_complex_clip_node(&mut self, complex_clip: &ComplexClipRegion) -> Yaml {
+        let mut t = new_table();
+        rect_node(&mut t, "rect", &complex_clip.rect);
+        yaml_node(
+            &mut t,
+            "radius",
+            maybe_radius_yaml(&complex_clip.radii).unwrap(),
+        );
+        enum_node(&mut t, "clip-mode", complex_clip.mode);
+        Yaml::Hash(t)
+    }
+
+    fn make_complex_clips_node(
+        &mut self,
+        complex_clip_count: usize,
+        complex_clips: ItemRange<ComplexClipRegion>,
+        list: &BuiltDisplayList,
+    ) -> Option<Yaml> {
+        if complex_clip_count == 0 {
+            return None;
+        }
+
+        let complex_items = list.get(complex_clips)
+            .map(|ccx| if ccx.radii.is_zero() {
+                rect_yaml(&ccx.rect)
+            } else {
+                self.make_complex_clip_node(&ccx)
+            })
+            .collect();
+        Some(Yaml::Array(complex_items))
+    }
+
+    fn make_clip_mask_image_node(&mut self, image_mask: &Option<ImageMask>) -> Option<Yaml> {
+        let mask = match image_mask {
+            &Some(ref mask) => mask,
+            &None => return None,
+        };
+
+        let mut mask_table = new_table();
+        if let Some(path) = self.path_for_image(mask.image) {
+            path_node(&mut mask_table, "image", &path);
+        }
+        rect_node(&mut mask_table, "rect", &mask.rect);
+        bool_node(&mut mask_table, "repeat", mask.repeat);
+        Some(Yaml::Hash(mask_table))
+    }
+
+    fn write_display_list_items(
+        &mut self,
+        list: &mut Vec<Yaml>,
+        display_list: &BuiltDisplayList,
+        scene: &Scene,
+        list_iterator: &mut BuiltDisplayListIter,
+        clip_id_mapper: &mut ClipIdMapper,
+    ) {
+        // continue_traversal is a big borrowck hack
+        let mut continue_traversal = None;
+        loop {
+            if let Some(traversal) = continue_traversal.take() {
+                *list_iterator = traversal;
+            }
+            let base = match list_iterator.next() {
+                Some(base) => base,
+                None => break,
+            };
+
+            let mut v = new_table();
+            rect_node(&mut v, "bounds", &base.rect());
+
+            rect_node(&mut v, "clip-rect", base.local_clip().clip_rect());
+            if let &LocalClip::RoundedRect(_, ref region) = base.local_clip() {
+                yaml_node(&mut v, "complex-clip", self.make_complex_clip_node(region));
+            }
+
+            let clip_and_scroll_yaml = match clip_id_mapper.map_info(&base.clip_and_scroll()) {
+                (scroll_id, Some(clip_id)) => {
+                    Yaml::Array(vec![Yaml::Integer(scroll_id), Yaml::Integer(clip_id)])
+                }
+                (scroll_id, None) => Yaml::Integer(scroll_id),
+            };
+            yaml_node(&mut v, "clip-and-scroll", clip_and_scroll_yaml);
+            bool_node(&mut v, "backface-visible", base.is_backface_visible());
+
+            match *base.item() {
+                Rectangle(item) => {
+                    str_node(&mut v, "type", "rect");
+                    color_node(&mut v, "color", item.color);
+                }
+                ClearRectangle => {
+                    str_node(&mut v, "type", "clear-rect");;
+                }
+                Line(item) => {
+                    str_node(&mut v, "type", "line");
+                    if let LineStyle::Wavy = item.style {
+                        f32_node(&mut v, "thickness", item.wavy_line_thickness);
+                    }
+                    str_node(&mut v, "orientation", item.orientation.as_str());
+                    color_node(&mut v, "color", item.color);
+                    str_node(&mut v, "style", item.style.as_str());
+                }
+                Text(item) => {
+                    let gi = display_list.get(base.glyphs());
+                    let mut indices: Vec<u32> = vec![];
+                    let mut offsets: Vec<f32> = vec![];
+                    for g in gi {
+                        indices.push(g.index);
+                        offsets.push(g.point.x);
+                        offsets.push(g.point.y);
+                    }
+                    u32_vec_node(&mut v, "glyphs", &indices);
+                    f32_vec_node(&mut v, "offsets", &offsets);
+
+                    let instance = self.font_instances.entry(item.font_key).or_insert_with(|| {
+                        println!("Warning: font instance key not found in font instances table!");
+                        CachedFontInstance {
+                            font_key: FontKey::new(IdNamespace(0), 0),
+                            glyph_size: Au::from_px(16),
+                        }
+                    });
+
+                    f32_node(
+                        &mut v,
+                        "size",
+                        instance.glyph_size.to_f32_px() * 12.0 / 16.0,
+                    );
+                    color_node(&mut v, "color", item.color);
+
+                    let entry = self.fonts.entry(instance.font_key).or_insert_with(|| {
+                        println!("Warning: font key not found in fonts table!");
+                        CachedFont::Raw(Some(vec![]), 0, None)
+                    });
+
+                    match entry {
+                        &mut CachedFont::Native(ref handle, ref mut path_opt) => {
+                            native_font_handle_to_yaml(&mut self.rsrc_gen, handle, &mut v, path_opt);
+                        }
+                        &mut CachedFont::Raw(ref mut bytes_opt, index, ref mut path_opt) => {
+                            if let Some(bytes) = bytes_opt.take() {
+                                let (path_file, path) = self.rsrc_gen.next_rsrc_paths(
+                                    "font",
+                                    "ttf",
+                                );
+                                let mut file = fs::File::create(&path_file).unwrap();
+                                file.write_all(&bytes).unwrap();
+                                *path_opt = Some(path);
+                            }
+
+                            path_node(&mut v, "font", path_opt.as_ref().unwrap());
+                            if index != 0 {
+                                u32_node(&mut v, "font-index", index);
+                            }
+                        }
+                    }
+                }
+                Image(item) => {
+                    if let Some(path) = self.path_for_image(item.image_key) {
+                        path_node(&mut v, "image", &path);
+                    }
+                    if let Some(&CachedImage {
+                        tiling: Some(tile_size),
+                        ..
+                    }) = self.images.get(&item.image_key)
+                    {
+                        u32_node(&mut v, "tile-size", tile_size as u32);
+                    }
+                    size_node(&mut v, "stretch-size", &item.stretch_size);
+                    size_node(&mut v, "tile-spacing", &item.tile_spacing);
+                    match item.image_rendering {
+                        ImageRendering::Auto => (),
+                        ImageRendering::CrispEdges => str_node(&mut v, "rendering", "crisp-edges"),
+                        ImageRendering::Pixelated => str_node(&mut v, "rendering", "pixelated"),
+                    };
+                    match item.alpha_type {
+                        AlphaType::PremultipliedAlpha => str_node(&mut v, "alpha-type", "premultiplied-alpha"),
+                        AlphaType::Alpha => str_node(&mut v, "alpha-type", "alpha"),
+                    };
+                }
+                YuvImage(_) => {
+                    str_node(&mut v, "type", "yuv-image");
+                    // TODO
+                    println!("TODO YAML YuvImage");
+                }
+                Border(item) => {
+                    str_node(&mut v, "type", "border");
+                    match item.details {
+                        BorderDetails::Normal(ref details) => {
+                            let trbl =
+                                vec![&details.top, &details.right, &details.bottom, &details.left];
+                            let widths: Vec<f32> = vec![
+                                item.widths.top,
+                                item.widths.right,
+                                item.widths.bottom,
+                                item.widths.left,
+                            ];
+                            let colors: Vec<String> =
+                                trbl.iter().map(|x| color_to_string(x.color)).collect();
+                            let styles: Vec<String> = trbl.iter()
+                                .map(|x| {
+                                    match x.style {
+                                        BorderStyle::None => "none",
+                                        BorderStyle::Solid => "solid",
+                                        BorderStyle::Double => "double",
+                                        BorderStyle::Dotted => "dotted",
+                                        BorderStyle::Dashed => "dashed",
+                                        BorderStyle::Hidden => "hidden",
+                                        BorderStyle::Ridge => "ridge",
+                                        BorderStyle::Inset => "inset",
+                                        BorderStyle::Outset => "outset",
+                                        BorderStyle::Groove => "groove",
+                                    }.to_owned()
+                                })
+                                .collect();
+                            yaml_node(&mut v, "width", f32_vec_yaml(&widths, true));
+                            str_node(&mut v, "border-type", "normal");
+                            yaml_node(&mut v, "color", string_vec_yaml(&colors, true));
+                            yaml_node(&mut v, "style", string_vec_yaml(&styles, true));
+                            if let Some(radius_node) = maybe_radius_yaml(&details.radius) {
+                                yaml_node(&mut v, "radius", radius_node);
+                            }
+                        }
+                        BorderDetails::Image(ref details) => {
+                            let widths: Vec<f32> = vec![
+                                item.widths.top,
+                                item.widths.right,
+                                item.widths.bottom,
+                                item.widths.left,
+                            ];
+                            let outset: Vec<f32> = vec![
+                                details.outset.top,
+                                details.outset.right,
+                                details.outset.bottom,
+                                details.outset.left,
+                            ];
+                            yaml_node(&mut v, "width", f32_vec_yaml(&widths, true));
+                            str_node(&mut v, "border-type", "image");
+                            if let Some(path) = self.path_for_image(details.image_key) {
+                                path_node(&mut v, "image", &path);
+                            }
+                            u32_node(&mut v, "image-width", details.patch.width);
+                            u32_node(&mut v, "image-height", details.patch.height);
+                            let slice: Vec<u32> = vec![
+                                details.patch.slice.top,
+                                details.patch.slice.right,
+                                details.patch.slice.bottom,
+                                details.patch.slice.left,
+                            ];
+                            yaml_node(&mut v, "slice", u32_vec_yaml(&slice, true));
+                            yaml_node(&mut v, "outset", f32_vec_yaml(&outset, true));
+                            match details.repeat_horizontal {
+                                RepeatMode::Stretch => {
+                                    str_node(&mut v, "repeat-horizontal", "stretch")
+                                }
+                                RepeatMode::Repeat => {
+                                    str_node(&mut v, "repeat-horizontal", "repeat")
+                                }
+                                RepeatMode::Round => str_node(&mut v, "repeat-horizontal", "round"),
+                                RepeatMode::Space => str_node(&mut v, "repeat-horizontal", "space"),
+                            };
+                            match details.repeat_vertical {
+                                RepeatMode::Stretch => {
+                                    str_node(&mut v, "repeat-vertical", "stretch")
+                                }
+                                RepeatMode::Repeat => str_node(&mut v, "repeat-vertical", "repeat"),
+                                RepeatMode::Round => str_node(&mut v, "repeat-vertical", "round"),
+                                RepeatMode::Space => str_node(&mut v, "repeat-vertical", "space"),
+                            };
+                        }
+                        BorderDetails::Gradient(ref details) => {
+                            let widths: Vec<f32> = vec![
+                                item.widths.top,
+                                item.widths.right,
+                                item.widths.bottom,
+                                item.widths.left,
+                            ];
+                            let outset: Vec<f32> = vec![
+                                details.outset.top,
+                                details.outset.right,
+                                details.outset.bottom,
+                                details.outset.left,
+                            ];
+                            yaml_node(&mut v, "width", f32_vec_yaml(&widths, true));
+                            str_node(&mut v, "border-type", "gradient");
+                            point_node(&mut v, "start", &details.gradient.start_point);
+                            point_node(&mut v, "end", &details.gradient.end_point);
+                            let mut stops = vec![];
+                            for stop in display_list.get(base.gradient_stops()) {
+                                stops.push(Yaml::Real(stop.offset.to_string()));
+                                stops.push(Yaml::String(color_to_string(stop.color)));
+                            }
+                            yaml_node(&mut v, "stops", Yaml::Array(stops));
+                            bool_node(
+                                &mut v,
+                                "repeat",
+                                details.gradient.extend_mode == ExtendMode::Repeat,
+                            );
+                            yaml_node(&mut v, "outset", f32_vec_yaml(&outset, true));
+                        }
+                        BorderDetails::RadialGradient(ref details) => {
+                            let widths: Vec<f32> = vec![
+                                item.widths.top,
+                                item.widths.right,
+                                item.widths.bottom,
+                                item.widths.left,
+                            ];
+                            let outset: Vec<f32> = vec![
+                                details.outset.top,
+                                details.outset.right,
+                                details.outset.bottom,
+                                details.outset.left,
+                            ];
+                            yaml_node(&mut v, "width", f32_vec_yaml(&widths, true));
+                            str_node(&mut v, "border-type", "radial-gradient");
+                            point_node(&mut v, "start-center", &details.gradient.start_center);
+                            f32_node(&mut v, "start-radius", details.gradient.start_radius);
+                            point_node(&mut v, "end-center", &details.gradient.end_center);
+                            f32_node(&mut v, "end-radius", details.gradient.end_radius);
+                            f32_node(&mut v, "ratio-xy", details.gradient.ratio_xy);
+                            let mut stops = vec![];
+                            for stop in display_list.get(base.gradient_stops()) {
+                                stops.push(Yaml::Real(stop.offset.to_string()));
+                                stops.push(Yaml::String(color_to_string(stop.color)));
+                            }
+                            yaml_node(&mut v, "stops", Yaml::Array(stops));
+                            bool_node(
+                                &mut v,
+                                "repeat",
+                                details.gradient.extend_mode == ExtendMode::Repeat,
+                            );
+                            yaml_node(&mut v, "outset", f32_vec_yaml(&outset, true));
+                        }
+                    }
+                }
+                BoxShadow(item) => {
+                    str_node(&mut v, "type", "box-shadow");
+                    rect_node(&mut v, "box-bounds", &item.box_bounds);
+                    vector_node(&mut v, "offset", &item.offset);
+                    color_node(&mut v, "color", item.color);
+                    f32_node(&mut v, "blur-radius", item.blur_radius);
+                    f32_node(&mut v, "spread-radius", item.spread_radius);
+                    if let Some(radius_node) = maybe_radius_yaml(&item.border_radius) {
+                        yaml_node(&mut v, "border-radius", radius_node);
+                    }
+                    let clip_mode = match item.clip_mode {
+                        BoxShadowClipMode::Outset => "outset",
+                        BoxShadowClipMode::Inset => "inset",
+                    };
+                    str_node(&mut v, "clip-mode", clip_mode);
+                }
+                Gradient(item) => {
+                    str_node(&mut v, "type", "gradient");
+                    point_node(&mut v, "start", &item.gradient.start_point);
+                    point_node(&mut v, "end", &item.gradient.end_point);
+                    size_node(&mut v, "tile-size", &item.tile_size);
+                    size_node(&mut v, "tile-spacing", &item.tile_spacing);
+                    let mut stops = vec![];
+                    for stop in display_list.get(base.gradient_stops()) {
+                        stops.push(Yaml::Real(stop.offset.to_string()));
+                        stops.push(Yaml::String(color_to_string(stop.color)));
+                    }
+                    yaml_node(&mut v, "stops", Yaml::Array(stops));
+                    bool_node(
+                        &mut v,
+                        "repeat",
+                        item.gradient.extend_mode == ExtendMode::Repeat,
+                    );
+                }
+                RadialGradient(item) => {
+                    str_node(&mut v, "type", "radial-gradient");
+                    point_node(&mut v, "start-center", &item.gradient.start_center);
+                    f32_node(&mut v, "start-radius", item.gradient.start_radius);
+                    point_node(&mut v, "end-center", &item.gradient.end_center);
+                    f32_node(&mut v, "end-radius", item.gradient.end_radius);
+                    f32_node(&mut v, "ratio-xy", item.gradient.ratio_xy);
+                    size_node(&mut v, "tile-size", &item.tile_size);
+                    size_node(&mut v, "tile-spacing", &item.tile_spacing);
+                    let mut stops = vec![];
+                    for stop in display_list.get(base.gradient_stops()) {
+                        stops.push(Yaml::Real(stop.offset.to_string()));
+                        stops.push(Yaml::String(color_to_string(stop.color)));
+                    }
+                    yaml_node(&mut v, "stops", Yaml::Array(stops));
+                    bool_node(
+                        &mut v,
+                        "repeat",
+                        item.gradient.extend_mode == ExtendMode::Repeat,
+                    );
+                }
+                Iframe(item) => {
+                    str_node(&mut v, "type", "iframe");
+                    u32_vec_node(&mut v, "id", &[item.pipeline_id.0, item.pipeline_id.1]);
+                }
+                PushStackingContext(item) => {
+                    str_node(&mut v, "type", "stacking-context");
+                    let filters = display_list.get(base.filters());
+                    write_sc(&mut v, &item.stacking_context, &scene.properties, filters);
+
+                    let mut sub_iter = base.sub_iter();
+                    self.write_display_list(&mut v, display_list, scene, &mut sub_iter, clip_id_mapper);
+                    continue_traversal = Some(sub_iter);
+                }
+                Clip(item) => {
+                    str_node(&mut v, "type", "clip");
+                    usize_node(&mut v, "id", clip_id_mapper.add_id(item.id));
+                    size_node(&mut v, "content-size", &base.rect().size);
+
+                    let (complex_clips, complex_clip_count) = base.complex_clip();
+                    if let Some(complex) = self.make_complex_clips_node(
+                        complex_clip_count,
+                        complex_clips,
+                        display_list,
+                    ) {
+                        yaml_node(&mut v, "complex", complex);
+                    }
+
+                    if let Some(mask_yaml) = self.make_clip_mask_image_node(&item.image_mask) {
+                        yaml_node(&mut v, "image-mask", mask_yaml);
+                    }
+                }
+                ClipChain(item) => {
+                    str_node(&mut v, "type", "clip-chain");
+
+                    let id = ClipId::ClipChain(item.id);
+                    u32_node(&mut v, "id", clip_id_mapper.map_id(&id) as u32);
+
+                    let clip_ids: Vec<u32> = display_list.get(base.clip_chain_items()).map(|clip_id| {
+                        clip_id_mapper.map_id(&clip_id) as u32
+                    }).collect();
+                    u32_vec_node(&mut v, "clips", &clip_ids);
+
+                    if let Some(parent) = item.parent {
+                        let parent = ClipId::ClipChain(parent);
+                        u32_node(&mut v, "parent", clip_id_mapper.map_id(&parent) as u32);
+                    }
+                }
+                ScrollFrame(item) => {
+                    str_node(&mut v, "type", "scroll-frame");
+                    usize_node(&mut v, "id", clip_id_mapper.add_id(item.id));
+                    size_node(&mut v, "content-size", &base.rect().size);
+                    rect_node(&mut v, "bounds", &base.local_clip().clip_rect());
+
+                    let (complex_clips, complex_clip_count) = base.complex_clip();
+                    if let Some(complex) = self.make_complex_clips_node(
+                        complex_clip_count,
+                        complex_clips,
+                        display_list,
+                    ) {
+                        yaml_node(&mut v, "complex", complex);
+                    }
+
+                    if let Some(mask_yaml) = self.make_clip_mask_image_node(&item.image_mask) {
+                        yaml_node(&mut v, "image-mask", mask_yaml);
+                    }
+                }
+                StickyFrame(item) => {
+                    str_node(&mut v, "type", "sticky-frame");
+                    usize_node(&mut v, "id", clip_id_mapper.add_id(item.id));
+                    rect_node(&mut v, "bounds", &base.local_clip().clip_rect());
+
+                    if let Some(margin) = item.margins.top {
+                        f32_node(&mut v, "margin-top", margin);
+                    }
+                    if let Some(margin) = item.margins.bottom {
+                        f32_node(&mut v, "margin-bottom", margin);
+                    }
+                    if let Some(margin) = item.margins.left {
+                        f32_node(&mut v, "margin-left", margin);
+                    }
+                    if let Some(margin) = item.margins.right {
+                        f32_node(&mut v, "margin-right", margin);
+                    }
+
+                    let horizontal = vec![
+                        Yaml::Real(item.horizontal_offset_bounds.min.to_string()),
+                        Yaml::Real(item.horizontal_offset_bounds.max.to_string()),
+                    ];
+                    let vertical = vec![
+                        Yaml::Real(item.vertical_offset_bounds.min.to_string()),
+                        Yaml::Real(item.vertical_offset_bounds.max.to_string()),
+                    ];
+
+                    yaml_node(&mut v, "horizontal-offset-bounds", Yaml::Array(horizontal));
+                    yaml_node(&mut v, "vertical-offset-bounds", Yaml::Array(vertical));
+
+                    let applied = vec![
+                        Yaml::Real(item.previously_applied_offset.x.to_string()),
+                        Yaml::Real(item.previously_applied_offset.y.to_string()),
+                    ];
+                    yaml_node(&mut v, "previously-applied-offset", Yaml::Array(applied));
+                }
+
+                PopStackingContext => return,
+                SetGradientStops => panic!("dummy item yielded?"),
+                PushShadow(shadow) => {
+                    str_node(&mut v, "type", "shadow");
+                    vector_node(&mut v, "offset", &shadow.offset);
+                    color_node(&mut v, "color", shadow.color);
+                    f32_node(&mut v, "blur-radius", shadow.blur_radius);
+                }
+                PopAllShadows => {
+                    str_node(&mut v, "type", "pop-all-shadows");
+                }
+            }
+            if !v.is_empty() {
+                list.push(Yaml::Hash(v));
+            }
+        }
+    }
+
+    fn write_display_list(
+        &mut self,
+        parent: &mut Table,
+        display_list: &BuiltDisplayList,
+        scene: &Scene,
+        list_iterator: &mut BuiltDisplayListIter,
+        clip_id_mapper: &mut ClipIdMapper,
+    ) {
+        let mut list = vec![];
+        self.write_display_list_items(&mut list, display_list, scene, list_iterator, clip_id_mapper);
+        parent.insert(Yaml::String("items".to_owned()), Yaml::Array(list));
+    }
+}
+
+impl webrender::ApiRecordingReceiver for YamlFrameWriterReceiver {
+    fn write_msg(&mut self, _: u32, msg: &ApiMsg) {
+        match *msg {
+            ApiMsg::UpdateResources(ref updates) => {
+                self.frame_writer.update_resources(updates);
+            }
+            ApiMsg::UpdateDocument(_, ref doc_msgs) => {
+                for doc_msg in doc_msgs {
+                    match *doc_msg {
+                        DocumentMsg::UpdateResources(ref resources) => {
+                            self.frame_writer.update_resources(resources);
+                        }
+                        DocumentMsg::SetDisplayList {
+                            ref epoch,
+                            ref pipeline_id,
+                            ref background,
+                            ref viewport_size,
+                            ref list_descriptor,
+                            ..
+                        } => {
+                            self.frame_writer.begin_write_display_list(
+                                &mut self.scene,
+                                epoch,
+                                pipeline_id,
+                                background,
+                                viewport_size,
+                                list_descriptor,
+                            );
+                        }
+                        DocumentMsg::SetRootPipeline(ref pipeline_id) => {
+                            self.scene.set_root_pipeline_id(pipeline_id.clone());
+                        }
+                        DocumentMsg::RemovePipeline(ref pipeline_id) => {
+                            self.scene.remove_pipeline(pipeline_id);
+                        }
+                        DocumentMsg::UpdateDynamicProperties(ref properties) => {
+                            self.scene.properties.set_properties(properties);
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+
+    fn write_payload(&mut self, _frame: u32, data: &[u8]) {
+        if self.frame_writer.dl_descriptor.is_some() {
+            self.frame_writer
+                .finish_write_display_list(&mut self.scene, data);
+        }
+    }
+}
+
+/// This structure allows mapping both `Clip` and `ClipExternalId`
+/// `ClipIds` onto one set of numeric ids. This prevents ids
+/// from clashing in the yaml output.
+struct ClipIdMapper {
+    hash_map: HashMap<ClipId, usize>,
+    current_clip_id: usize,
+}
+
+impl ClipIdMapper {
+    fn new() -> ClipIdMapper {
+        ClipIdMapper {
+            hash_map: HashMap::new(),
+            current_clip_id: 1,
+        }
+    }
+
+    fn add_id(&mut self, id: ClipId) -> usize {
+        self.hash_map.insert(id, self.current_clip_id);
+        self.current_clip_id += 1;
+        self.current_clip_id - 1
+    }
+
+    fn map_id(&self, id: &ClipId) -> usize {
+        if id.is_root_scroll_node() {
+            return 0;
+        }
+        *self.hash_map.get(id).unwrap()
+    }
+
+    fn map_info(&self, info: &ClipAndScrollInfo) -> (i64, Option<i64>) {
+        (
+            self.map_id(&info.scroll_node_id) as i64,
+            info.clip_node_id.map(|ref id| self.map_id(id) as i64),
+        )
+    }
+}
diff --git a/gfx/wrench/src/yaml_helper.rs b/gfx/wrench/src/yaml_helper.rs
new file mode 100644
index 000000000000..60ce06d09969
--- /dev/null
+++ b/gfx/wrench/src/yaml_helper.rs
@@ -0,0 +1,577 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use app_units::Au;
+use euclid::{Angle, TypedSize2D};
+use parse_function::parse_function;
+use std::f32;
+use std::str::FromStr;
+use webrender::api::*;
+use yaml_rust::{Yaml, YamlLoader};
+
+pub trait YamlHelper {
+    fn as_f32(&self) -> Option<f32>;
+    fn as_force_f32(&self) -> Option<f32>;
+    fn as_vec_f32(&self) -> Option<Vec<f32>>;
+    fn as_vec_u32(&self) -> Option<Vec<u32>>;
+    fn as_vec_u64(&self) -> Option<Vec<u64>>;
+    fn as_pipeline_id(&self) -> Option<PipelineId>;
+    fn as_rect(&self) -> Option<LayoutRect>;
+    fn as_size(&self) -> Option<LayoutSize>;
+    fn as_point(&self) -> Option<LayoutPoint>;
+    fn as_vector(&self) -> Option<LayoutVector2D>;
+    fn as_matrix4d(&self) -> Option<LayoutTransform>;
+    fn as_transform(&self, transform_origin: &LayoutPoint) -> Option<LayoutTransform>;
+    fn as_colorf(&self) -> Option<ColorF>;
+    fn as_vec_colorf(&self) -> Option<Vec<ColorF>>;
+    fn as_px_to_au(&self) -> Option<Au>;
+    fn as_pt_to_au(&self) -> Option<Au>;
+    fn as_vec_string(&self) -> Option<Vec<String>>;
+    fn as_border_radius_component(&self) -> LayoutSize;
+    fn as_border_radius(&self) -> Option<BorderRadius>;
+    fn as_transform_style(&self) -> Option<TransformStyle>;
+    fn as_clip_mode(&self) -> Option<ClipMode>;
+    fn as_mix_blend_mode(&self) -> Option<MixBlendMode>;
+    fn as_scroll_policy(&self) -> Option<ScrollPolicy>;
+    fn as_filter_op(&self) -> Option<FilterOp>;
+    fn as_vec_filter_op(&self) -> Option<Vec<FilterOp>>;
+}
+
+fn string_to_color(color: &str) -> Option<ColorF> {
+    match color {
+        "red" => Some(ColorF::new(1.0, 0.0, 0.0, 1.0)),
+        "green" => Some(ColorF::new(0.0, 1.0, 0.0, 1.0)),
+        "blue" => Some(ColorF::new(0.0, 0.0, 1.0, 1.0)),
+        "white" => Some(ColorF::new(1.0, 1.0, 1.0, 1.0)),
+        "black" => Some(ColorF::new(0.0, 0.0, 0.0, 1.0)),
+        "yellow" => Some(ColorF::new(1.0, 1.0, 0.0, 1.0)),
+        s => {
+            let items: Vec<f32> = s.split_whitespace()
+                .map(|s| f32::from_str(s).unwrap())
+                .collect();
+            if items.len() == 3 {
+                Some(ColorF::new(
+                    items[0] / 255.0,
+                    items[1] / 255.0,
+                    items[2] / 255.0,
+                    1.0,
+                ))
+            } else if items.len() == 4 {
+                Some(ColorF::new(
+                    items[0] / 255.0,
+                    items[1] / 255.0,
+                    items[2] / 255.0,
+                    items[3],
+                ))
+            } else {
+                None
+            }
+        }
+    }
+}
+
+pub trait StringEnum: Sized {
+    fn from_str(&str) -> Option<Self>;
+    fn as_str(&self) -> &'static str;
+}
+
+macro_rules! define_string_enum {
+    ($T:ident, [ $( $y:ident = $x:expr ),* ]) => {
+        impl StringEnum for $T {
+            fn from_str(text: &str) -> Option<$T> {
+                match text {
+                $( $x => Some($T::$y), )*
+                    _ => {
+                        println!("Unrecognized {} value '{}'", stringify!($T), text);
+                        None
+                    }
+                }
+            }
+            fn as_str(&self) -> &'static str {
+                match *self {
+                $( $T::$y => $x, )*
+                }
+            }
+        }
+    }
+}
+
+define_string_enum!(TransformStyle, [Flat = "flat", Preserve3D = "preserve-3d"]);
+
+define_string_enum!(
+    MixBlendMode,
+    [
+        Normal = "normal",
+        Multiply = "multiply",
+        Screen = "screen",
+        Overlay = "overlay",
+        Darken = "darken",
+        Lighten = "lighten",
+        ColorDodge = "color-dodge",
+        ColorBurn = "color-burn",
+        HardLight = "hard-light",
+        SoftLight = "soft-light",
+        Difference = "difference",
+        Exclusion = "exclusion",
+        Hue = "hue",
+        Saturation = "saturation",
+        Color = "color",
+        Luminosity = "luminosity"
+    ]
+);
+
+define_string_enum!(ScrollPolicy, [Scrollable = "scrollable", Fixed = "fixed"]);
+
+define_string_enum!(
+    LineOrientation,
+    [Horizontal = "horizontal", Vertical = "vertical"]
+);
+
+define_string_enum!(
+    LineStyle,
+    [
+        Solid = "solid",
+        Dotted = "dotted",
+        Dashed = "dashed",
+        Wavy = "wavy"
+    ]
+);
+
+define_string_enum!(ClipMode, [Clip = "clip", ClipOut = "clip-out"]);
+
+// Rotate around `axis` by `degrees` angle
+fn make_rotation(
+    origin: &LayoutPoint,
+    degrees: f32,
+    axis_x: f32,
+    axis_y: f32,
+    axis_z: f32,
+) -> LayoutTransform {
+    let pre_transform = LayoutTransform::create_translation(origin.x, origin.y, 0.0);
+    let post_transform = LayoutTransform::create_translation(-origin.x, -origin.y, -0.0);
+
+    let theta = 2.0f32 * f32::consts::PI - degrees.to_radians();
+    let transform =
+        LayoutTransform::identity().pre_rotate(axis_x, axis_y, axis_z, Angle::radians(theta));
+
+    pre_transform.pre_mul(&transform).pre_mul(&post_transform)
+}
+
+// Create a skew matrix, specified in degrees.
+fn make_skew(
+    skew_x: f32,
+    skew_y: f32,
+) -> LayoutTransform {
+    let alpha = Angle::radians(skew_x.to_radians());
+    let beta = Angle::radians(skew_y.to_radians());
+    LayoutTransform::create_skew(alpha, beta)
+}
+
+impl YamlHelper for Yaml {
+    fn as_f32(&self) -> Option<f32> {
+        match *self {
+            Yaml::Integer(iv) => Some(iv as f32),
+            Yaml::Real(ref sv) => f32::from_str(sv.as_str()).ok(),
+            _ => None,
+        }
+    }
+
+    fn as_force_f32(&self) -> Option<f32> {
+        match *self {
+            Yaml::Integer(iv) => Some(iv as f32),
+            Yaml::String(ref sv) | Yaml::Real(ref sv) => f32::from_str(sv.as_str()).ok(),
+            _ => None,
+        }
+    }
+
+    fn as_vec_f32(&self) -> Option<Vec<f32>> {
+        match *self {
+            Yaml::String(ref s) | Yaml::Real(ref s) => s.split_whitespace()
+                .map(|v| f32::from_str(v))
+                .collect::<Result<Vec<_>, _>>()
+                .ok(),
+            Yaml::Array(ref v) => v.iter()
+                .map(|v| match *v {
+                    Yaml::Integer(k) => Ok(k as f32),
+                    Yaml::String(ref k) | Yaml::Real(ref k) => f32::from_str(k).map_err(|_| false),
+                    _ => Err(false),
+                })
+                .collect::<Result<Vec<_>, _>>()
+                .ok(),
+            Yaml::Integer(k) => Some(vec![k as f32]),
+            _ => None,
+        }
+    }
+
+    fn as_vec_u32(&self) -> Option<Vec<u32>> {
+        if let Some(v) = self.as_vec() {
+            Some(v.iter().map(|v| v.as_i64().unwrap() as u32).collect())
+        } else {
+            None
+        }
+    }
+
+    fn as_vec_u64(&self) -> Option<Vec<u64>> {
+        if let Some(v) = self.as_vec() {
+            Some(v.iter().map(|v| v.as_i64().unwrap() as u64).collect())
+        } else {
+            None
+        }
+    }
+
+    fn as_pipeline_id(&self) -> Option<PipelineId> {
+        if let Some(v) = self.as_vec() {
+            let a = v.get(0).and_then(|v| v.as_i64()).map(|v| v as u32);
+            let b = v.get(1).and_then(|v| v.as_i64()).map(|v| v as u32);
+            match (a, b) {
+                (Some(a), Some(b)) if v.len() == 2 => Some(PipelineId(a, b)),
+                _ => None,
+            }
+        } else {
+            None
+        }
+    }
+
+    fn as_px_to_au(&self) -> Option<Au> {
+        match self.as_force_f32() {
+            Some(fv) => Some(Au::from_f32_px(fv)),
+            None => None,
+        }
+    }
+
+    fn as_pt_to_au(&self) -> Option<Au> {
+        match self.as_force_f32() {
+            Some(fv) => Some(Au::from_f32_px(fv * 16. / 12.)),
+            None => None,
+        }
+    }
+
+    fn as_rect(&self) -> Option<LayoutRect> {
+        if self.is_badvalue() {
+            return None;
+        }
+
+        if let Some(nums) = self.as_vec_f32() {
+            if nums.len() == 4 {
+                return Some(LayoutRect::new(
+                    LayoutPoint::new(nums[0], nums[1]),
+                    LayoutSize::new(nums[2], nums[3]),
+                ));
+            }
+        }
+
+        None
+    }
+
+    fn as_size(&self) -> Option<LayoutSize> {
+        if self.is_badvalue() {
+            return None;
+        }
+
+        if let Some(nums) = self.as_vec_f32() {
+            if nums.len() == 2 {
+                return Some(LayoutSize::new(nums[0], nums[1]));
+            }
+        }
+
+        None
+    }
+
+    fn as_point(&self) -> Option<LayoutPoint> {
+        if self.is_badvalue() {
+            return None;
+        }
+
+        if let Some(nums) = self.as_vec_f32() {
+            if nums.len() == 2 {
+                return Some(LayoutPoint::new(nums[0], nums[1]));
+            }
+        }
+
+        None
+    }
+
+    fn as_vector(&self) -> Option<LayoutVector2D> {
+        self.as_point().map(|p| p.to_vector())
+    }
+
+    fn as_matrix4d(&self) -> Option<LayoutTransform> {
+        if let Some(nums) = self.as_vec_f32() {
+            assert_eq!(nums.len(), 16, "expected 16 floats, got '{:?}'", self);
+            Some(LayoutTransform::row_major(
+                nums[0],
+                nums[1],
+                nums[2],
+                nums[3],
+                nums[4],
+                nums[5],
+                nums[6],
+                nums[7],
+                nums[8],
+                nums[9],
+                nums[10],
+                nums[11],
+                nums[12],
+                nums[13],
+                nums[14],
+                nums[15],
+            ))
+        } else {
+            None
+        }
+    }
+
+    fn as_transform(&self, transform_origin: &LayoutPoint) -> Option<LayoutTransform> {
+        if let Some(transform) = self.as_matrix4d() {
+            return Some(transform);
+        }
+
+        match *self {
+            Yaml::String(ref string) => {
+                let mut slice = string.as_str();
+                let mut transform = LayoutTransform::identity();
+                while !slice.is_empty() {
+                    let (function, ref args, reminder) = parse_function(slice);
+                    slice = reminder;
+                    let mx = match function {
+                        "translate" if args.len() >= 2 => {
+                            let z = args.get(2).and_then(|a| a.parse().ok()).unwrap_or(0.);
+                            LayoutTransform::create_translation(
+                                args[0].parse().unwrap(),
+                                args[1].parse().unwrap(),
+                                z,
+                            )
+                        }
+                        "rotate" | "rotate-z" if args.len() == 1 => {
+                            make_rotation(transform_origin, args[0].parse().unwrap(), 0.0, 0.0, 1.0)
+                        }
+                        "rotate-x" if args.len() == 1 => {
+                            make_rotation(transform_origin, args[0].parse().unwrap(), 1.0, 0.0, 0.0)
+                        }
+                        "rotate-y" if args.len() == 1 => {
+                            make_rotation(transform_origin, args[0].parse().unwrap(), 0.0, 1.0, 0.0)
+                        }
+                        "scale" if args.len() >= 1 => {
+                            let x = args[0].parse().unwrap();
+                            // Default to uniform X/Y scale if Y unspecified.
+                            let y = args.get(1).and_then(|a| a.parse().ok()).unwrap_or(x);
+                            // Default to no Z scale if unspecified.
+                            let z = args.get(2).and_then(|a| a.parse().ok()).unwrap_or(1.0);
+                            LayoutTransform::create_scale(x, y, z)
+                        }
+                        "scale-x" if args.len() == 1 => {
+                            LayoutTransform::create_scale(args[0].parse().unwrap(), 1.0, 1.0)
+                        }
+                        "scale-y" if args.len() == 1 => {
+                            LayoutTransform::create_scale(1.0, args[0].parse().unwrap(), 1.0)
+                        }
+                        "scale-z" if args.len() == 1 => {
+                            LayoutTransform::create_scale(1.0, 1.0, args[0].parse().unwrap())
+                        }
+                        "skew" if args.len() >= 1 => {
+                            // Default to no Y skew if unspecified.
+                            let skew_y = args.get(1).and_then(|a| a.parse().ok()).unwrap_or(0.0);
+                            make_skew(args[0].parse().unwrap(), skew_y)
+                        }
+                        "skew-x" if args.len() == 1 => {
+                            make_skew(args[0].parse().unwrap(), 0.0)
+                        }
+                        "skew-y" if args.len() == 1 => {
+                            make_skew(0.0, args[0].parse().unwrap())
+                        }
+                        "perspective" if args.len() == 1 => {
+                            LayoutTransform::create_perspective(args[0].parse().unwrap())
+                        }
+                        _ => {
+                            println!("unknown function {}", function);
+                            break;
+                        }
+                    };
+                    transform = transform.post_mul(&mx);
+                }
+                Some(transform)
+            }
+            Yaml::Array(ref array) => {
+                let transform = array.iter().fold(
+                    LayoutTransform::identity(),
+                    |u, yaml| match yaml.as_transform(transform_origin) {
+                        Some(ref transform) => u.pre_mul(transform),
+                        None => u,
+                    },
+                );
+                Some(transform)
+            }
+            Yaml::BadValue => None,
+            _ => {
+                println!("unknown transform {:?}", self);
+                None
+            }
+        }
+    }
+
+    fn as_colorf(&self) -> Option<ColorF> {
+        if let Some(mut nums) = self.as_vec_f32() {
+            assert!(
+                nums.len() == 3 || nums.len() == 4,
+                "color expected a color name, or 3-4 floats; got '{:?}'",
+                self
+            );
+
+            if nums.len() == 3 {
+                nums.push(1.0);
+            }
+            return Some(ColorF::new(
+                nums[0] / 255.0,
+                nums[1] / 255.0,
+                nums[2] / 255.0,
+                nums[3],
+            ));
+        }
+
+        if let Some(s) = self.as_str() {
+            string_to_color(s)
+        } else {
+            None
+        }
+    }
+
+    fn as_vec_colorf(&self) -> Option<Vec<ColorF>> {
+        if let Some(v) = self.as_vec() {
+            Some(v.iter().map(|v| v.as_colorf().unwrap()).collect())
+        } else if let Some(color) = self.as_colorf() {
+            Some(vec![color])
+        } else {
+            None
+        }
+    }
+
+    fn as_vec_string(&self) -> Option<Vec<String>> {
+        if let Some(v) = self.as_vec() {
+            Some(v.iter().map(|v| v.as_str().unwrap().to_owned()).collect())
+        } else if let Some(s) = self.as_str() {
+            Some(vec![s.to_owned()])
+        } else {
+            None
+        }
+    }
+
+    fn as_border_radius_component(&self) -> LayoutSize {
+        if let Yaml::Integer(integer) = *self {
+            return LayoutSize::new(integer as f32, integer as f32);
+        }
+        self.as_size().unwrap_or(TypedSize2D::zero())
+    }
+
+    fn as_border_radius(&self) -> Option<BorderRadius> {
+        if let Some(size) = self.as_size() {
+            return Some(BorderRadius::uniform_size(size));
+        }
+
+        match *self {
+            Yaml::BadValue => None,
+            Yaml::String(ref s) | Yaml::Real(ref s) => {
+                let fv = f32::from_str(s).unwrap();
+                Some(BorderRadius::uniform(fv))
+            }
+            Yaml::Integer(v) => Some(BorderRadius::uniform(v as f32)),
+            Yaml::Array(ref array) if array.len() == 4 => {
+                let top_left = array[0].as_border_radius_component();
+                let top_right = array[1].as_border_radius_component();
+                let bottom_left = array[2].as_border_radius_component();
+                let bottom_right = array[3].as_border_radius_component();
+                Some(BorderRadius {
+                    top_left,
+                    top_right,
+                    bottom_left,
+                    bottom_right,
+                })
+            }
+            Yaml::Hash(_) => {
+                let top_left = self["top-left"].as_border_radius_component();
+                let top_right = self["top-right"].as_border_radius_component();
+                let bottom_left = self["bottom-left"].as_border_radius_component();
+                let bottom_right = self["bottom-right"].as_border_radius_component();
+                Some(BorderRadius {
+                    top_left,
+                    top_right,
+                    bottom_left,
+                    bottom_right,
+                })
+            }
+            _ => {
+                panic!("Invalid border radius specified: {:?}", self);
+            }
+        }
+    }
+
+    fn as_transform_style(&self) -> Option<TransformStyle> {
+        self.as_str().and_then(|x| StringEnum::from_str(x))
+    }
+
+    fn as_mix_blend_mode(&self) -> Option<MixBlendMode> {
+        self.as_str().and_then(|x| StringEnum::from_str(x))
+    }
+
+    fn as_scroll_policy(&self) -> Option<ScrollPolicy> {
+        self.as_str().and_then(|x| StringEnum::from_str(x))
+    }
+
+    fn as_clip_mode(&self) -> Option<ClipMode> {
+        self.as_str().and_then(|x| StringEnum::from_str(x))
+    }
+
+    fn as_filter_op(&self) -> Option<FilterOp> {
+        if let Some(s) = self.as_str() {
+            match parse_function(s) {
+                ("blur", ref args, _) if args.len() == 1 => {
+                    Some(FilterOp::Blur(args[0].parse().unwrap()))
+                }
+                ("brightness", ref args, _) if args.len() == 1 => {
+                    Some(FilterOp::Brightness(args[0].parse().unwrap()))
+                }
+                ("contrast", ref args, _) if args.len() == 1 => {
+                    Some(FilterOp::Contrast(args[0].parse().unwrap()))
+                }
+                ("grayscale", ref args, _) if args.len() == 1 => {
+                    Some(FilterOp::Grayscale(args[0].parse().unwrap()))
+                }
+                ("hue-rotate", ref args, _) if args.len() == 1 => {
+                    Some(FilterOp::HueRotate(args[0].parse().unwrap()))
+                }
+                ("invert", ref args, _) if args.len() == 1 => {
+                    Some(FilterOp::Invert(args[0].parse().unwrap()))
+                }
+                ("opacity", ref args, _) if args.len() == 1 => {
+                    let amount: f32 = args[0].parse().unwrap();
+                    Some(FilterOp::Opacity(amount.into(), amount))
+                }
+                ("saturate", ref args, _) if args.len() == 1 => {
+                    Some(FilterOp::Saturate(args[0].parse().unwrap()))
+                }
+                ("sepia", ref args, _) if args.len() == 1 => {
+                    Some(FilterOp::Sepia(args[0].parse().unwrap()))
+                }
+                ("drop-shadow", ref args, _) if args.len() == 3 => {
+                    let str = format!("---\noffset: {}\nblur-radius: {}\ncolor: {}\n", args[0], args[1], args[2]);
+                    let mut yaml_doc = YamlLoader::load_from_str(&str).expect("Failed to parse drop-shadow");
+                    let yaml = yaml_doc.pop().unwrap();
+                    Some(FilterOp::DropShadow(yaml["offset"].as_vector().unwrap(),
+                                              yaml["blur-radius"].as_f32().unwrap(),
+                                              yaml["color"].as_colorf().unwrap()))
+                }
+                (_, _, _) => None,
+            }
+        } else {
+            None
+        }
+    }
+
+    fn as_vec_filter_op(&self) -> Option<Vec<FilterOp>> {
+        if let Some(v) = self.as_vec() {
+            Some(v.iter().map(|x| x.as_filter_op().unwrap()).collect())
+        } else {
+            self.as_filter_op().map(|op| vec![op])
+        }
+    }
+}

From a73a6015a89dc6e57c9718a42d54fa476ed11cfb Mon Sep 17 00:00:00 2001
From: Mark Banner <standard8@mozilla.com>
Date: Fri, 26 Jan 2018 16:23:04 +0000
Subject: [PATCH 17/46] Bug 1433492 - Fix a missed callback to .then conversion
 causing RSS feeds to not show their read/unread status properly after a
 restart. r=mak

MozReview-Commit-ID: 3iG8JbyGJul

--HG--
extra : rebase_source : 2240904ed63bed5db93354bcc9dd255b0c2fc40c
---
 toolkit/components/places/nsLivemarkService.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/toolkit/components/places/nsLivemarkService.js b/toolkit/components/places/nsLivemarkService.js
index 141bc5207e18..0698144c2328 100644
--- a/toolkit/components/places/nsLivemarkService.js
+++ b/toolkit/components/places/nsLivemarkService.js
@@ -601,9 +601,9 @@ Livemark.prototype = {
 
     // Update visited status for each entry.
     for (let child of this._children) {
-      history.hasVisits(child.uri, isVisited => {
+      history.hasVisits(child.uri).then(isVisited => {
         this.updateURIVisitedStatus(child.uri, isVisited);
-      });
+      }).catch(Cu.reportError);
     }
 
     return this._children;

From dd6c928ffd9860be65cb45244b20f5a23df4516f Mon Sep 17 00:00:00 2001
From: Mark Banner <standard8@mozilla.com>
Date: Fri, 26 Jan 2018 16:40:25 +0000
Subject: [PATCH 18/46] Bug 1433492 - Remove remaining instances of
 PlacesUtils.asyncHistory.isURIVisited being called directly. r=mak

MozReview-Commit-ID: B1sqEtJcq4L

--HG--
extra : rebase_source : 62a7c91e06f28712e9ba29a2e3ef73fa7dd9a0e1
---
 browser/base/content/test/general/head.js     | 22 +++++------------
 .../components/places/tests/browser/head.js   | 12 ----------
 .../places/tests/history/test_removeVisits.js | 24 +++++++++----------
 3 files changed, 18 insertions(+), 40 deletions(-)

diff --git a/browser/base/content/test/general/head.js b/browser/base/content/test/general/head.js
index fcbb0c8f3e3c..09a4079906c9 100644
--- a/browser/base/content/test/general/head.js
+++ b/browser/base/content/test/general/head.js
@@ -304,22 +304,12 @@ function promiseTabLoaded(aTab) {
  * @param aShouldBeCleared
  *        True if each visit to the URI should be cleared, false otherwise
  */
-function promiseHistoryClearedState(aURIs, aShouldBeCleared) {
-  return new Promise(resolve => {
-    let callbackCount = 0;
-    let niceStr = aShouldBeCleared ? "no longer" : "still";
-    function callbackDone() {
-      if (++callbackCount == aURIs.length)
-        resolve();
-    }
-    aURIs.forEach(function(aURI) {
-      PlacesUtils.asyncHistory.isURIVisited(aURI, function(uri, isVisited) {
-        is(isVisited, !aShouldBeCleared,
-           "history visit " + uri.spec + " should " + niceStr + " exist");
-        callbackDone();
-      });
-    });
-  });
+async function promiseHistoryClearedState(aURIs, aShouldBeCleared) {
+  for (let uri of aURIs) {
+    let visited = await PlacesUtils.history.hasVisits(uri);
+    Assert.equal(visited, !aShouldBeCleared,
+      `history visit ${uri.spec} should ${aShouldBeCleared ? "no longer" : "still"} exist`);
+  }
 }
 
 var FullZoomHelper = {
diff --git a/browser/components/places/tests/browser/head.js b/browser/components/places/tests/browser/head.js
index 1b8324ea7fd1..d138719fc395 100644
--- a/browser/components/places/tests/browser/head.js
+++ b/browser/components/places/tests/browser/head.js
@@ -142,18 +142,6 @@ function synthesizeClickOnSelectedTreeCell(aTree, aOptions) {
                              aTree.ownerGlobal);
 }
 
-/**
- * Asynchronously check a url is visited.
- *
- * @param aURI The URI.
- * @return {Promise}
- * @resolves When the check has been added successfully.
- * @rejects JavaScript exception.
- */
- function promiseIsURIVisited(aURI) {
-   return PlacesUtils.history.hasVisits(aURI);
- }
-
 /**
  * Makes the specified toolbar visible or invisible and returns a Promise object
  * that is resolved when the toolbar has completed any animations associated
diff --git a/toolkit/components/places/tests/history/test_removeVisits.js b/toolkit/components/places/tests/history/test_removeVisits.js
index ab33c9058763..bd1bb24cb735 100644
--- a/toolkit/components/places/tests/history/test_removeVisits.js
+++ b/toolkit/components/places/tests/history/test_removeVisits.js
@@ -44,8 +44,8 @@ add_task(async function remove_visits_outside_unbookmarked_uri() {
   }
   root.containerOpen = false;
 
-  info("PlacesUtils.history.hasVisits should return true.");
-  Assert.ok(await PlacesUtils.history.hasVisits(TEST_URI));
+  Assert.ok(await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should exist");
 
   await PlacesTestUtils.promiseAsyncUpdates();
   info("Frecency should be positive.");
@@ -95,8 +95,8 @@ add_task(async function remove_visits_outside_bookmarked_uri() {
   }
   root.containerOpen = false;
 
-  info("asyncHistory.isURIVisited should return true.");
-  Assert.ok(await PlacesUtils.history.hasVisits(TEST_URI));
+  Assert.ok(await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should exist");
   await PlacesTestUtils.promiseAsyncUpdates();
 
   info("Frecency should be positive.");
@@ -140,8 +140,8 @@ add_task(async function remove_visits_unbookmarked_uri() {
   }
   root.containerOpen = false;
 
-  info("asyncHistory.isURIVisited should return true.");
-  Assert.ok(await PlacesUtils.history.hasVisits(TEST_URI));
+  Assert.ok(await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should exist");
   await PlacesTestUtils.promiseAsyncUpdates();
 
   info("Frecency should be positive.");
@@ -191,8 +191,8 @@ add_task(async function remove_visits_bookmarked_uri() {
   }
   root.containerOpen = false;
 
-  info("asyncHistory.isURIVisited should return true.");
-  Assert.ok(await PlacesUtils.history.hasVisits(TEST_URI));
+  Assert.ok(await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should exist");
   await PlacesTestUtils.promiseAsyncUpdates();
 
   info("Frecency should be positive.");
@@ -232,8 +232,8 @@ add_task(async function remove_all_visits_unbookmarked_uri() {
   Assert.equal(root.childCount, 0);
   root.containerOpen = false;
 
-  info("asyncHistory.isURIVisited should return false.");
-  Assert.equal(false, await PlacesUtils.history.hasVisits(TEST_URI));
+  Assert.equal(false, await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should not exist");
 
   await cleanup();
 });
@@ -276,8 +276,8 @@ add_task(async function remove_all_visits_bookmarked_uri() {
   Assert.equal(root.childCount, 0);
   root.containerOpen = false;
 
-  info("asyncHistory.isURIVisited should return false.");
-  Assert.equal(false, await PlacesUtils.history.hasVisits(TEST_URI));
+  Assert.equal(false, await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should not exist");
 
   info("URI should be bookmarked");
   Assert.ok(await PlacesUtils.bookmarks.fetch({url: TEST_URI}));

From fd6008421c02f29585ffda1c2ccb9e484ba0926a Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Thu, 25 Jan 2018 15:13:20 +0900
Subject: [PATCH 19/46] Bug 1433033 - Remove a few packages from the
 toolchain-build image. r=dustin

python-dev was required to build mercurial, but the need for that was
removed in bug 1429669.
The others were required for mingw32 toolchains, but they are using a
different docker image and will switch to another different docker image.

--HG--
extra : rebase_source : b65c586a325f220c565e79afb3d3c9acc9f922bc
---
 taskcluster/docker/toolchain-build/Dockerfile | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/taskcluster/docker/toolchain-build/Dockerfile b/taskcluster/docker/toolchain-build/Dockerfile
index de510a87c169..1a4466dcf955 100644
--- a/taskcluster/docker/toolchain-build/Dockerfile
+++ b/taskcluster/docker/toolchain-build/Dockerfile
@@ -43,8 +43,6 @@ RUN for task in $DOCKER_IMAGE_PACKAGES; do \
       echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/" >> /etc/apt/sources.list; \
     done
 
-RUN dpkg --add-architecture i386
-
 RUN apt-get -o Acquire::Check-Valid-Until=false update -q && \
     apt-get install -yyq --no-install-recommends \
       autoconf \
@@ -61,17 +59,14 @@ RUN apt-get -o Acquire::Check-Valid-Until=false update -q && \
       libbz2-dev \
       libssl-dev \
       libtool \
-      libx11-dev:i386 \
       make \
       mercurial \
       ninja-build \
       p7zip-full \
       procps \
-      python-dev \
       python-pip \
       python-setuptools \
       python-virtualenv \
-      scons \
       subversion \
       tar \
       unzip \

From ce90eb437bc0295f5779fc91714f5a73e1bec5c9 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Thu, 25 Jan 2018 15:24:05 +0900
Subject: [PATCH 20/46] Bug 1433033 - Set Acquire::Check-Valid-Until in apt
 config file rather than pass it on the command line. r=dustin

Also set it in the android-build image, which is Debian-snapshot-based too.

--HG--
extra : rebase_source : a64a0d6833c98025bfe287e197a9ef255b12a9bc
---
 taskcluster/docker/android-build/Dockerfile   | 2 ++
 taskcluster/docker/debian7-build/Dockerfile   | 7 ++++---
 taskcluster/docker/toolchain-build/Dockerfile | 7 ++++---
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/taskcluster/docker/android-build/Dockerfile b/taskcluster/docker/android-build/Dockerfile
index 8229460fa81b..b39ed9b9f65a 100644
--- a/taskcluster/docker/android-build/Dockerfile
+++ b/taskcluster/docker/android-build/Dockerfile
@@ -32,6 +32,8 @@ CMD ["/bin/bash", "--login"]
 # Set apt sources list to a snapshot.
 COPY sources.list /etc/apt/
 
+RUN echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99check-valid-until
+
 # rsync is required for l10n single locale repacks.  less, screen, and
 # vim, help debugging interactive tasks in Task Cluster.
 # git and openssh-client are used to upload GeckoView javadoc to Github.
diff --git a/taskcluster/docker/debian7-build/Dockerfile b/taskcluster/docker/debian7-build/Dockerfile
index 925d71da65cb..5aa8ba75feb3 100644
--- a/taskcluster/docker/debian7-build/Dockerfile
+++ b/taskcluster/docker/debian7-build/Dockerfile
@@ -31,9 +31,10 @@ CMD ["/bin/bash", "--login"]
 # Set apt sources list to a snapshot.
 RUN for s in debian_wheezy debian_wheezy-updates debian_wheezy-backports debian-security_wheezy/updates; do \
       echo "deb http://snapshot.debian.org/archive/${s%_*}/20171210T214726Z/ ${s#*_} main"; \
-    done > /etc/apt/sources.list
+    done > /etc/apt/sources.list && \
+    echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99check-valid-until
 
-RUN apt-get -o Acquire::Check-Valid-Until=false update -q && \
+RUN apt-get update -q && \
     apt-get install -yyq --no-install-recommends \
       apt-transport-https \
       ca-certificates
@@ -49,7 +50,7 @@ RUN dpkg --add-architecture $ARCH
 # Ideally, we wouldn't need gcc-multilib and the extra linux-libc-dev,
 # but the latter is required to make the former installable, and the former
 # because of bug 1409276.
-RUN apt-get -o Acquire::Check-Valid-Until=false update -q && \
+RUN apt-get update -q && \
     apt-get install -yyq --no-install-recommends \
       autoconf2.13 \
       automake \
diff --git a/taskcluster/docker/toolchain-build/Dockerfile b/taskcluster/docker/toolchain-build/Dockerfile
index 1a4466dcf955..e6c760094c18 100644
--- a/taskcluster/docker/toolchain-build/Dockerfile
+++ b/taskcluster/docker/toolchain-build/Dockerfile
@@ -31,9 +31,10 @@ CMD ["/bin/bash", "--login"]
 # Set apt sources list to a snapshot.
 RUN for s in debian_wheezy debian_wheezy-updates debian-security_wheezy/updates; do \
       echo "deb http://snapshot.debian.org/archive/${s%_*}/20171210T214726Z/ ${s#*_} main"; \
-    done > /etc/apt/sources.list
+    done > /etc/apt/sources.list && \
+    echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99check-valid-until
 
-RUN apt-get -o Acquire::Check-Valid-Until=false update -q && \
+RUN apt-get update -q && \
     apt-get install -yyq --no-install-recommends \
       apt-transport-https \
       ca-certificates
@@ -43,7 +44,7 @@ RUN for task in $DOCKER_IMAGE_PACKAGES; do \
       echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/" >> /etc/apt/sources.list; \
     done
 
-RUN apt-get -o Acquire::Check-Valid-Until=false update -q && \
+RUN apt-get update -q && \
     apt-get install -yyq --no-install-recommends \
       autoconf \
       automake \

From 04b424481dea89892e74f56e3504079df48af6e4 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Thu, 25 Jan 2018 22:09:56 +0900
Subject: [PATCH 21/46] Bug 1433033 - Set apt defaults to avoid passing -q, -y
 and --no-install-recommends to it. r=dustin

--HG--
extra : rebase_source : 53c8f00ef24dc8757d8eb45d1a9e08da48539a7a
---
 taskcluster/docker/android-build/Dockerfile   | 12 ++++++++----
 taskcluster/docker/debian7-build/Dockerfile   | 14 +++++++++-----
 taskcluster/docker/toolchain-build/Dockerfile | 14 +++++++++-----
 3 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/taskcluster/docker/android-build/Dockerfile b/taskcluster/docker/android-build/Dockerfile
index b39ed9b9f65a..91b5b43609af 100644
--- a/taskcluster/docker/android-build/Dockerfile
+++ b/taskcluster/docker/android-build/Dockerfile
@@ -32,13 +32,17 @@ CMD ["/bin/bash", "--login"]
 # Set apt sources list to a snapshot.
 COPY sources.list /etc/apt/
 
-RUN echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99check-valid-until
+RUN ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+    ) > /etc/apt/apt.conf.d/99taskcluster
 
 # rsync is required for l10n single locale repacks.  less, screen, and
 # vim, help debugging interactive tasks in Task Cluster.
 # git and openssh-client are used to upload GeckoView javadoc to Github.
-RUN apt-get update -q && \
-    apt-get install -yyq --no-install-recommends \
+RUN apt-get update && \
+    apt-get install \
       autoconf2.13 \
       build-essential \
       base-files \
@@ -125,7 +129,7 @@ USER root
 # https://github.com/sonatype/docker-nexus/blob/fffd2c61b2368292040910c055cf690c8e76a272/oss/Dockerfile.
 
 # Observe missing --no-install-recommends, in order to install glib2.0/gconf/etc.
-RUN apt-get install -yyq \
+RUN apt-get install \
       libgconf2-4 \
       libgnome2-0 \
       openjdk-8-jdk-headless \
diff --git a/taskcluster/docker/debian7-build/Dockerfile b/taskcluster/docker/debian7-build/Dockerfile
index 5aa8ba75feb3..47bf92736b09 100644
--- a/taskcluster/docker/debian7-build/Dockerfile
+++ b/taskcluster/docker/debian7-build/Dockerfile
@@ -32,10 +32,14 @@ CMD ["/bin/bash", "--login"]
 RUN for s in debian_wheezy debian_wheezy-updates debian_wheezy-backports debian-security_wheezy/updates; do \
       echo "deb http://snapshot.debian.org/archive/${s%_*}/20171210T214726Z/ ${s#*_} main"; \
     done > /etc/apt/sources.list && \
-    echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99check-valid-until
+    ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+    ) > /etc/apt/apt.conf.d/99taskcluster
 
-RUN apt-get update -q && \
-    apt-get install -yyq --no-install-recommends \
+RUN apt-get update && \
+    apt-get install \
       apt-transport-https \
       ca-certificates
 
@@ -50,8 +54,8 @@ RUN dpkg --add-architecture $ARCH
 # Ideally, we wouldn't need gcc-multilib and the extra linux-libc-dev,
 # but the latter is required to make the former installable, and the former
 # because of bug 1409276.
-RUN apt-get update -q && \
-    apt-get install -yyq --no-install-recommends \
+RUN apt-get update && \
+    apt-get install \
       autoconf2.13 \
       automake \
       bzip2 \
diff --git a/taskcluster/docker/toolchain-build/Dockerfile b/taskcluster/docker/toolchain-build/Dockerfile
index e6c760094c18..863282d0718f 100644
--- a/taskcluster/docker/toolchain-build/Dockerfile
+++ b/taskcluster/docker/toolchain-build/Dockerfile
@@ -32,10 +32,14 @@ CMD ["/bin/bash", "--login"]
 RUN for s in debian_wheezy debian_wheezy-updates debian-security_wheezy/updates; do \
       echo "deb http://snapshot.debian.org/archive/${s%_*}/20171210T214726Z/ ${s#*_} main"; \
     done > /etc/apt/sources.list && \
-    echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99check-valid-until
+    ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+    ) > /etc/apt/apt.conf.d/99taskcluster
 
-RUN apt-get update -q && \
-    apt-get install -yyq --no-install-recommends \
+RUN apt-get update && \
+    apt-get install \
       apt-transport-https \
       ca-certificates
 
@@ -44,8 +48,8 @@ RUN for task in $DOCKER_IMAGE_PACKAGES; do \
       echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/" >> /etc/apt/sources.list; \
     done
 
-RUN apt-get update -q && \
-    apt-get install -yyq --no-install-recommends \
+RUN apt-get update && \
+    apt-get install \
       autoconf \
       automake \
       bison \

From 39720324989529ee736f0bdd89770c816f7c6260 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Thu, 25 Jan 2018 21:57:13 +0900
Subject: [PATCH 22/46] Bug 1433033 - Don't `apt-get clean` from Debian-based
 docker images. r=dustin

The base images from docker hub actually contain a
/etc/apt/apt.conf.d/docker-clean that does the equivalent of an apt-get
clean after installing packages.

--HG--
extra : rebase_source : 190de9e3b10a0309cf9cfb3260a91477a5a93ba3
---
 taskcluster/docker/android-build/Dockerfile   | 8 ++------
 taskcluster/docker/debian7-build/Dockerfile   | 4 +---
 taskcluster/docker/toolchain-build/Dockerfile | 4 +---
 3 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/taskcluster/docker/android-build/Dockerfile b/taskcluster/docker/android-build/Dockerfile
index 91b5b43609af..9e65c0704a7e 100644
--- a/taskcluster/docker/android-build/Dockerfile
+++ b/taskcluster/docker/android-build/Dockerfile
@@ -73,9 +73,7 @@ RUN apt-get update && \
       wget \
       xz-utils \
       yasm \
-      zip \
-    && \
-    apt-get clean
+      zip
 
 # %include python/mozbuild/mozbuild/action/tooltool.py
 COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
@@ -132,9 +130,7 @@ USER root
 RUN apt-get install \
       libgconf2-4 \
       libgnome2-0 \
-      openjdk-8-jdk-headless \
-    && \
-    apt-get clean
+      openjdk-8-jdk-headless
 
 ENV NEXUS_VERSION=2.12.0-01 \
     NEXUS_SHA1SUM=1a9aaad8414baffe0a2fd46eed1f41b85f4049e6 \
diff --git a/taskcluster/docker/debian7-build/Dockerfile b/taskcluster/docker/debian7-build/Dockerfile
index 47bf92736b09..7717c274dc00 100644
--- a/taskcluster/docker/debian7-build/Dockerfile
+++ b/taskcluster/docker/debian7-build/Dockerfile
@@ -98,9 +98,7 @@ RUN apt-get update && \
       libpulse-dev:$ARCH \
       libx11-xcb-dev:$ARCH \
       libxss-dev:$ARCH \
-      libxt-dev:$ARCH \
-    && \
-    apt-get clean
+      libxt-dev:$ARCH
 
 # %include testing/mozharness/external_tools/robustcheckout.py
 COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
diff --git a/taskcluster/docker/toolchain-build/Dockerfile b/taskcluster/docker/toolchain-build/Dockerfile
index 863282d0718f..ac3e8ea464b6 100644
--- a/taskcluster/docker/toolchain-build/Dockerfile
+++ b/taskcluster/docker/toolchain-build/Dockerfile
@@ -80,9 +80,7 @@ RUN apt-get update && \
       wget \
       xz-utils \
       zip \
-      zlib1g-dev \
-    && \
-    apt-get clean
+      zlib1g-dev
 
 # %include testing/mozharness/external_tools/robustcheckout.py
 COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py

From db4394b2577e008aa709fd31263c99910f426c67 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Thu, 25 Jan 2018 22:15:19 +0900
Subject: [PATCH 23/46] Bug 1433033 - Create a base Debian docker image.
 r=dustin

That image is used to derive all the debian7-* images, and its
definition is parametrized, which will allow to create other images
based on other versions of Debian, from the same definition.

XZ_OPT is kept in each of those because we don't want to automatically
set it in all further derived images.

--HG--
extra : rebase_source : 7f4597c1ea4af83627a9373dbdc7945d20b7d996
---
 taskcluster/ci/docker-image/kind.yml          | 31 +++++----
 taskcluster/docker/debian-base/Dockerfile     | 69 +++++++++++++++++++
 .../docker/debian-base/setup_packages.sh      |  5 ++
 taskcluster/docker/debian7-build/Dockerfile   | 65 ++---------------
 taskcluster/docker/toolchain-build/Dockerfile | 64 ++---------------
 5 files changed, 98 insertions(+), 136 deletions(-)
 create mode 100644 taskcluster/docker/debian-base/Dockerfile
 create mode 100755 taskcluster/docker/debian-base/setup_packages.sh

diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml
index c0983c3f795a..5e0f91da697d 100644
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -25,38 +25,39 @@ jobs:
     symbol: I(db)
   valgrind-build:
     symbol: I(vb)
+  debian7-base:
+    symbol: I(deb7-base)
+    definition: debian-base
+    args:
+      DIST: wheezy
+      BASE_TAG: '20171210'
+      SNAPSHOT: '20171210T214726Z'
+    packages:
+      - deb7-git
+      - deb7-make
+      - deb7-mercurial
+      - deb7-python
+      - deb7-xz-utils
   toolchain-build:
     symbol: I(toolchain)
+    parent: debian7-base
     packages:
       - deb7-cmake
-      - deb7-git
-      - deb7-make
-      - deb7-mercurial
       - deb7-ninja
-      - deb7-python
-      - deb7-xz-utils
   debian7-amd64-build:
     symbol: I(deb7)
+    parent: debian7-base
     definition: debian7-build
     packages:
-      - deb7-git
-      - deb7-make
-      - deb7-mercurial
-      - deb7-python
       - deb7-valgrind
-      - deb7-xz-utils
     args:
       ARCH: amd64
   debian7-i386-build:
     symbol: I(deb7-32)
+    parent: debian7-base
     definition: debian7-build
     packages:
-      - deb7-git
-      - deb7-make
-      - deb7-mercurial
-      - deb7-python
       - deb7-valgrind
-      - deb7-xz-utils
     args:
       ARCH: i386
   lint:
diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile
new file mode 100644
index 000000000000..22dbc57d7c94
--- /dev/null
+++ b/taskcluster/docker/debian-base/Dockerfile
@@ -0,0 +1,69 @@
+# %ARG DIST
+# %ARG BASE_TAG
+FROM debian:$DIST-$BASE_TAG
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+### Add worker user and setup its workspace.
+RUN mkdir /builds && \
+    groupadd -g 500 worker && \
+    useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \
+    mkdir -p /builds/worker/workspace && \
+    chown -R worker:worker /builds
+
+# Declare default working folder
+WORKDIR /builds/worker
+
+# Set variable normally configured at login, by the shells parent process, these
+# are taken from GNU su manual
+ENV HOME=/builds/worker \
+    SHELL=/bin/bash \
+    USER=worker \
+    LOGNAME=worker \
+    HOSTNAME=taskcluster-worker \
+    DEBIAN_FRONTEND=noninteractive
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]
+
+# %ARG SNAPSHOT
+# Set apt sources list to a snapshot.
+RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \
+      echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \
+    done > /etc/apt/sources.list && \
+    ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+    ) > /etc/apt/apt.conf.d/99taskcluster
+
+RUN apt-get update && \
+    apt-get install \
+      apt-transport-https \
+      ca-certificates
+
+COPY setup_packages.sh /usr/local/sbin/
+
+# %ARG DOCKER_IMAGE_PACKAGES
+RUN /usr/local/sbin/setup_packages.sh $DOCKER_IMAGE_PACKAGES && \
+    apt-get update && \
+    apt-get install \
+      git \
+      make \
+      mercurial \
+      python \
+      xz-utils
+
+# %include testing/mozharness/external_tools/robustcheckout.py
+COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
+
+# %include taskcluster/docker/recipes/hgrc
+COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
+
+# Add pip configuration, among other things.
+# %include taskcluster/docker/recipes/dot-config
+COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
+
+# %include taskcluster/docker/recipes/run-task
+COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
+
+RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/*
diff --git a/taskcluster/docker/debian-base/setup_packages.sh b/taskcluster/docker/debian-base/setup_packages.sh
new file mode 100755
index 000000000000..4d95560deb2e
--- /dev/null
+++ b/taskcluster/docker/debian-base/setup_packages.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for task in "$@"; do
+  echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/"
+done > /etc/apt/sources.list.d/99packages.list
diff --git a/taskcluster/docker/debian7-build/Dockerfile b/taskcluster/docker/debian7-build/Dockerfile
index 7717c274dc00..9065cac37ff0 100644
--- a/taskcluster/docker/debian7-build/Dockerfile
+++ b/taskcluster/docker/debian7-build/Dockerfile
@@ -1,52 +1,15 @@
-FROM debian:wheezy-20171210
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
 MAINTAINER Mike Hommey <mhommey@mozilla.com>
 
-### Add worker user and setup its workspace.
-RUN mkdir /builds && \
-    groupadd -g 500 worker && \
-    useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \
-    mkdir -p /builds/worker/workspace && \
-    chown -R worker:worker /builds
-
-# Declare default working folder
-WORKDIR /builds/worker
-
 VOLUME /builds/worker/checkouts
 VOLUME /builds/worker/workspace
 VOLUME /builds/worker/tooltool-cache
 
-# Set variable normally configured at login, by the shells parent process, these
-# are taken from GNU su manual
-ENV HOME=/builds/worker \
-    SHELL=/bin/bash \
-    USER=worker \
-    LOGNAME=worker \
-    HOSTNAME=taskcluster-worker \
-    XZ_OPT=-T0 \
-    DEBIAN_FRONTEND=noninteractive
-
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
-
-# Set apt sources list to a snapshot.
-RUN for s in debian_wheezy debian_wheezy-updates debian_wheezy-backports debian-security_wheezy/updates; do \
-      echo "deb http://snapshot.debian.org/archive/${s%_*}/20171210T214726Z/ ${s#*_} main"; \
-    done > /etc/apt/sources.list && \
-    ( echo 'quiet "true";'; \
-      echo 'APT::Get::Assume-Yes "true";'; \
-      echo 'APT::Install-Recommends "false";'; \
-      echo 'Acquire::Check-Valid-Until "false";'; \
-    ) > /etc/apt/apt.conf.d/99taskcluster
-
-RUN apt-get update && \
-    apt-get install \
-      apt-transport-https \
-      ca-certificates
+ENV XZ_OPT=-T0
 
 # %ARG DOCKER_IMAGE_PACKAGES
-RUN for task in $DOCKER_IMAGE_PACKAGES; do \
-      echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/" >> /etc/apt/sources.list; \
-    done
+RUN /usr/local/sbin/setup_packages.sh $DOCKER_IMAGE_PACKAGES
 
 # %ARG ARCH
 RUN dpkg --add-architecture $ARCH
@@ -63,13 +26,9 @@ RUN apt-get update && \
       file \
       gawk \
       gcc-multilib \
-      git \
       gnupg \
-      make \
-      mercurial \
       p7zip-full \
       procps \
-      python \
       python-pip \
       python-setuptools \
       python-virtualenv \
@@ -82,7 +41,6 @@ RUN apt-get update && \
       wget \
       x11-utils \
       xvfb \
-      xz-utils \
       yasm/wheezy-backports \
       zip \
       linux-libc-dev/wheezy-backports \
@@ -100,23 +58,10 @@ RUN apt-get update && \
       libxss-dev:$ARCH \
       libxt-dev:$ARCH
 
-# %include testing/mozharness/external_tools/robustcheckout.py
-COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-
-# %include taskcluster/docker/recipes/hgrc
-COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
-
-# Add pip configuration, among other things.
-# %include taskcluster/docker/recipes/dot-config
-COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
-
 # Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
 # %include taskcluster/docker/recipes/xvfb.sh
 COPY topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh
 
-# %include taskcluster/docker/recipes/run-task
-COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
-
 # Stubbed out credentials; mozharness looks for this file an issues a WARNING
 # if it's not found, which causes the build to fail.  Note that this needs to
 # be in the parent of the workspace directory and in the directory where
@@ -129,5 +74,3 @@ COPY topsrcdir/taskcluster/docker/desktop-build/oauth.txt /builds/worker/
 # the directory where mozharness is run (not its --work-dir)
 # %include taskcluster/docker/desktop-build/buildprops.json
 COPY topsrcdir/taskcluster/docker/desktop-build/buildprops.json /builds/worker/
-
-RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/*
diff --git a/taskcluster/docker/toolchain-build/Dockerfile b/taskcluster/docker/toolchain-build/Dockerfile
index ac3e8ea464b6..6c651e51f6c5 100644
--- a/taskcluster/docker/toolchain-build/Dockerfile
+++ b/taskcluster/docker/toolchain-build/Dockerfile
@@ -1,52 +1,15 @@
-FROM debian:wheezy-20171210
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
 MAINTAINER Mike Hommey <mhommey@mozilla.com>
 
-### Add worker user and setup its workspace.
-RUN mkdir /builds && \
-    groupadd -g 500 worker && \
-    useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \
-    mkdir -p /builds/worker/workspace && \
-    chown -R worker:worker /builds
-
-# Declare default working folder
-WORKDIR /builds/worker
-
 VOLUME /builds/worker/checkouts
 VOLUME /builds/worker/workspace
 VOLUME /builds/worker/tooltool-cache
 
-# Set variable normally configured at login, by the shells parent process, these
-# are taken from GNU su manual
-ENV HOME=/builds/worker \
-    SHELL=/bin/bash \
-    USER=worker \
-    LOGNAME=worker \
-    HOSTNAME=taskcluster-worker \
-    XZ_OPT=-T0 \
-    DEBIAN_FRONTEND=noninteractive
-
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
-
-# Set apt sources list to a snapshot.
-RUN for s in debian_wheezy debian_wheezy-updates debian-security_wheezy/updates; do \
-      echo "deb http://snapshot.debian.org/archive/${s%_*}/20171210T214726Z/ ${s#*_} main"; \
-    done > /etc/apt/sources.list && \
-    ( echo 'quiet "true";'; \
-      echo 'APT::Get::Assume-Yes "true";'; \
-      echo 'APT::Install-Recommends "false";'; \
-      echo 'Acquire::Check-Valid-Until "false";'; \
-    ) > /etc/apt/apt.conf.d/99taskcluster
-
-RUN apt-get update && \
-    apt-get install \
-      apt-transport-https \
-      ca-certificates
+ENV XZ_OPT=-T0
 
 # %ARG DOCKER_IMAGE_PACKAGES
-RUN for task in $DOCKER_IMAGE_PACKAGES; do \
-      echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task/$task/runs/0/artifacts/public/build/ debian/" >> /etc/apt/sources.list; \
-    done
+RUN /usr/local/sbin/setup_packages.sh $DOCKER_IMAGE_PACKAGES
 
 RUN apt-get update && \
     apt-get install \
@@ -59,13 +22,10 @@ RUN apt-get update && \
       flex \
       gawk \
       gcc-multilib \
-      git \
       gnupg \
       libbz2-dev \
       libssl-dev \
       libtool \
-      make \
-      mercurial \
       ninja-build \
       p7zip-full \
       procps \
@@ -78,21 +38,5 @@ RUN apt-get update && \
       uuid \
       uuid-dev \
       wget \
-      xz-utils \
       zip \
       zlib1g-dev
-
-# %include testing/mozharness/external_tools/robustcheckout.py
-COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-
-# %include taskcluster/docker/recipes/hgrc
-COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
-
-# Add pip configuration, among other things.
-# %include taskcluster/docker/recipes/dot-config
-COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
-
-# %include taskcluster/docker/recipes/run-task
-COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
-
-RUN chown -R worker:worker /builds/worker/bin && chmod 755 /builds/worker/bin/*

From 220ae81745920ea242263f24ea1c2109fe561597 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Thu, 25 Jan 2018 22:23:15 +0900
Subject: [PATCH 24/46] Bug 1433033 - Create a mercurial package for Debian
 stretch. r=dustin

--HG--
extra : rebase_source : 6408957ddc6acf671c7fa190ec156e347124f588
---
 taskcluster/ci/config.yml        |  1 +
 taskcluster/ci/packages/kind.yml | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml
index 8d4b4e1a5500..bfc6e3d97a11 100644
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -41,6 +41,7 @@ treeherder:
         'BMcs': 'Beetmover checksums,'
         'Aries': 'Aries Device Image'
         'Deb7': 'Packages for Debian 7'
+        'Deb9': 'Packages for Debian 9'
         'Nexus 5-L': 'Nexus 5-L Device Image'
         'I': 'Docker Image Builds'
         'TL': 'Toolchain builds for Linux 64-bits'
diff --git a/taskcluster/ci/packages/kind.yml b/taskcluster/ci/packages/kind.yml
index 4d97a617386d..29bb6e92e673 100644
--- a/taskcluster/ci/packages/kind.yml
+++ b/taskcluster/ci/packages/kind.yml
@@ -82,6 +82,22 @@ jobs:
                -e "s/__DATE__/$(date --rfc-2822)/" \
                -e "s/__CODENAME__/wheezy/" debian/changelog
 
+  deb9-mercurial:
+    description: "Modern Mercurial for Debian stretch"
+    treeherder:
+      symbol: Deb9(hg)
+    run:
+      using: debian-package
+      dist: stretch
+      tarball:
+        url: https://www.mercurial-scm.org/release/mercurial-4.4.2.tar.gz
+        sha256: dc2f72caccd6b760226753d48c2f4e8889fe176a6b23ef50775caac55ce28b85
+      pre-build-command: >-
+        cp -r contrib/debian debian &&
+        sed -i -e "s/__VERSION__/$(awk -F\" '$2 {print $2}' mercurial/__version__.py)-1.deb9moz1/" \
+               -e "s/__DATE__/$(date --rfc-2822)/" \
+               -e "s/__CODENAME__/stretch/" debian/changelog
+
   deb7-git:
     description: "Modern git for Debian wheezy"
     treeherder:

From ae1ce5d5716c88e096f4ee9a1d1800800c385513 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Thu, 25 Jan 2018 22:26:06 +0900
Subject: [PATCH 25/46] Bug 1433033 - Workarounds for Debian stretch. r=dustin

The apt in Debian stretch doesn't allow repositories with a Release file
not being GPG signed. Setting up GPG signatures on the
taskcluster-artifact-based repositories is a tricky process, and not
strictly necessary. It turns out not creating a Release file at all
works just as well, and works across all current Debian versions. The
packages priorities remain the same, such that packages from those
repositories are still prefered over the ones from the main Debian
repository (as long as versions are higher).

See comment in cloud-mirror-workaround.sh for that part.

--HG--
extra : rebase_source : df5af330859a314285a6c1922d899489997d2f19
---
 taskcluster/docker/debian-base/Dockerfile              |  2 ++
 .../docker/debian-base/cloud-mirror-workaround.sh      | 10 ++++++++++
 taskcluster/taskgraph/transforms/job/debian_package.py |  4 +---
 3 files changed, 13 insertions(+), 3 deletions(-)
 create mode 100755 taskcluster/docker/debian-base/cloud-mirror-workaround.sh

diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile
index 22dbc57d7c94..ea693fe9987d 100644
--- a/taskcluster/docker/debian-base/Dockerfile
+++ b/taskcluster/docker/debian-base/Dockerfile
@@ -42,9 +42,11 @@ RUN apt-get update && \
       ca-certificates
 
 COPY setup_packages.sh /usr/local/sbin/
+COPY cloud-mirror-workaround.sh /usr/local/sbin/
 
 # %ARG DOCKER_IMAGE_PACKAGES
 RUN /usr/local/sbin/setup_packages.sh $DOCKER_IMAGE_PACKAGES && \
+    echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";' > /etc/apt/apt.conf.d/99cloud-mirror-workaround && \
     apt-get update && \
     apt-get install \
       git \
diff --git a/taskcluster/docker/debian-base/cloud-mirror-workaround.sh b/taskcluster/docker/debian-base/cloud-mirror-workaround.sh
new file mode 100755
index 000000000000..53819a4d7f9b
--- /dev/null
+++ b/taskcluster/docker/debian-base/cloud-mirror-workaround.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# The apt in Debian stretch mishandles the HTTP redirects from queue.taskcluster.net to
+# cloud-mirror.taskcluster.net, and unescapes the url. This apt method wrapper
+# strips redirections and sends directly to the S3 bucket. This has the downside of
+# always hitting us-west-2 independently of the zone this runs on, but this is only
+# used when creating docker images, so shouldn't generate huge cross-AWS-zone S3
+# transfers.
+
+/usr/lib/apt/methods/https | sed -u 's,^New-URI: https://cloud-mirror.taskcluster.net/v1/redirect/[^/]*/[^/]*/https://,New-URI: https://,'
diff --git a/taskcluster/taskgraph/transforms/job/debian_package.py b/taskcluster/taskgraph/transforms/job/debian_package.py
index 7926de6615cd..98a85d11915e 100644
--- a/taskcluster/taskgraph/transforms/job/debian_package.py
+++ b/taskcluster/taskgraph/transforms/job/debian_package.py
@@ -182,9 +182,7 @@ def docker_worker_debian_package(config, job, taskdesc):
         'cd {artifacts} && '
         # Make the artifacts directory usable as an APT repository.
         'apt-ftparchive sources debian | gzip -c9 > debian/Sources.gz && '
-        'apt-ftparchive packages debian | gzip -c9 > debian/Packages.gz && '
-        'apt-ftparchive release -o APT::FTPArchive::Release::Codename={dist} debian > Release && '
-        'mv Release debian/'
+        'apt-ftparchive packages debian | gzip -c9 > debian/Packages.gz'
         .format(
             package=package,
             snapshot=run['snapshot'],

From 3ab004d96615fae245ecc0a1880f468f3e1072d8 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Thu, 25 Jan 2018 22:31:48 +0900
Subject: [PATCH 26/46] Bug 1433033 - Derive the android-build image from a
 base Debian stretch image. r=dustin

--HG--
extra : rebase_source : 8be13be0a0c3979f6596514f62765a9b741fd8f7
---
 taskcluster/ci/docker-image/kind.yml          | 10 +++
 taskcluster/docker/android-build/Dockerfile   | 69 +------------------
 taskcluster/docker/android-build/sources.list |  3 -
 3 files changed, 12 insertions(+), 70 deletions(-)
 delete mode 100644 taskcluster/docker/android-build/sources.list

diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml
index 5e0f91da697d..8075aae99b7d 100644
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -62,8 +62,18 @@ jobs:
       ARCH: i386
   lint:
     symbol: I(lnt)
+  debian9-base:
+    symbol: I(deb9-base)
+    definition: debian-base
+    args:
+      DIST: stretch
+      BASE_TAG: '20170620'
+      SNAPSHOT: '20170830T000511Z'
+    packages:
+      - deb9-mercurial
   android-build:
     symbol: I(agb)
+    parent: debian9-base
   index-task:
     symbol: I(idx)
   funsize-update-generator:
diff --git a/taskcluster/docker/android-build/Dockerfile b/taskcluster/docker/android-build/Dockerfile
index 9e65c0704a7e..416cea320534 100644
--- a/taskcluster/docker/android-build/Dockerfile
+++ b/taskcluster/docker/android-build/Dockerfile
@@ -1,43 +1,11 @@
-FROM debian:stretch-20170620
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
 MAINTAINER Nick Alexander <nalexander@mozilla.com>
 
-### Add worker user and setup its workspace.
-RUN mkdir -p /builds && \
-    groupadd -g 500 worker && \
-    useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker && \
-    chown -R worker:worker /builds && \
-    mkdir -p /builds/worker/workspace && \
-    chown -R worker:worker /builds/worker/workspace
-
-# Declare default working folder
-WORKDIR /builds/worker
-
 VOLUME /builds/worker/checkouts
 VOLUME /builds/worker/workspace
 VOLUME /builds/worker/tooltool-cache
 
-# Set variables normally configured at login, by the shell' parent process,
-# these are taken from GNU su manual.  Also set DEBIAN_FRONTEND to advise
-# apt-get of our situation.
-ENV HOME=/builds/worker \
-    SHELL=/bin/bash \
-    USER=worker \
-    LOGNAME=worker \
-    HOSTNAME=taskcluster-worker \
-    DEBIAN_FRONTEND=noninteractive
-
-# Set a default command useful for debugging.
-CMD ["/bin/bash", "--login"]
-
-# Set apt sources list to a snapshot.
-COPY sources.list /etc/apt/
-
-RUN ( echo 'quiet "true";'; \
-      echo 'APT::Get::Assume-Yes "true";'; \
-      echo 'APT::Install-Recommends "false";'; \
-      echo 'Acquire::Check-Valid-Until "false";'; \
-    ) > /etc/apt/apt.conf.d/99taskcluster
-
 # rsync is required for l10n single locale repacks.  less, screen, and
 # vim, help debugging interactive tasks in Task Cluster.
 # git and openssh-client are used to upload GeckoView javadoc to Github.
@@ -46,20 +14,15 @@ RUN apt-get update && \
       autoconf2.13 \
       build-essential \
       base-files \
-      ca-certificates \
       ccache \
       cmake \
       curl \
       file \
-      git \
       gnupg \
       less \
-      make \
       openssh-client \
       procps \
-      python \
       python-cryptography \
-      python-dev \
       python-pip \
       python-setuptools \
       python-virtualenv \
@@ -71,41 +34,13 @@ RUN apt-get update && \
       uuid \
       vim \
       wget \
-      xz-utils \
       yasm \
       zip
 
-# %include python/mozbuild/mozbuild/action/tooltool.py
-COPY topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
-
-# %include testing/mozharness/external_tools/robustcheckout.py
-COPY topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-
-# %include taskcluster/docker/recipes/hgrc
-COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
-
-# %include taskcluster/docker/recipes/common.sh
-COPY topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh
-
-# %include taskcluster/docker/recipes/install-mercurial.sh
-COPY topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
-
-# %include taskcluster/docker/recipes/debian-build-system-setup.sh
-COPY topsrcdir/taskcluster/docker/recipes/debian-build-system-setup.sh /setup/system-setup.sh
-
-RUN bash /setup/system-setup.sh
-
 # Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb.
 # %include taskcluster/docker/recipes/xvfb.sh
 COPY topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh
 
-# %include taskcluster/docker/recipes/run-task
-COPY topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task
-
-# Add pip configuration, among other things.
-# %include taskcluster/docker/recipes/dot-config
-COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
-
 # Stubbed out credentials; mozharness looks for this file an issues a WARNING
 # if it's not found, which causes the build to fail.  Note that this needs to
 # be in the parent of the workspace directory and in the directory where
diff --git a/taskcluster/docker/android-build/sources.list b/taskcluster/docker/android-build/sources.list
deleted file mode 100644
index 41987f8e8dc5..000000000000
--- a/taskcluster/docker/android-build/sources.list
+++ /dev/null
@@ -1,3 +0,0 @@
-deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/20170830T000511Z/ stretch main
-deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/20170830T000511Z/ stretch-updates main
-deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/20170830T000511Z/ stretch/updates main

From 9d3a568f0343cb031f1a19a61543a076c6ca9d10 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Fri, 26 Jan 2018 12:40:26 +0900
Subject: [PATCH 27/46] Bug 1433033 - Make APT retry when it fails to download
 files. r=dustin

We've observed apt failures multiple times where it apparently fails to
get a file in full from snapshot.debian.org. Making it retry
automatically rather than retriggering tasks seems better.

--HG--
extra : rebase_source : f3ffb415ccc30b7e7c44e6a48b29eb20e69efdd5
---
 taskcluster/docker/debian-base/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile
index ea693fe9987d..01e95d4e12a8 100644
--- a/taskcluster/docker/debian-base/Dockerfile
+++ b/taskcluster/docker/debian-base/Dockerfile
@@ -34,6 +34,7 @@ RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-sec
       echo 'APT::Get::Assume-Yes "true";'; \
       echo 'APT::Install-Recommends "false";'; \
       echo 'Acquire::Check-Valid-Until "false";'; \
+      echo 'Acquire::Retries "5";'; \
     ) > /etc/apt/apt.conf.d/99taskcluster
 
 RUN apt-get update && \

From 3a35e750f388623b5b621486baf2925861fa57b9 Mon Sep 17 00:00:00 2001
From: Sylvestre Ledru <sledru@mozilla.com>
Date: Fri, 26 Jan 2018 12:50:07 +0100
Subject: [PATCH 28/46] Bug 1433417 - Fix a bunch of typo in the doc r=ahal

MozReview-Commit-ID: LRgL0CMJdDP

--HG--
extra : rebase_source : b99364bb96c1501a674a1726a3b5fdf0664a8e4f
---
 browser/experiments/docs/manifest.rst                         | 2 +-
 build/docs/defining-binaries.rst                              | 2 +-
 build/docs/glossary.rst                                       | 2 +-
 build/docs/mozinfo.rst                                        | 2 +-
 mobile/android/docs/downloadcontenttelemetry.rst              | 2 +-
 mobile/android/docs/push.rst                                  | 2 +-
 mobile/android/docs/shutdown.rst                              | 2 +-
 taskcluster/docs/cron.rst                                     | 2 +-
 taskcluster/docs/optimization.rst                             | 2 +-
 taskcluster/docs/parameters.rst                               | 2 +-
 taskcluster/docs/taskgraph.rst                                | 2 +-
 taskcluster/docs/try.rst                                      | 4 ++--
 testing/mozbase/docs/manifestparser.rst                       | 4 ++--
 testing/mozbase/docs/mozlog.rst                               | 4 ++--
 testing/mozbase/docs/mozversion.rst                           | 4 ++--
 testing/web-platform/tests/tools/wptrunner/README.rst         | 2 +-
 toolkit/components/extensions/docs/events.rst                 | 2 +-
 toolkit/components/extensions/docs/functions.rst              | 2 +-
 .../components/telemetry/docs/data/first-shutdown-ping.rst    | 2 +-
 toolkit/components/telemetry/docs/data/main-ping.rst          | 2 +-
 toolkit/components/telemetry/docs/data/sync-ping.rst          | 2 +-
 toolkit/crashreporter/docs/index.rst                          | 2 +-
 tools/lint/docs/linters/eslint-plugin-mozilla.rst             | 4 ++--
 23 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/browser/experiments/docs/manifest.rst b/browser/experiments/docs/manifest.rst
index d5b4935dcbe4..ab8c13f82c22 100644
--- a/browser/experiments/docs/manifest.rst
+++ b/browser/experiments/docs/manifest.rst
@@ -320,7 +320,7 @@ client should remember what the expiration events were for an experiment
 and honor them.
 
 The rationale here is that we want to prevent an accidental deletion
-or temporary failure on the server to inadvertantly deactivate
+or temporary failure on the server to inadvertently deactivate
 supposed-to-be-active experiments. We also don't want premature deletion
 of an experiment from the manifest to result in indefinite activation
 periods.
diff --git a/build/docs/defining-binaries.rst b/build/docs/defining-binaries.rst
index 388ce435d1ed..fdac27e26a80 100644
--- a/build/docs/defining-binaries.rst
+++ b/build/docs/defining-binaries.rst
@@ -275,7 +275,7 @@ In some cases, for convenience, it is possible to set both
 This allows to use ``mylib`` in the ``USE_LIBS`` of another library or
 executable.
 
-When refering to a ``Library`` name building both types of libraries in
+When referring to a ``Library`` name building both types of libraries in
 ``USE_LIBS``, the shared library is chosen to be linked. But sometimes,
 it is wanted to link the static version, in which case the ``Library`` name
 needs to be prefixed with ``static:`` in ``USE_LIBS``
diff --git a/build/docs/glossary.rst b/build/docs/glossary.rst
index f846effe576e..e040f1af47b1 100644
--- a/build/docs/glossary.rst
+++ b/build/docs/glossary.rst
@@ -32,7 +32,7 @@ Glossary
        A large part of the build system consists of copying files
        around to appropriate places. We write out special files
        describing the set of required operations so we can process the
-       actions effeciently. These files are install manifests.
+       actions efficiently. These files are install manifests.
 
    clobber build
       A build performed with an initially empty object directory. All
diff --git a/build/docs/mozinfo.rst b/build/docs/mozinfo.rst
index e7d2ed6749f2..d44b814beee8 100644
--- a/build/docs/mozinfo.rst
+++ b/build/docs/mozinfo.rst
@@ -54,7 +54,7 @@ bits
 
    Universal Mac builds do not have this key defined.
 
-   Unkown processor architectures (see ``processor`` below) may not have
+   Unknown processor architectures (see ``processor`` below) may not have
    this key defined.
 
    Optional.
diff --git a/mobile/android/docs/downloadcontenttelemetry.rst b/mobile/android/docs/downloadcontenttelemetry.rst
index 6c6f9bf2784c..3f7ab0938965 100644
--- a/mobile/android/docs/downloadcontenttelemetry.rst
+++ b/mobile/android/docs/downloadcontenttelemetry.rst
@@ -26,7 +26,7 @@ Telemetry extras sent for a successful content download might look like this:
         "content": "25610abb-5dc8-fd75-40e7-990507f010c4"
     }
 
-For failed content downloads an additional ``error`` field contains the error type that occured when downloading the content. The value can be one of:
+For failed content downloads an additional ``error`` field contains the error type that occurred when downloading the content. The value can be one of:
 
 - no_network
 - network_metered
diff --git a/mobile/android/docs/push.rst b/mobile/android/docs/push.rst
index b1b5d48f2b15..5c67dc5b2830 100644
--- a/mobile/android/docs/push.rst
+++ b/mobile/android/docs/push.rst
@@ -125,7 +125,7 @@ registration due to inactivity or an unexpected server event.  Each
 `PushSubscription` is associated to a given *uaid* and correponds to a unique
 (per-*uaid*) *chid* (Channel ID) on the autopush server.  An individual *chid*
 is potentially long-lived, but clients must expect the service to expire *chid*s
-as part of regular maintainence.  The `PushManager` uses an `AutopushClient`
+as part of regular maintenance.  The `PushManager` uses an `AutopushClient`
 instance to interact with the autopush server.
 
 Between the `PushManager`, the `PushManagerStorage`, and assorted GCM event
diff --git a/mobile/android/docs/shutdown.rst b/mobile/android/docs/shutdown.rst
index 9ec0655f9de4..2004a7bfb519 100644
--- a/mobile/android/docs/shutdown.rst
+++ b/mobile/android/docs/shutdown.rst
@@ -27,7 +27,7 @@ shutdown as well, so as to
 1) provide an immediate visual feedback to the user that Firefox is indeed quitting
 
 2) avoid a state where the UI is still running "normally" while the rendering engine is already
-   shutting down, which could lead to loosing incoming external tabs if they were to arrive within
+   shutting down, which could lead to losing incoming external tabs if they were to arrive within
    that period.
 
 Therefore, shutdown of the native UI was originally started simultaneously with notifying Gecko.
diff --git a/taskcluster/docs/cron.rst b/taskcluster/docs/cron.rst
index 970446065aa4..faecbd04e673 100644
--- a/taskcluster/docs/cron.rst
+++ b/taskcluster/docs/cron.rst
@@ -55,4 +55,4 @@ tasks with scopes for that particular job, by name.  For example, the
     job from using any of the scopes afforded to the cron task itself (the
     ``..cron:*`` scope).  This is simply because the cron task runs arbitrary
     code from the repo, and that code can be easily modified to create tasks
-    with any scopes that it posesses.
+    with any scopes that it possesses.
diff --git a/taskcluster/docs/optimization.rst b/taskcluster/docs/optimization.rst
index e6c705116f31..39fd3db3d8ad 100644
--- a/taskcluster/docs/optimization.rst
+++ b/taskcluster/docs/optimization.rst
@@ -31,7 +31,7 @@ Optimizing Target Tasks
 
 In some cases, such as try pushes, tasks in the target task set have been
 explicitly requested and are thus excluded from optimization. In other cases,
-the target task set is almost the entire task graph, so targetted tasks are
+the target task set is almost the entire task graph, so targeted tasks are
 considered for optimization. This behavior is controlled with the
 ``optimize_target_tasks`` parameter.
 
diff --git a/taskcluster/docs/parameters.rst b/taskcluster/docs/parameters.rst
index 5a426e57e135..b97e12596cb8 100644
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -16,7 +16,7 @@ best to find a recent decision task's ``parameters.yml`` file, and modify that
 file if necessary, rather than starting from scratch.  This ensures you have a
 complete set of parameters.
 
-The properties of the parameters object are described here, divided rougly by
+The properties of the parameters object are described here, divided roughly by
 topic.
 
 Push Information
diff --git a/taskcluster/docs/taskgraph.rst b/taskcluster/docs/taskgraph.rst
index 2995bb841e7e..ab065f11c23d 100644
--- a/taskcluster/docs/taskgraph.rst
+++ b/taskcluster/docs/taskgraph.rst
@@ -152,7 +152,7 @@ As part of the execution of the Gecko decision task we generate a
 ``public/runnable-jobs.json.gz`` file. It contains a subset of all the data
 contained within the ``full-task-graph.json``.
 
-This file has the minimum ammount of data needed by Treeherder to show all
+This file has the minimum amount of data needed by Treeherder to show all
 tasks that can be scheduled on a push.
 
 
diff --git a/taskcluster/docs/try.rst b/taskcluster/docs/try.rst
index adeeaa2f1fd4..a6fd41772a7a 100644
--- a/taskcluster/docs/try.rst
+++ b/taskcluster/docs/try.rst
@@ -2,10 +2,10 @@ Try
 ===
 
 "Try" is a way to "try out" a proposed change safely before review, without
-officialy landing it.  This functionality has been around for a *long* time in
+officially landing it.  This functionality has been around for a *long* time in
 various forms, and can sometimes show its age.
 
-Access to "push to try" is typically avilable to a much larger group of
+Access to "push to try" is typically available to a much larger group of
 developers than those who can land changes in integration and release branches.
 Specifically, try pushes are allowed for anyone with `SCM Level`_ 1, while
 integration branches are at SCM level 3.
diff --git a/testing/mozbase/docs/manifestparser.rst b/testing/mozbase/docs/manifestparser.rst
index 9265ec605c17..e640b917173a 100644
--- a/testing/mozbase/docs/manifestparser.rst
+++ b/testing/mozbase/docs/manifestparser.rst
@@ -259,12 +259,12 @@ There is a two- or three-layered approach to the manifestparser
 architecture, depending on your needs:
 
 1. ManifestParser: this is a generic parser for .ini manifests that
-facilitates the `[include:]` logic and the inheritence of
+facilitates the `[include:]` logic and the inheritance of
 metadata. Despite the internal variable being called `self.tests`
 (an oversight), this layer has nothing in particular to do with tests.
 
 2. TestManifest: this is a harness-agnostic integration layer that is
-test-specific. TestManifest faciliates `skip-if` logic.
+test-specific. TestManifest facilitates `skip-if` logic.
 
 3. Optionally, a harness will have an integration layer than inherits
 from TestManifest if more harness-specific customization is desired at
diff --git a/testing/mozbase/docs/mozlog.rst b/testing/mozbase/docs/mozlog.rst
index bfaa955bb5d4..90f4e1ec0d39 100644
--- a/testing/mozbase/docs/mozlog.rst
+++ b/testing/mozbase/docs/mozlog.rst
@@ -190,7 +190,7 @@ harness that loads JavaScript-based tests in a browser. Each url
 loaded would be a single test, with corresponding ``test_start`` and
 ``test_end`` messages. If there can be more than one JS-defined test
 on a page, however, it it useful to track the results of those tests
-seperately. Therefore each of those tests is a subtest, and one
+separately. Therefore each of those tests is a subtest, and one
 ``test_status`` message must be generated for each subtest result.
 
 Subtests must have a name that is unique within their parent test.
@@ -401,7 +401,7 @@ More Complete Example
 ---------------------
 
 This example shows a complete toy testharness set up to used
-structured logging. It is avaliable as `structured_example.py <_static/structured_example.py>`_:
+structured logging. It is available as `structured_example.py <_static/structured_example.py>`_:
 
 .. literalinclude:: _static/structured_example.py
 
diff --git a/testing/mozbase/docs/mozversion.rst b/testing/mozbase/docs/mozversion.rst
index e2b1b4590c20..b9c70218b694 100644
--- a/testing/mozbase/docs/mozversion.rst
+++ b/testing/mozbase/docs/mozversion.rst
@@ -60,7 +60,7 @@ Options
 '''''''''
 
 This is the path to the target application binary or .apk. If this is omitted
-then the current directory is checked for the existance of an
+then the current directory is checked for the existence of an
 application.ini file. If not found, then it is assumed the target
 application is a remote Firefox OS instance.
 
@@ -70,7 +70,7 @@ application is a remote Firefox OS instance.
 
 The path to the sources.xml that accompanies the target application (Firefox OS
 only). If this is omitted then the current directory is checked for the
-existance of a sources.xml file.
+existence of a sources.xml file.
 
 Examples
 ````````
diff --git a/testing/web-platform/tests/tools/wptrunner/README.rst b/testing/web-platform/tests/tools/wptrunner/README.rst
index 780518aae114..6d751a659ea8 100644
--- a/testing/web-platform/tests/tools/wptrunner/README.rst
+++ b/testing/web-platform/tests/tools/wptrunner/README.rst
@@ -197,7 +197,7 @@ language e.g.::
 
   if debug and (platform == "linux" or platform == "osx"): FAIL
 
-For test expectations the avaliable variables are those in the
+For test expectations the available variables are those in the
 `run_info` which for desktop are `version`, `os`, `bits`, `processor`,
 `debug` and `product`.
 
diff --git a/toolkit/components/extensions/docs/events.rst b/toolkit/components/extensions/docs/events.rst
index 479b082a5f9b..6ee4a20d7582 100644
--- a/toolkit/components/extensions/docs/events.rst
+++ b/toolkit/components/extensions/docs/events.rst
@@ -34,7 +34,7 @@ code such as:
 .. code-block:: js
 
    browser.myapi.onSomething.addListener(param1 => {
-     console.log(`Something happend: ${param1}`);
+     console.log(`Something happened: ${param1}`);
    });
 
 Note that the schema syntax looks similar to that for a function,
diff --git a/toolkit/components/extensions/docs/functions.rst b/toolkit/components/extensions/docs/functions.rst
index 1e08740aacfa..aa34b09fb2b6 100644
--- a/toolkit/components/extensions/docs/functions.rst
+++ b/toolkit/components/extensions/docs/functions.rst
@@ -152,7 +152,7 @@ which can be useful while developing a new API.
 Implementing a function in a child process
 ------------------------------------------
 Most functions are implemented in the main process, but there are
-occassionally reasons to implement a function in a child process, such as:
+occasionally reasons to implement a function in a child process, such as:
 
 - The function has one or more parameters of a type that cannot be automatically
   sent to the main process using the structured clone algorithm.
diff --git a/toolkit/components/telemetry/docs/data/first-shutdown-ping.rst b/toolkit/components/telemetry/docs/data/first-shutdown-ping.rst
index 2fd2ce5eb67e..fe20c5df4502 100644
--- a/toolkit/components/telemetry/docs/data/first-shutdown-ping.rst
+++ b/toolkit/components/telemetry/docs/data/first-shutdown-ping.rst
@@ -3,7 +3,7 @@
 =====================
 
 This ping is a duplicate of the main-ping sent on first shutdown. Enabling pingsender
-for first sessions will impact existing engagment metrics. The ``first-shutdown`` ping enables a
+for first sessions will impact existing engagement metrics. The ``first-shutdown`` ping enables a
 more accurate view of users that churn after the first session. This ping exists as a
 stopgap until existing metrics are re-evaluated, allowing us to use the first session
 ``main-pings`` directly.
diff --git a/toolkit/components/telemetry/docs/data/main-ping.rst b/toolkit/components/telemetry/docs/data/main-ping.rst
index 46070b4dd3e0..84a24b5be26c 100644
--- a/toolkit/components/telemetry/docs/data/main-ping.rst
+++ b/toolkit/components/telemetry/docs/data/main-ping.rst
@@ -469,7 +469,7 @@ Structure:
     "gc": {
       "random": [
         {
-          // "completed" or "aborted" if an OOM occured.
+          // "completed" or "aborted" if an OOM occurred.
           "status": "completed",
           // Timestamps are in milliseconds since startup. All the times here
           // are wall-clock times, which may not be monotonically increasing.
diff --git a/toolkit/components/telemetry/docs/data/sync-ping.rst b/toolkit/components/telemetry/docs/data/sync-ping.rst
index 5eadb43b19b7..cac8a10f57b5 100644
--- a/toolkit/components/telemetry/docs/data/sync-ping.rst
+++ b/toolkit/components/telemetry/docs/data/sync-ping.rst
@@ -2,7 +2,7 @@
 "sync" ping
 ===========
 
-This is an aggregated format that contains information about each sync that occurred during a timeframe. It is submitted every 12 hours, and on browser shutdown, but only if the ``syncs`` property would not be empty. The ping does not contain the enviroment block, nor the clientId.
+This is an aggregated format that contains information about each sync that occurred during a timeframe. It is submitted every 12 hours, and on browser shutdown, but only if the ``syncs`` property would not be empty. The ping does not contain the environment block, nor the clientId.
 
 Each item in the ``syncs`` property is generated after a sync is completed, for both successful and failed syncs, and contains measurements pertaining to sync performance and error information.
 
diff --git a/toolkit/crashreporter/docs/index.rst b/toolkit/crashreporter/docs/index.rst
index 0f22bd0df2d0..6f7deee29802 100644
--- a/toolkit/crashreporter/docs/index.rst
+++ b/toolkit/crashreporter/docs/index.rst
@@ -20,7 +20,7 @@ together.
 Breakpad
    Breakpad is a library and set of tools to make collecting process
    information (notably dumps from crashes) easy. Breakpad is a 3rd
-   party project (originaly developed by Google) that is imported into
+   party project (originally developed by Google) that is imported into
    the tree.
 
 Dump files
diff --git a/tools/lint/docs/linters/eslint-plugin-mozilla.rst b/tools/lint/docs/linters/eslint-plugin-mozilla.rst
index ae4712535933..dff52faff1e2 100644
--- a/tools/lint/docs/linters/eslint-plugin-mozilla.rst
+++ b/tools/lint/docs/linters/eslint-plugin-mozilla.rst
@@ -71,8 +71,8 @@ can be used instead.
 balanced-listeners
 ------------------
 
-Checks that for every occurence of 'addEventListener' or 'on' there is an
-occurence of 'removeEventListener' or 'off' with the same event name.
+Checks that for every occurrence of 'addEventListener' or 'on' there is an
+occurrence of 'removeEventListener' or 'off' with the same event name.
 
 import-browser-window-globals
 -----------------------------

From 904937e12f7ec69af53301d2ac368d48a374851c Mon Sep 17 00:00:00 2001
From: Sylvestre Ledru <sledru@mozilla.com>
Date: Fri, 26 Jan 2018 23:58:29 +0100
Subject: [PATCH 29/46] Bug 1433410 - Add codespell support for mach lint
 r=ahal

MozReview-Commit-ID: Ii6QjPMN0Ks

--HG--
extra : rebase_source : f5aec5f35d5f882132fb09fe094dc2cfc41b1392
---
 taskcluster/docker/lint/system-setup.sh |   1 +
 tools/lint/codespell.yml                |  14 +++
 tools/lint/spell/__init__.py            | 117 ++++++++++++++++++++++++
 3 files changed, 132 insertions(+)
 create mode 100644 tools/lint/codespell.yml
 create mode 100644 tools/lint/spell/__init__.py

diff --git a/taskcluster/docker/lint/system-setup.sh b/taskcluster/docker/lint/system-setup.sh
index c32110e96292..53df4d509df0 100644
--- a/taskcluster/docker/lint/system-setup.sh
+++ b/taskcluster/docker/lint/system-setup.sh
@@ -10,6 +10,7 @@ mkdir -p /setup
 cd /setup
 
 apt_packages=()
+apt_packages+=('codespell')
 apt_packages+=('curl')
 apt_packages+=('locales')
 apt_packages+=('git')
diff --git a/tools/lint/codespell.yml b/tools/lint/codespell.yml
new file mode 100644
index 000000000000..c336eda67827
--- /dev/null
+++ b/tools/lint/codespell.yml
@@ -0,0 +1,14 @@
+---
+codespell:
+    description: Check code for common misspellings
+    include: ['.']
+    exclude:
+        - third_party
+    # List of extensions coming from:
+    # tools/lint/{flake8,eslint}.yml
+    # tools/mach_commands.py (clang-format)
+    # + documentation
+    # + localization files
+    extensions: ['js', 'jsm', 'jsx', 'xml', 'html', 'xhtml', 'cpp', 'c', 'h', 'configure', 'py', 'properties', 'rst']
+    type: external
+    payload: spell:lint
diff --git a/tools/lint/spell/__init__.py b/tools/lint/spell/__init__.py
new file mode 100644
index 000000000000..c870511b465a
--- /dev/null
+++ b/tools/lint/spell/__init__.py
@@ -0,0 +1,117 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import os
+import signal
+import which
+import re
+
+# Py3/Py2 compatibility.
+try:
+    from json.decoder import JSONDecodeError
+except ImportError:
+    JSONDecodeError = ValueError
+
+from mozlint import result
+from mozprocess import ProcessHandlerMixin
+
+
+CODESPELL_NOT_FOUND = """
+Unable to locate codespell, please ensure it is installed and in
+your PATH or set the CODESPELL environment variable.
+
+https://github.com/lucasdemarchi/codespell or your system's package manager.
+""".strip()
+
+results = []
+
+CODESPELL_FORMAT_REGEX = re.compile(r'(.*):(.*): (.*) ==> (.*)$')
+
+
+class CodespellProcess(ProcessHandlerMixin):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs['processOutputLine'] = [self.process_line]
+        ProcessHandlerMixin.__init__(self, *args, **kwargs)
+
+    def process_line(self, line):
+        try:
+            match = CODESPELL_FORMAT_REGEX.match(line)
+            abspath, line, typo, correct = match.groups()
+        except AttributeError:
+            print('Unable to match regex against output: {}'.format(line))
+            return
+
+        # Ignore false positive like aParent (which would be fixed to apparent)
+        # See https://github.com/lucasdemarchi/codespell/issues/314
+        m = re.match(r'^[a-z][A-Z][a-z]*', typo)
+        if m:
+            return
+        res = {'path': os.path.relpath(abspath, self.config['root']),
+               'message': typo + " ==> " + correct,
+               'level': "warning",
+               'lineno': line,
+               }
+        results.append(result.from_config(self.config, **res))
+
+    def run(self, *args, **kwargs):
+        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        ProcessHandlerMixin.run(self, *args, **kwargs)
+        signal.signal(signal.SIGINT, orig)
+
+
+def run_process(config, cmd):
+    proc = CodespellProcess(config, cmd)
+    proc.run()
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        proc.kill()
+
+
+def get_codespell_binary():
+    """
+    Returns the path of the first codespell binary available
+    if not found returns None
+    """
+    binary = os.environ.get('CODESPELL')
+    if binary:
+        return binary
+
+    try:
+        return which.which('codespell')
+    except which.WhichError:
+        return None
+
+
+def lint(paths, config, fix=None, **lintargs):
+
+    binary = get_codespell_binary()
+
+    if not binary:
+        print(CODESPELL_NOT_FOUND)
+        if 'MOZ_AUTOMATION' in os.environ:
+            return 1
+        return []
+
+    config['root'] = lintargs['root']
+    cmd_args = [binary,
+                '--disable-colors',
+                # Silence some warnings:
+                # 1: disable warnings about wrong encoding
+                # 2: disable warnings about binary file
+                '--quiet-level=3',
+                ]
+
+# Disabled for now because of
+# https://github.com/lucasdemarchi/codespell/issues/314
+#    if fix:
+#        cmd_args.append('--write-changes')
+
+    base_command = cmd_args + paths
+
+    run_process(config, base_command)
+    return results

From a5428cd701a953f1aecfe7954204c9212c8356c6 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Fri, 26 Jan 2018 11:43:31 +0900
Subject: [PATCH 30/46] Bug 1432392 - Switch the spidermonkey rust bindings job
 to a Debian-based docker image. r=dustin

This job requires cmake, which should be fixed, but in the meanwhile,
create a separate docker image with it installed, based on the image we
use for other spidermonkey builds.

--HG--
extra : rebase_source : da43a7999b6bd86dbba816358d907c902415bed4
---
 taskcluster/ci/docker-image/kind.yml                 |  5 +++++
 taskcluster/ci/spidermonkey/linux.yml                |  2 +-
 .../docker/debian7-mozjs-rust-build/Dockerfile       | 12 ++++++++++++
 3 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 taskcluster/docker/debian7-mozjs-rust-build/Dockerfile

diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml
index 8075aae99b7d..2be19612329e 100644
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -60,6 +60,11 @@ jobs:
       - deb7-valgrind
     args:
       ARCH: i386
+  debian7-mozjs-rust-build:
+    symbol: I(deb7jsrs)
+    parent: debian7-amd64-build
+    packages:
+      - deb7-cmake
   lint:
     symbol: I(lnt)
   debian9-base:
diff --git a/taskcluster/ci/spidermonkey/linux.yml b/taskcluster/ci/spidermonkey/linux.yml
index 4258b0fe92fb..52fc8f002df8 100644
--- a/taskcluster/ci/spidermonkey/linux.yml
+++ b/taskcluster/ci/spidermonkey/linux.yml
@@ -41,7 +41,7 @@ sm-rust-bindings-linux64/debug:
         tier: 2
         platform: linux64/debug
     worker:
-        docker-image: {in-tree: desktop-build}
+        docker-image: {in-tree: debian7-mozjs-rust-build}
     run:
         using: spidermonkey-rust-bindings
         spidermonkey-variant: plain
diff --git a/taskcluster/docker/debian7-mozjs-rust-build/Dockerfile b/taskcluster/docker/debian7-mozjs-rust-build/Dockerfile
new file mode 100644
index 000000000000..2fda25c03bcf
--- /dev/null
+++ b/taskcluster/docker/debian7-mozjs-rust-build/Dockerfile
@@ -0,0 +1,12 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/workspace
+VOLUME /builds/worker/tooltool-cache
+
+# %ARG DOCKER_IMAGE_PACKAGES
+RUN /usr/local/sbin/setup_packages.sh $DOCKER_IMAGE_PACKAGES && \
+    apt-get update && \
+    apt-get install cmake

From b42986e45ece8056fce592a88bfa5ad788d4cab6 Mon Sep 17 00:00:00 2001
From: Chris Manchester <cmanchester@mozilla.com>
Date: Fri, 26 Jan 2018 14:53:44 -0800
Subject: [PATCH 31/46] Bug 1429947 - Unset WIN64_LINK and WIN64_LIB in in-tree
 mozconfigs during an artifact build. r=nalexander

MozReview-Commit-ID: 9mIaxs0xroN

--HG--
extra : rebase_source : 623f50f4186722d6a0da9f01bd6d57b072068bd7
---
 build/mozconfig.no-compile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/build/mozconfig.no-compile b/build/mozconfig.no-compile
index aab7e2d6b830..08762e507ab9 100644
--- a/build/mozconfig.no-compile
+++ b/build/mozconfig.no-compile
@@ -17,5 +17,7 @@ unset MAKECAB
 unset TOOLCHAIN_PREFIX
 unset BINDGEN_CFLAGS
 unset LLVM_CONFIG
+unset WIN64_LINK
+unset WIN64_LIB
 
 unset MOZ_STDCXX_COMPAT

From 4615d385bd407564e8b0c873a4e9ea63784824f5 Mon Sep 17 00:00:00 2001
From: shindli <shindli@mozilla.com>
Date: Sat, 27 Jan 2018 01:20:16 +0200
Subject: [PATCH 32/46] Backed out changeset e414096f1c3b (bug 1433410) for
 yaml lint  failures in  tools/lint/codespell.yml:12:81 on a CLOSED TREE

---
 taskcluster/docker/lint/system-setup.sh |   1 -
 tools/lint/codespell.yml                |  14 ---
 tools/lint/spell/__init__.py            | 117 ------------------------
 3 files changed, 132 deletions(-)
 delete mode 100644 tools/lint/codespell.yml
 delete mode 100644 tools/lint/spell/__init__.py

diff --git a/taskcluster/docker/lint/system-setup.sh b/taskcluster/docker/lint/system-setup.sh
index 53df4d509df0..c32110e96292 100644
--- a/taskcluster/docker/lint/system-setup.sh
+++ b/taskcluster/docker/lint/system-setup.sh
@@ -10,7 +10,6 @@ mkdir -p /setup
 cd /setup
 
 apt_packages=()
-apt_packages+=('codespell')
 apt_packages+=('curl')
 apt_packages+=('locales')
 apt_packages+=('git')
diff --git a/tools/lint/codespell.yml b/tools/lint/codespell.yml
deleted file mode 100644
index c336eda67827..000000000000
--- a/tools/lint/codespell.yml
+++ /dev/null
@@ -1,14 +0,0 @@
----
-codespell:
-    description: Check code for common misspellings
-    include: ['.']
-    exclude:
-        - third_party
-    # List of extensions coming from:
-    # tools/lint/{flake8,eslint}.yml
-    # tools/mach_commands.py (clang-format)
-    # + documentation
-    # + localization files
-    extensions: ['js', 'jsm', 'jsx', 'xml', 'html', 'xhtml', 'cpp', 'c', 'h', 'configure', 'py', 'properties', 'rst']
-    type: external
-    payload: spell:lint
diff --git a/tools/lint/spell/__init__.py b/tools/lint/spell/__init__.py
deleted file mode 100644
index c870511b465a..000000000000
--- a/tools/lint/spell/__init__.py
+++ /dev/null
@@ -1,117 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function
-
-import os
-import signal
-import which
-import re
-
-# Py3/Py2 compatibility.
-try:
-    from json.decoder import JSONDecodeError
-except ImportError:
-    JSONDecodeError = ValueError
-
-from mozlint import result
-from mozprocess import ProcessHandlerMixin
-
-
-CODESPELL_NOT_FOUND = """
-Unable to locate codespell, please ensure it is installed and in
-your PATH or set the CODESPELL environment variable.
-
-https://github.com/lucasdemarchi/codespell or your system's package manager.
-""".strip()
-
-results = []
-
-CODESPELL_FORMAT_REGEX = re.compile(r'(.*):(.*): (.*) ==> (.*)$')
-
-
-class CodespellProcess(ProcessHandlerMixin):
-    def __init__(self, config, *args, **kwargs):
-        self.config = config
-        kwargs['processOutputLine'] = [self.process_line]
-        ProcessHandlerMixin.__init__(self, *args, **kwargs)
-
-    def process_line(self, line):
-        try:
-            match = CODESPELL_FORMAT_REGEX.match(line)
-            abspath, line, typo, correct = match.groups()
-        except AttributeError:
-            print('Unable to match regex against output: {}'.format(line))
-            return
-
-        # Ignore false positive like aParent (which would be fixed to apparent)
-        # See https://github.com/lucasdemarchi/codespell/issues/314
-        m = re.match(r'^[a-z][A-Z][a-z]*', typo)
-        if m:
-            return
-        res = {'path': os.path.relpath(abspath, self.config['root']),
-               'message': typo + " ==> " + correct,
-               'level': "warning",
-               'lineno': line,
-               }
-        results.append(result.from_config(self.config, **res))
-
-    def run(self, *args, **kwargs):
-        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
-        ProcessHandlerMixin.run(self, *args, **kwargs)
-        signal.signal(signal.SIGINT, orig)
-
-
-def run_process(config, cmd):
-    proc = CodespellProcess(config, cmd)
-    proc.run()
-    try:
-        proc.wait()
-    except KeyboardInterrupt:
-        proc.kill()
-
-
-def get_codespell_binary():
-    """
-    Returns the path of the first codespell binary available
-    if not found returns None
-    """
-    binary = os.environ.get('CODESPELL')
-    if binary:
-        return binary
-
-    try:
-        return which.which('codespell')
-    except which.WhichError:
-        return None
-
-
-def lint(paths, config, fix=None, **lintargs):
-
-    binary = get_codespell_binary()
-
-    if not binary:
-        print(CODESPELL_NOT_FOUND)
-        if 'MOZ_AUTOMATION' in os.environ:
-            return 1
-        return []
-
-    config['root'] = lintargs['root']
-    cmd_args = [binary,
-                '--disable-colors',
-                # Silence some warnings:
-                # 1: disable warnings about wrong encoding
-                # 2: disable warnings about binary file
-                '--quiet-level=3',
-                ]
-
-# Disabled for now because of
-# https://github.com/lucasdemarchi/codespell/issues/314
-#    if fix:
-#        cmd_args.append('--write-changes')
-
-    base_command = cmd_args + paths
-
-    run_process(config, base_command)
-    return results

From ac49bf6b8591369a6436adc85efdce708ad7abaa Mon Sep 17 00:00:00 2001
From: Doug Thayer <dothayer@mozilla.com>
Date: Tue, 23 Jan 2018 10:06:31 -0800
Subject: [PATCH 33/46] Bug 1426721 - Add async/bulk encryption interface to
 SDR r=keeler,MattN

Since encryption can be somewhat CPU intensive, if we're encrypting
a large number of strings we want to be able to do so in a background
thread. This will be consumed by the profile migrators when importing
logins.

MozReview-Commit-ID: JoJGOgMzZ4u

--HG--
extra : rebase_source : 4677482b4e9b1df7c7ca70a0e817204ef6638cdf
---
 security/manager/ssl/SecretDecoderRing.cpp    | 80 ++++++++++++++++++-
 security/manager/ssl/SecretDecoderRing.h      |  2 +-
 security/manager/ssl/nsISecretDecoderRing.idl | 14 ++++
 security/manager/ssl/tests/unit/test_sdr.js   | 42 +++++++++-
 toolkit/components/passwordmgr/crypto-SDR.js  | 45 +++++++++++
 .../passwordmgr/nsILoginManagerCrypto.idl     | 11 +++
 6 files changed, 190 insertions(+), 4 deletions(-)

diff --git a/security/manager/ssl/SecretDecoderRing.cpp b/security/manager/ssl/SecretDecoderRing.cpp
index 927c821b4723..fed9ace4d62a 100644
--- a/security/manager/ssl/SecretDecoderRing.cpp
+++ b/security/manager/ssl/SecretDecoderRing.cpp
@@ -10,6 +10,8 @@
 #include "mozilla/Base64.h"
 #include "mozilla/Casting.h"
 #include "mozilla/Services.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/dom/Promise.h"
 #include "nsCOMPtr.h"
 #include "nsIInterfaceRequestor.h"
 #include "nsIInterfaceRequestorUtils.h"
@@ -23,10 +25,41 @@
 #include "ssl.h" // For SSL_ClearSessionCache
 
 using namespace mozilla;
+using dom::Promise;
 
-// NOTE: Should these be the thread-safe versions?
 NS_IMPL_ISUPPORTS(SecretDecoderRing, nsISecretDecoderRing)
 
+void BackgroundSdrEncryptStrings(const nsTArray<nsCString>& plaintexts,
+                                 RefPtr<Promise>& aPromise) {
+  nsCOMPtr<nsISecretDecoderRing> sdrService =
+    do_GetService(NS_SECRETDECODERRING_CONTRACTID);
+  InfallibleTArray<nsString> cipherTexts(plaintexts.Length());
+
+  nsresult rv = NS_ERROR_FAILURE;
+  for (uint32_t i = 0; i < plaintexts.Length(); ++i) {
+    const nsCString& plaintext = plaintexts[i];
+    nsCString cipherText;
+    rv = sdrService->EncryptString(plaintext, cipherText);
+
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      break;
+    }
+
+    cipherTexts.AppendElement(NS_ConvertASCIItoUTF16(cipherText));
+  }
+
+  nsCOMPtr<nsIRunnable> runnable(
+    NS_NewRunnableFunction("BackgroundSdrEncryptStringsResolve",
+                           [rv, aPromise = Move(aPromise), cipherTexts = Move(cipherTexts)]() {
+                             if (NS_FAILED(rv)) {
+                               aPromise->MaybeReject(rv);
+                             } else {
+                               aPromise->MaybeResolve(cipherTexts);
+                             }
+                           }));
+  NS_DispatchToMainThread(runnable);
+}
+
 SecretDecoderRing::SecretDecoderRing()
 {
 }
@@ -132,6 +165,51 @@ SecretDecoderRing::EncryptString(const nsACString& text,
   return NS_OK;
 }
 
+NS_IMETHODIMP
+SecretDecoderRing::AsyncEncryptStrings(uint32_t plaintextsCount,
+                                       const char16_t** plaintexts,
+                                       JSContext* aCx,
+                                       nsISupports** aPromise) {
+  MOZ_RELEASE_ASSERT(NS_IsMainThread());
+  NS_ENSURE_ARG(plaintextsCount);
+  NS_ENSURE_ARG_POINTER(plaintexts);
+  NS_ENSURE_ARG_POINTER(aCx);
+
+  nsIGlobalObject* globalObject =
+    xpc::NativeGlobal(JS::CurrentGlobalOrNull(aCx));
+
+  if (NS_WARN_IF(!globalObject)) {
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  ErrorResult result;
+  RefPtr<Promise> promise = Promise::Create(globalObject, result);
+  if (NS_WARN_IF(result.Failed())) {
+    return result.StealNSResult();
+  }
+
+  InfallibleTArray<nsCString> plaintextsUtf8(plaintextsCount);
+  for (uint32_t i = 0; i < plaintextsCount; ++i) {
+    plaintextsUtf8.AppendElement(NS_ConvertUTF16toUTF8(plaintexts[i]));
+  }
+  nsCOMPtr<nsIRunnable> runnable(
+    NS_NewRunnableFunction("BackgroundSdrEncryptStrings",
+      [promise, plaintextsUtf8 = Move(plaintextsUtf8)]() mutable {
+        BackgroundSdrEncryptStrings(plaintextsUtf8, promise);
+      }));
+
+  nsCOMPtr<nsIThread> encryptionThread;
+  nsresult rv = NS_NewNamedThread("AsyncSDRThread",
+                                  getter_AddRefs(encryptionThread),
+                                  runnable);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  promise.forget(aPromise);
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 SecretDecoderRing::DecryptString(const nsACString& encryptedBase64Text,
                          /*out*/ nsACString& decryptedText)
diff --git a/security/manager/ssl/SecretDecoderRing.h b/security/manager/ssl/SecretDecoderRing.h
index 3bdcbe5c9ec5..2ebb3c75eb3d 100644
--- a/security/manager/ssl/SecretDecoderRing.h
+++ b/security/manager/ssl/SecretDecoderRing.h
@@ -20,7 +20,7 @@ class SecretDecoderRing : public nsISecretDecoderRing
                         , public nsNSSShutDownObject
 {
 public:
-  NS_DECL_ISUPPORTS
+  NS_DECL_THREADSAFE_ISUPPORTS
   NS_DECL_NSISECRETDECODERRING
 
   SecretDecoderRing();
diff --git a/security/manager/ssl/nsISecretDecoderRing.idl b/security/manager/ssl/nsISecretDecoderRing.idl
index d54b94a84b8d..a3b8d9c54455 100644
--- a/security/manager/ssl/nsISecretDecoderRing.idl
+++ b/security/manager/ssl/nsISecretDecoderRing.idl
@@ -22,6 +22,20 @@ interface nsISecretDecoderRing: nsISupports {
   [must_use]
   ACString encryptString(in ACString text);
 
+  /**
+   * Run encryptString on multiple strings, asynchronously. This will allow you
+   * to not jank the browser if you need to encrypt a large number of strings
+   * all at once. This method accepts an array of wstrings which it will convert
+   * to UTF-8 internally before encrypting.
+   *
+   * @param plaintextsCount the number of strings to encrypt.
+   * @param plaintexts the strings to encrypt.
+   * @return A promise for the list of encrypted strings, encoded as Base64.
+   */
+  [implicit_jscontext, must_use]
+  nsISupports asyncEncryptStrings(in unsigned long plaintextsCount,
+                                  [array, size_is(plaintextsCount)] in wstring plaintexts);
+
   /**
    * Decrypt Base64 input.
    * See the encryptString() documentation - this method has basically the same
diff --git a/security/manager/ssl/tests/unit/test_sdr.js b/security/manager/ssl/tests/unit/test_sdr.js
index 8a41ec2d5354..fe1aec2675a6 100644
--- a/security/manager/ssl/tests/unit/test_sdr.js
+++ b/security/manager/ssl/tests/unit/test_sdr.js
@@ -21,7 +21,7 @@ const gTokenPasswordDialogs = {
   QueryInterface: XPCOMUtils.generateQI([Ci.nsITokenPasswordDialogs])
 };
 
-function run_test() {
+add_task(function testEncryptString() {
   let sdr = Cc["@mozilla.org/security/sdr;1"]
               .getService(Ci.nsISecretDecoderRing);
 
@@ -78,4 +78,42 @@ function run_test() {
     equal(gSetPasswordShownCount, 1,
           "changePassword() dialog should have been shown exactly once");
   }
-}
+});
+
+add_task(async function testAsyncEncryptStrings() {
+  let sdr = Cc["@mozilla.org/security/sdr;1"]
+              .getService(Ci.nsISecretDecoderRing);
+
+  // Test valid inputs for encryptString() and decryptString().
+  let inputs = [
+    "",
+    " ", // First printable latin1 character (code point 32).
+    "foo",
+    "1234567890`~!@#$%^&*()-_=+{[}]|\\:;'\",<.>/?",
+    "¡äöüÿ", // Misc + last printable latin1 character (code point 255).
+    "aaa 一二三", // Includes Unicode with code points outside [0, 255].
+  ];
+
+  let encrypteds = await sdr.asyncEncryptStrings(inputs.length, inputs);
+  for (let i = 0; i < inputs.length; i++) {
+    let encrypted = encrypteds[i];
+    let input = inputs[i];
+    let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                      .createInstance(Ci.nsIScriptableUnicodeConverter);
+    converter.charset = "UTF-8";
+
+    let convertedInput = converter.ConvertFromUnicode(input);
+    convertedInput += converter.Finish();
+    notEqual(convertedInput, encrypted,
+             "Encrypted input should not just be the input itself");
+
+    try {
+      atob(encrypted);
+    } catch (e) {
+      ok(false, `encryptString() should have returned Base64: ${e}`);
+    }
+
+    equal(convertedInput, sdr.decryptString(encrypted),
+          "decryptString(encryptString(input)) should return input");
+  }
+});
diff --git a/toolkit/components/passwordmgr/crypto-SDR.js b/toolkit/components/passwordmgr/crypto-SDR.js
index 8705c3efcebe..b82905731d7b 100644
--- a/toolkit/components/passwordmgr/crypto-SDR.js
+++ b/toolkit/components/passwordmgr/crypto-SDR.js
@@ -99,6 +99,51 @@ LoginManagerCrypto_SDR.prototype = {
   },
 
 
+  /*
+   * encryptMany
+   *
+   * Encrypts the specified strings, using the SecretDecoderRing.
+   *
+   * Returns a promise which resolves with the the encrypted strings,
+   * or throws/rejects with an error if there was a problem.
+   */
+  async encryptMany(plaintexts) {
+    if (!Array.isArray(plaintexts) || !plaintexts.length) {
+      throw Components.Exception("Need at least one plaintext to encrypt",
+                                 Cr.NS_ERROR_INVALID_ARG);
+    }
+
+    let cipherTexts;
+
+    let wasLoggedIn = this.isLoggedIn;
+    let canceledMP = false;
+
+    this._uiBusy = true;
+    try {
+      cipherTexts = await this._decoderRing.asyncEncryptStrings(plaintexts.length, plaintexts);
+    } catch (e) {
+      this.log("Failed to encrypt strings. (" + e.name + ")");
+      // If the user clicks Cancel, we get NS_ERROR_FAILURE.
+      // (unlike decrypting, which gets NS_ERROR_NOT_AVAILABLE).
+      if (e.result == Cr.NS_ERROR_FAILURE) {
+        canceledMP = true;
+        throw Components.Exception("User canceled master password entry", Cr.NS_ERROR_ABORT);
+      } else {
+        throw Components.Exception("Couldn't encrypt strings", Cr.NS_ERROR_FAILURE);
+      }
+    } finally {
+      this._uiBusy = false;
+      // If we triggered a master password prompt, notify observers.
+      if (!wasLoggedIn && this.isLoggedIn) {
+        this._notifyObservers("passwordmgr-crypto-login");
+      } else if (canceledMP) {
+        this._notifyObservers("passwordmgr-crypto-loginCanceled");
+      }
+    }
+    return cipherTexts;
+  },
+
+
   /*
    * decrypt
    *
diff --git a/toolkit/components/passwordmgr/nsILoginManagerCrypto.idl b/toolkit/components/passwordmgr/nsILoginManagerCrypto.idl
index 8af36a258bce..1dc8a39e3917 100644
--- a/toolkit/components/passwordmgr/nsILoginManagerCrypto.idl
+++ b/toolkit/components/passwordmgr/nsILoginManagerCrypto.idl
@@ -28,6 +28,17 @@ interface nsILoginManagerCrypto : nsISupports {
    */
   AString encrypt(in AString plainText);
 
+  /*
+   * encryptMany
+   *
+   * @param plainTexts
+   *        The strings to be encrypted.
+   *
+   * Encrypts the specified strings, similar to encrypt, but returning a promise
+   * which resolves with the the encrypted strings.
+   */
+  jsval encryptMany(in jsval plainTexts);
+
   /**
    * decrypt
    *

From 57731e3f909567e16a694caa1a0047f16ed3f341 Mon Sep 17 00:00:00 2001
From: Doug Thayer <dothayer@mozilla.com>
Date: Thu, 4 Jan 2018 15:10:54 -0800
Subject: [PATCH 34/46] Bug 1426721 - Encrypt Chrome logins in background
 thread r=keeler,MattN

A significant chunk of migration jank that I observe locally
happens due to login encryption. This patch reduces the locally
observed jank (measured importing 100 logins) from 180ms to 25ms.

Try is green, and as far as I can tell I don't see any thread
safety issues, but I'm not 100% sure on that. I don't see any
red flags inside the SecretDecoderRing::Encrypt implementation.

I only moved Chrome logins over since I wanted to frontload any
potential issues with the whole approach. It shouldn't be too
hard to move the MSMigrationUtils and IEProfileMigrator uses
over though.


MozReview-Commit-ID: 75edUqJlk8x

--HG--
extra : rebase_source : 0a8e16c46e05972fb01c9703b52cdb5755b0b40b
---
 .../migration/ChromeProfileMigrator.js        |  11 +-
 .../components/migration/MigrationUtils.jsm   |  14 ++
 .../components/passwordmgr/LoginHelper.jsm    | 152 ++++++++++++++----
 .../passwordmgr/nsILoginManager.idl           |  14 ++
 .../passwordmgr/nsILoginManagerStorage.idl    |   4 +-
 .../components/passwordmgr/nsLoginManager.js  |  20 +++
 .../components/passwordmgr/storage-json.js    |   6 +-
 .../passwordmgr/storage-mozStorage.js         |   6 +-
 8 files changed, 189 insertions(+), 38 deletions(-)

diff --git a/browser/components/migration/ChromeProfileMigrator.js b/browser/components/migration/ChromeProfileMigrator.js
index 446e740221b8..ed63960adda2 100644
--- a/browser/components/migration/ChromeProfileMigrator.js
+++ b/browser/components/migration/ChromeProfileMigrator.js
@@ -412,7 +412,7 @@ async function GetWindowsPasswordsResource(aProfileFolder) {
         return;
       }
       let crypto = new OSCrypto();
-
+      let logins = [];
       for (let row of rows) {
         try {
           let origin_url = NetUtil.newURI(row.getResultByName("origin_url"));
@@ -454,11 +454,18 @@ async function GetWindowsPasswordsResource(aProfileFolder) {
               throw new Error("Login data scheme type not supported: " +
                               row.getResultByName("scheme"));
           }
-          MigrationUtils.insertLoginWrapper(loginInfo);
+          logins.push(loginInfo);
         } catch (e) {
           Cu.reportError(e);
         }
       }
+      try {
+        if (logins.length > 0) {
+          await MigrationUtils.insertLoginsWrapper(logins);
+        }
+      } catch (e) {
+        Cu.reportError(e);
+      }
       crypto.finalize();
       aCallback(true);
     },
diff --git a/browser/components/migration/MigrationUtils.jsm b/browser/components/migration/MigrationUtils.jsm
index a513515e0a06..bb80b4a2f9dd 100644
--- a/browser/components/migration/MigrationUtils.jsm
+++ b/browser/components/migration/MigrationUtils.jsm
@@ -1058,6 +1058,20 @@ this.MigrationUtils = Object.freeze({
     }
   },
 
+  async insertLoginsWrapper(logins) {
+    this._importQuantities.logins += logins.length;
+    let inserted = await LoginHelper.maybeImportLogins(logins);
+    // Note that this means that if we import a login that has a newer password
+    // than we know about, we will update the login, and an undo of the import
+    // will not revert this. This seems preferable over removing the login
+    // outright or storing the old password in the undo file.
+    if (gKeepUndoData) {
+      for (let {guid, timePasswordChanged} of inserted) {
+        gUndoData.get("logins").push({guid, timePasswordChanged});
+      }
+    }
+  },
+
   initializeUndoData() {
     gKeepUndoData = true;
     gUndoData = new Map([["bookmarks", []], ["visits", []], ["logins", []]]);
diff --git a/toolkit/components/passwordmgr/LoginHelper.jsm b/toolkit/components/passwordmgr/LoginHelper.jsm
index 7c7a76c0b58f..c43091315cdd 100644
--- a/toolkit/components/passwordmgr/LoginHelper.jsm
+++ b/toolkit/components/passwordmgr/LoginHelper.jsm
@@ -210,6 +210,51 @@ this.LoginHelper = {
     return false;
   },
 
+  /**
+   * Helper to check if there are any duplicates of the existing login. If the
+   * duplicates need to have the password updated, this performs that update.
+   *
+   * @param {nsILoginInfo} aLogin - The login to search for.
+   * @return {boolean} - true if duplicates exist, otherwise false.
+   */
+  checkForDuplicatesAndMaybeUpdate(aLogin) {
+    // While here we're passing formSubmitURL and httpRealm, they could be empty/null and get
+    // ignored in that case, leading to multiple logins for the same username.
+    let existingLogins = Services.logins.findLogins({}, aLogin.hostname,
+                                                    aLogin.formSubmitURL,
+                                                    aLogin.httpRealm);
+    // Check for an existing login that matches *including* the password.
+    // If such a login exists, we do not need to add a new login.
+    if (existingLogins.some(l => aLogin.matches(l, false /* ignorePassword */))) {
+      return true;
+    }
+    // Now check for a login with the same username, where it may be that we have an
+    // updated password.
+    let foundMatchingLogin = false;
+    for (let existingLogin of existingLogins) {
+      if (aLogin.username == existingLogin.username) {
+        foundMatchingLogin = true;
+        existingLogin.QueryInterface(Ci.nsILoginMetaInfo);
+        if (aLogin.password != existingLogin.password &
+           aLogin.timePasswordChanged > existingLogin.timePasswordChanged) {
+          // if a login with the same username and different password already exists and it's older
+          // than the current one, update its password and timestamp.
+          let propBag = Cc["@mozilla.org/hash-property-bag;1"].
+                        createInstance(Ci.nsIWritablePropertyBag);
+          propBag.setProperty("password", aLogin.password);
+          propBag.setProperty("timePasswordChanged", aLogin.timePasswordChanged);
+          Services.logins.modifyLogin(existingLogin, propBag);
+        }
+      }
+    }
+    // if the new login is an update or is older than an exiting login, don't add it.
+    if (foundMatchingLogin) {
+      return true;
+    }
+
+    return false;
+  },
+
   doLoginsMatch(aLogin1, aLogin2, {
     ignorePassword = false,
     ignoreSchemes = false,
@@ -571,42 +616,87 @@ this.LoginHelper = {
     login.timeLastUsed = loginData.timeLastUsed || loginData.timeCreated;
     login.timePasswordChanged = loginData.timePasswordChanged || loginData.timeCreated;
     login.timesUsed = loginData.timesUsed || 1;
-    // While here we're passing formSubmitURL and httpRealm, they could be empty/null and get
-    // ignored in that case, leading to multiple logins for the same username.
-    let existingLogins = Services.logins.findLogins({}, login.hostname,
-                                                    login.formSubmitURL,
-                                                    login.httpRealm);
-    // Check for an existing login that matches *including* the password.
-    // If such a login exists, we do not need to add a new login.
-    if (existingLogins.some(l => login.matches(l, false /* ignorePassword */))) {
-      return null;
-    }
-    // Now check for a login with the same username, where it may be that we have an
-    // updated password.
-    let foundMatchingLogin = false;
-    for (let existingLogin of existingLogins) {
-      if (login.username == existingLogin.username) {
-        foundMatchingLogin = true;
-        existingLogin.QueryInterface(Ci.nsILoginMetaInfo);
-        if (login.password != existingLogin.password &
-           login.timePasswordChanged > existingLogin.timePasswordChanged) {
-          // if a login with the same username and different password already exists and it's older
-          // than the current one, update its password and timestamp.
-          let propBag = Cc["@mozilla.org/hash-property-bag;1"].
-                        createInstance(Ci.nsIWritablePropertyBag);
-          propBag.setProperty("password", login.password);
-          propBag.setProperty("timePasswordChanged", login.timePasswordChanged);
-          Services.logins.modifyLogin(existingLogin, propBag);
-        }
-      }
-    }
-    // if the new login is an update or is older than an exiting login, don't add it.
-    if (foundMatchingLogin) {
+    if (this.checkForDuplicatesAndMaybeUpdate(login)) {
       return null;
     }
     return Services.logins.addLogin(login);
   },
 
+  /**
+   * Equivalent to maybeImportLogin, except asynchronous and for multiple logins.
+   * @param {Object[]} loginDatas - For each login, the data that needs to be added.
+   * @returns {nsILoginInfo[]} the newly added logins, filtered if no login was added.
+   */
+  async maybeImportLogins(loginDatas) {
+    let loginsToAdd = [];
+    let loginMap = new Map();
+    for (let loginData of loginDatas) {
+      // create a new login
+      let login = Cc["@mozilla.org/login-manager/loginInfo;1"].createInstance(Ci.nsILoginInfo);
+      login.init(loginData.hostname,
+                 loginData.formSubmitURL || (typeof(loginData.httpRealm) == "string" ? null : ""),
+                 typeof(loginData.httpRealm) == "string" ? loginData.httpRealm : null,
+                 loginData.username,
+                 loginData.password,
+                 loginData.usernameElement || "",
+                 loginData.passwordElement || "");
+
+      login.QueryInterface(Ci.nsILoginMetaInfo);
+      login.timeCreated = loginData.timeCreated;
+      login.timeLastUsed = loginData.timeLastUsed || loginData.timeCreated;
+      login.timePasswordChanged = loginData.timePasswordChanged || loginData.timeCreated;
+      login.timesUsed = loginData.timesUsed || 1;
+
+      try {
+        // Ensure we only send checked logins through, since the validation is optimized
+        // out from the bulk APIs below us.
+        this.checkLoginValues(login);
+      } catch (e) {
+        Cu.reportError(e);
+        continue;
+      }
+
+      // First, we need to check the logins that we've already decided to add, to
+      // see if this is a duplicate. This should mirror the logic inside
+      // checkForDuplicatesAndMaybeUpdate, but only for the array of logins we're
+      // adding.
+      let newLogins = loginMap.get(login.hostname) || [];
+      if (!newLogins) {
+        loginMap.set(login.hostname, newLogins);
+      } else {
+        if (newLogins.some(l => login.matches(l, false /* ignorePassword */))) {
+          continue;
+        }
+        let foundMatchingNewLogin = false;
+        for (let newLogin of newLogins) {
+          if (login.username == newLogin.username) {
+            foundMatchingNewLogin = true;
+            newLogin.QueryInterface(Ci.nsILoginMetaInfo);
+            if (login.password != newLogin.password &
+                login.timePasswordChanged > newLogin.timePasswordChanged) {
+              // if a login with the same username and different password already exists and it's older
+              // than the current one, update its password and timestamp.
+              newLogin.password = login.password;
+              newLogin.timePasswordChanged = login.timePasswordChanged;
+            }
+          }
+        }
+
+        if (foundMatchingNewLogin) {
+          continue;
+        }
+      }
+
+      if (this.checkForDuplicatesAndMaybeUpdate(login)) {
+        continue;
+      }
+
+      newLogins.push(login);
+      loginsToAdd.push(login);
+    }
+    return Services.logins.addLogins(loginsToAdd);
+  },
+
   /**
    * Convert an array of nsILoginInfo to vanilla JS objects suitable for
    * sending over IPC.
diff --git a/toolkit/components/passwordmgr/nsILoginManager.idl b/toolkit/components/passwordmgr/nsILoginManager.idl
index 30b5a0449d70..e2563a12f59f 100644
--- a/toolkit/components/passwordmgr/nsILoginManager.idl
+++ b/toolkit/components/passwordmgr/nsILoginManager.idl
@@ -40,6 +40,20 @@ interface nsILoginManager : nsISupports {
   nsILoginInfo addLogin(in nsILoginInfo aLogin);
 
 
+  /**
+   * Like addLogin, but asynchronous and for many logins.
+   *
+   * @param aLogins
+   *        A JS Array of nsILoginInfos to add.
+   * @return A promise which resolves with a JS Array of cloned logins with
+   *         the guids set.
+   *
+   * Default values for each login's nsILoginMetaInfo properties will be
+   * created. However, if the caller specifies non-default values, they will
+   * be used instead.
+   */
+  jsval addLogins(in jsval aLogins);
+
   /**
    * Remove a login from the login manager.
    *
diff --git a/toolkit/components/passwordmgr/nsILoginManagerStorage.idl b/toolkit/components/passwordmgr/nsILoginManagerStorage.idl
index 4ad3dbfe9a92..28cd45bcd73a 100644
--- a/toolkit/components/passwordmgr/nsILoginManagerStorage.idl
+++ b/toolkit/components/passwordmgr/nsILoginManagerStorage.idl
@@ -50,13 +50,15 @@ interface nsILoginManagerStorage : nsISupports {
    *
    * @param aLogin
    *        The login to be added.
+   * @param aPreEncrypted
+   *        Whether the login was already encrypted or not.
    * @return a clone of the login info with the guid set (even if it was not provided).
    *
    * Default values for the login's nsILoginMetaInfo properties will be
    * created. However, if the caller specifies non-default values, they will
    * be used instead.
    */
-  nsILoginInfo addLogin(in nsILoginInfo aLogin);
+  nsILoginInfo addLogin(in nsILoginInfo aLogin, [optional] in boolean aPreEncrypted);
 
 
   /**
diff --git a/toolkit/components/passwordmgr/nsLoginManager.js b/toolkit/components/passwordmgr/nsLoginManager.js
index 255dbd2a8b9c..b4a6ebb9f4c9 100644
--- a/toolkit/components/passwordmgr/nsLoginManager.js
+++ b/toolkit/components/passwordmgr/nsLoginManager.js
@@ -307,6 +307,26 @@ LoginManager.prototype = {
     return this._storage.addLogin(login);
   },
 
+  async addLogins(logins) {
+    let crypto = Cc["@mozilla.org/login-manager/crypto/SDR;1"].
+                 getService(Ci.nsILoginManagerCrypto);
+    let plaintexts = logins.map(l => l.username).concat(logins.map(l => l.password));
+    let ciphertexts = await crypto.encryptMany(plaintexts);
+    let usernames = ciphertexts.slice(0, logins.length);
+    let passwords = ciphertexts.slice(logins.length);
+    for (let i = 0; i < logins.length; i++) {
+      let plaintextUsername = logins[i].username;
+      let plaintextPassword = logins[i].password;
+      logins[i].username = usernames[i];
+      logins[i].password = passwords[i];
+      log.debug("Adding login");
+      this._storage.addLogin(logins[i], true);
+      // Reset the username and password to keep the same guarantees as addLogin
+      logins[i].username = plaintextUsername;
+      logins[i].password = plaintextPassword;
+    }
+  },
+
   /**
    * Remove the specified login from the stored logins.
    */
diff --git a/toolkit/components/passwordmgr/storage-json.js b/toolkit/components/passwordmgr/storage-json.js
index 292ec7e6f43c..04f8ef326758 100644
--- a/toolkit/components/passwordmgr/storage-json.js
+++ b/toolkit/components/passwordmgr/storage-json.js
@@ -99,13 +99,15 @@ this.LoginManagerStorage_json.prototype = {
     return this._store._save();
   },
 
-  addLogin(login) {
+  addLogin(login, preEncrypted = false) {
     this._store.ensureDataReady();
 
     // Throws if there are bogus values.
     LoginHelper.checkLoginValues(login);
 
-    let [encUsername, encPassword, encType] = this._encryptLogin(login);
+    let [encUsername, encPassword, encType] = preEncrypted ?
+      [login.username, login.password, this._crypto.defaultEncType] :
+      this._encryptLogin(login);
 
     // Clone the login, so we don't modify the caller's object.
     let loginClone = login.clone();
diff --git a/toolkit/components/passwordmgr/storage-mozStorage.js b/toolkit/components/passwordmgr/storage-mozStorage.js
index bf564fea87e2..e3e7422e22d0 100644
--- a/toolkit/components/passwordmgr/storage-mozStorage.js
+++ b/toolkit/components/passwordmgr/storage-mozStorage.js
@@ -196,11 +196,13 @@ LoginManagerStorage_mozStorage.prototype = {
   },
 
 
-  addLogin(login) {
+  addLogin(login, preEncrypted = false) {
     // Throws if there are bogus values.
     LoginHelper.checkLoginValues(login);
 
-    let [encUsername, encPassword, encType] = this._encryptLogin(login);
+    let [encUsername, encPassword, encType] = preEncrypted ?
+      [login.username, login.password, this._crypto.defaultEncType] :
+      this._encryptLogin(login);
 
     // Clone the login, so we don't modify the caller's object.
     let loginClone = login.clone();

From 520f9865db583d188a1471d191f28f38c3cb4b31 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh+mozilla@glandium.org>
Date: Fri, 26 Jan 2018 14:39:07 +0900
Subject: [PATCH 35/46] Bug 1432397 - Switch mingw builds to a Debian
 stretch-based docker image. r=dustin

Don't build ucl when building upx, Debian stretch has a recent enough
version. In fact, the last upstream version doesn't build with GCC in
Debian stretch (http://bugs.debian.org/811707)

--HG--
extra : rebase_source : aae67773b9dd3b99f6ddf9ab7f59a628037e6925
---
 taskcluster/ci/build/windows.yml            |  1 +
 taskcluster/ci/docker-image/kind.yml        |  3 ++
 taskcluster/ci/toolchain/linux.yml          | 10 ++--
 taskcluster/docker/mingw32-build/Dockerfile | 53 +++++++++++++++++++++
 taskcluster/scripts/misc/build-upx.sh       | 13 -----
 5 files changed, 62 insertions(+), 18 deletions(-)
 create mode 100644 taskcluster/docker/mingw32-build/Dockerfile

diff --git a/taskcluster/ci/build/windows.yml b/taskcluster/ci/build/windows.yml
index 4b49fbb9d715..790159cfefb2 100755
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -650,6 +650,7 @@ win32-mingw32/debug:
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
+        docker-image: {in-tree: mingw32-build}
         max-run-time: 7200
     run:
         using: mozharness
diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml
index 2be19612329e..346cae704b62 100644
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -79,6 +79,9 @@ jobs:
   android-build:
     symbol: I(agb)
     parent: debian9-base
+  mingw32-build:
+    symbol: I(mingw)
+    parent: debian9-base
   index-task:
     symbol: I(idx)
   funsize-update-generator:
diff --git a/taskcluster/ci/toolchain/linux.yml b/taskcluster/ci/toolchain/linux.yml
index 3008858de4ba..39d9e59d061b 100755
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -482,7 +482,7 @@ linux64-upx:
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
-        docker-image: {in-tree: desktop-build}
+        docker-image: {in-tree: mingw32-build}
         max-run-time: 3600
     run:
         using: toolchain-script
@@ -498,7 +498,7 @@ linux64-wine:
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
-        docker-image: {in-tree: desktop-build}
+        docker-image: {in-tree: mingw32-build}
         max-run-time: 10800
     run:
         using: toolchain-script
@@ -514,7 +514,7 @@ linux64-mingw32-gcc:
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
-        docker-image: {in-tree: desktop-build}
+        docker-image: {in-tree: mingw32-build}
         max-run-time: 10800
     run:
         using: toolchain-script
@@ -532,7 +532,7 @@ linux64-mingw32-nsis:
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
-        docker-image: {in-tree: desktop-build}
+        docker-image: {in-tree: mingw32-build}
         max-run-time: 3600
     run:
         using: toolchain-script
@@ -550,7 +550,7 @@ linux64-mingw32-fxc2:
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
-        docker-image: {in-tree: desktop-build}
+        docker-image: {in-tree: mingw32-build}
         max-run-time: 1800
     run:
         using: toolchain-script
diff --git a/taskcluster/docker/mingw32-build/Dockerfile b/taskcluster/docker/mingw32-build/Dockerfile
new file mode 100644
index 000000000000..e5924c996d12
--- /dev/null
+++ b/taskcluster/docker/mingw32-build/Dockerfile
@@ -0,0 +1,53 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/workspace
+VOLUME /builds/worker/tooltool-cache
+
+ENV XZ_OPT=-T0
+
+RUN dpkg --add-architecture i386
+
+# Ideally, we wouldn't need gcc-multilib and the extra linux-libc-dev,
+# but the latter is required to make the former installable, and the former
+# because of bug 1409276.
+RUN apt-get update && \
+    apt-get install \
+      autoconf2.13 \
+      bison \
+      bzip2 \
+      flex \
+      curl \
+      file \
+      gawk \
+      g++-multilib \
+      gnupg \
+      libucl-dev \
+      p7zip-full \
+      scons \
+      tar \
+      unzip \
+      uuid \
+      wget \
+      x11-utils \
+      xvfb \
+      yasm \
+      zip \
+      zlib1g-dev \
+      libfreetype6-dev:i386 \
+      libx11-dev:i386
+
+# Stubbed out credentials; mozharness looks for this file an issues a WARNING
+# if it's not found, which causes the build to fail.  Note that this needs to
+# be in the parent of the workspace directory and in the directory where
+# mozharness is run (not its --work-dir).  See Bug 1169652.
+# %include taskcluster/docker/desktop-build/oauth.txt
+COPY topsrcdir/taskcluster/docker/desktop-build/oauth.txt /builds/worker/
+
+# stubbed out buildprops, which keeps mozharness from choking
+# Note that this needs to be in the parent of the workspace directory and in
+# the directory where mozharness is run (not its --work-dir)
+# %include taskcluster/docker/desktop-build/buildprops.json
+COPY topsrcdir/taskcluster/docker/desktop-build/buildprops.json /builds/worker/
diff --git a/taskcluster/scripts/misc/build-upx.sh b/taskcluster/scripts/misc/build-upx.sh
index 85135aa6506b..9a3b7b15f929 100755
--- a/taskcluster/scripts/misc/build-upx.sh
+++ b/taskcluster/scripts/misc/build-upx.sh
@@ -10,23 +10,10 @@ mkdir -p $INSTALL_DIR/bin
 
 cd $WORKSPACE
 
-# --------------
-
-wget --progress=dot:mega http://www.oberhumer.com/opensource/ucl/download/ucl-1.03.tar.gz
-echo "5847003d136fbbca1334dd5de10554c76c755f7c  ucl-1.03.tar.gz" | sha1sum -c -
-tar xf ucl-1.03.tar.gz
-cd ucl-1.03
-./configure
-make -j$(nproc)
-
-# --------------
-
-cd ..
 git clone -n https://github.com/upx/upx.git upx-clone
 cd upx-clone
 git checkout d31947e1f016e87f24f88b944439bbee892f0429 # Asserts integrity of the clone (right?)
 git submodule update --init --recursive
-export UPX_UCLDIR=$WORKSPACE/ucl-1.03
 cd src
 make -j$(nproc)
 cp upx.out $INSTALL_DIR/bin/upx

From 9c6223e5a077d8ba65ecd955d9486a270111faf2 Mon Sep 17 00:00:00 2001
From: Kyle Machulis <kyle@nonpolynomial.com>
Date: Mon, 22 Jan 2018 12:50:11 -0800
Subject: [PATCH 36/46] Bug 1432271 - Remove dom/time; r=baku

dom/time contained the TimeService and TimeManager classes, used for
setting time via Gecko on FirefoxOS. Since FirefoxOS is no longer in
the code base, the directory can be removed.

MozReview-Commit-ID: 8PEk3e6HA67

--HG--
extra : rebase_source : 63a0a6c665792ab1885bd4f81261db9be887ffd1
---
 dom/base/Navigator.cpp           |  23 -----
 dom/base/Navigator.h             |  11 +-
 dom/base/moz.build               |   1 -
 dom/base/nsGlobalWindowInner.cpp |  13 ---
 dom/base/nsGlobalWindowInner.h   |   3 -
 dom/base/nsGlobalWindowOuter.cpp |   1 -
 dom/base/nsPIDOMWindow.h         |   3 -
 dom/bindings/Bindings.conf       |   4 -
 dom/ipc/ContentChild.cpp         |   4 -
 dom/ipc/ContentParent.cpp        |   3 -
 dom/moz.build                    |   1 -
 dom/time/DateCacheCleaner.cpp    |  54 ----------
 dom/time/DateCacheCleaner.h      |  19 ----
 dom/time/TimeChangeObserver.cpp  | 166 -------------------------------
 dom/time/TimeChangeObserver.h    |  23 -----
 dom/time/TimeManager.cpp         |  51 ----------
 dom/time/TimeManager.h           |  62 ------------
 dom/time/TimeService.cpp         |  40 --------
 dom/time/TimeService.h           |  38 -------
 dom/time/moz.build               |  32 ------
 dom/time/nsITimeService.idl      |  20 ----
 dom/webidl/MozTimeManager.webidl |  18 ----
 dom/webidl/Navigator.webidl      |   8 --
 dom/webidl/moz.build             |   6 +-
 layout/build/nsLayoutModule.cpp  |   7 --
 layout/build/nsLayoutStatics.cpp |   1 -
 26 files changed, 3 insertions(+), 609 deletions(-)
 delete mode 100644 dom/time/DateCacheCleaner.cpp
 delete mode 100644 dom/time/DateCacheCleaner.h
 delete mode 100644 dom/time/TimeChangeObserver.cpp
 delete mode 100644 dom/time/TimeChangeObserver.h
 delete mode 100644 dom/time/TimeManager.cpp
 delete mode 100644 dom/time/TimeManager.h
 delete mode 100644 dom/time/TimeService.cpp
 delete mode 100644 dom/time/TimeService.h
 delete mode 100644 dom/time/moz.build
 delete mode 100644 dom/time/nsITimeService.idl
 delete mode 100644 dom/webidl/MozTimeManager.webidl

diff --git a/dom/base/Navigator.cpp b/dom/base/Navigator.cpp
index d81ea94fe333..bc5d4bb4ace3 100644
--- a/dom/base/Navigator.cpp
+++ b/dom/base/Navigator.cpp
@@ -62,7 +62,6 @@
 #include "nsIStringStream.h"
 #include "nsIHttpChannel.h"
 #include "nsIHttpChannelInternal.h"
-#include "TimeManager.h"
 #include "nsStreamUtils.h"
 #include "WidgetUtils.h"
 #include "nsIPresentationService.h"
@@ -198,7 +197,6 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(Navigator)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mStorageManager)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCredentials)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaDevices)
-  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTimeManager)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mServiceWorkerContainer)
 
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow)
@@ -248,10 +246,6 @@ Navigator::Invalidate()
 
   mMediaDevices = nullptr;
 
-  if (mTimeManager) {
-    mTimeManager = nullptr;
-  }
-
   if (mPresentation) {
     mPresentation = nullptr;
   }
@@ -1502,23 +1496,6 @@ Navigator::GetConnection(ErrorResult& aRv)
   return mConnection;
 }
 
-#ifdef MOZ_TIME_MANAGER
-time::TimeManager*
-Navigator::GetMozTime(ErrorResult& aRv)
-{
-  if (!mWindow) {
-    aRv.Throw(NS_ERROR_UNEXPECTED);
-    return nullptr;
-  }
-
-  if (!mTimeManager) {
-    mTimeManager = new time::TimeManager(mWindow);
-  }
-
-  return mTimeManager;
-}
-#endif
-
 already_AddRefed<ServiceWorkerContainer>
 Navigator::ServiceWorker()
 {
diff --git a/dom/base/Navigator.h b/dom/base/Navigator.h
index 51f429cd2d70..89c43192a754 100644
--- a/dom/base/Navigator.h
+++ b/dom/base/Navigator.h
@@ -75,11 +75,8 @@ class VRDisplay;
 class VRServiceTest;
 class StorageManager;
 
-namespace time {
-class TimeManager;
-} // namespace time
-
-class Navigator final : public nsISupports
+class Navigator final : public nsIDOMNavigator
+                      , public nsIMozNavigatorNetwork
                       , public nsWrapperCache
 {
 public:
@@ -182,9 +179,6 @@ public:
   bool IsWebVRContentPresenting() const;
   void RequestVRPresentation(VRDisplay& aDisplay);
   nsINetworkProperties* GetNetworkProperties();
-#ifdef MOZ_TIME_MANAGER
-  time::TimeManager* GetMozTime(ErrorResult& aRv);
-#endif // MOZ_TIME_MANAGER
 
   Presentation* GetPresentation(ErrorResult& aRv);
 
@@ -271,7 +265,6 @@ private:
   RefPtr<network::Connection> mConnection;
   RefPtr<CredentialsContainer> mCredentials;
   RefPtr<MediaDevices> mMediaDevices;
-  RefPtr<time::TimeManager> mTimeManager;
   RefPtr<ServiceWorkerContainer> mServiceWorkerContainer;
   nsCOMPtr<nsPIDOMWindowInner> mWindow;
   RefPtr<Presentation> mPresentation;
diff --git a/dom/base/moz.build b/dom/base/moz.build
index bf36fd6699da..833dac114387 100644
--- a/dom/base/moz.build
+++ b/dom/base/moz.build
@@ -426,7 +426,6 @@ LOCAL_INCLUDES += [
     '../events',
     '../media',
     '../network',
-    '../time',
     '/caps',
     '/docshell/base',
     '/dom/base',
diff --git a/dom/base/nsGlobalWindowInner.cpp b/dom/base/nsGlobalWindowInner.cpp
index d307190b41e6..9d3135f92453 100644
--- a/dom/base/nsGlobalWindowInner.cpp
+++ b/dom/base/nsGlobalWindowInner.cpp
@@ -228,7 +228,6 @@
 #include "mozilla/DOMEventTargetHelper.h"
 #include "prrng.h"
 #include "nsSandboxFlags.h"
-#include "TimeChangeObserver.h"
 #include "mozilla/dom/AudioContext.h"
 #include "mozilla/dom/BrowserElementDictionariesBinding.h"
 #include "mozilla/dom/cache/CacheStorage.h"
@@ -6872,18 +6871,6 @@ nsGlobalWindowInner::IsVRContentPresenting() const
   return false;
 }
 
-void
-nsGlobalWindowInner::EnableTimeChangeNotifications()
-{
-  mozilla::time::AddWindowListener(this);
-}
-
-void
-nsGlobalWindowInner::DisableTimeChangeNotifications()
-{
-  mozilla::time::RemoveWindowListener(this);
-}
-
 void
 nsGlobalWindowInner::AddSizeOfIncludingThis(nsWindowSizes& aWindowSizes) const
 {
diff --git a/dom/base/nsGlobalWindowInner.h b/dom/base/nsGlobalWindowInner.h
index cac0f9f591fd..46545884fceb 100644
--- a/dom/base/nsGlobalWindowInner.h
+++ b/dom/base/nsGlobalWindowInner.h
@@ -489,9 +489,6 @@ public:
   virtual void DisableOrientationChangeListener() override;
 #endif
 
-  virtual void EnableTimeChangeNotifications() override;
-  virtual void DisableTimeChangeNotifications() override;
-
   bool IsClosedOrClosing() {
     return mCleanedUp;
   }
diff --git a/dom/base/nsGlobalWindowOuter.cpp b/dom/base/nsGlobalWindowOuter.cpp
index 347bf61d72a6..4fc5eb212acd 100644
--- a/dom/base/nsGlobalWindowOuter.cpp
+++ b/dom/base/nsGlobalWindowOuter.cpp
@@ -226,7 +226,6 @@
 #include "mozilla/DOMEventTargetHelper.h"
 #include "prrng.h"
 #include "nsSandboxFlags.h"
-#include "TimeChangeObserver.h"
 #include "mozilla/dom/AudioContext.h"
 #include "mozilla/dom/BrowserElementDictionariesBinding.h"
 #include "mozilla/dom/cache/CacheStorage.h"
diff --git a/dom/base/nsPIDOMWindow.h b/dom/base/nsPIDOMWindow.h
index 6e1440965463..3b198a473f16 100644
--- a/dom/base/nsPIDOMWindow.h
+++ b/dom/base/nsPIDOMWindow.h
@@ -558,9 +558,6 @@ public:
   virtual void DisableOrientationChangeListener() = 0;
 #endif
 
-  virtual void EnableTimeChangeNotifications() = 0;
-  virtual void DisableTimeChangeNotifications() = 0;
-
   /**
    * Tell this window that there is an observer for gamepad input
    *
diff --git a/dom/bindings/Bindings.conf b/dom/bindings/Bindings.conf
index 2950e5f49d7c..ad2f46fb2baf 100644
--- a/dom/bindings/Bindings.conf
+++ b/dom/bindings/Bindings.conf
@@ -601,10 +601,6 @@ DOMInterfaces = {
     'notflattened': True
 },
 
-'MozTimeManager': {
-    'nativeType': 'mozilla::dom::time::TimeManager',
-},
-
 'MozStorageAsyncStatementParams': {
     'headerFile': 'mozilla/storage/mozStorageAsyncStatementParams.h',
     'nativeType': 'mozilla::storage::AsyncStatementParams',
diff --git a/dom/ipc/ContentChild.cpp b/dom/ipc/ContentChild.cpp
index b320f4bd9110..1a51d1dafa93 100644
--- a/dom/ipc/ContentChild.cpp
+++ b/dom/ipc/ContentChild.cpp
@@ -211,7 +211,6 @@
 #include "nsIPrincipal.h"
 #include "DomainPolicy.h"
 #include "mozilla/dom/ipc/StructuredCloneData.h"
-#include "mozilla/dom/time/DateCacheCleaner.h"
 #include "mozilla/ipc/CrashReporterClient.h"
 #include "mozilla/net/NeckoMessageUtils.h"
 #include "mozilla/widget/PuppetBidiKeyboard.h"
@@ -1222,9 +1221,6 @@ ContentChild::InitXPCOM(const XPCOMInitData& aXPCOMInit,
   nsCOMPtr<nsIURI> ucsURL = DeserializeURI(aXPCOMInit.userContentSheetURL());
   nsLayoutStylesheetCache::SetUserContentCSSURL(ucsURL);
 
-  // This will register cross-process observer.
-  mozilla::dom::time::InitializeDateCacheCleaner();
-
   GfxInfoBase::SetFeatureStatus(aXPCOMInit.gfxFeatureStatus());
 
   DataStorage::SetCachedStorageEntries(aXPCOMInit.dataStorage());
diff --git a/dom/ipc/ContentParent.cpp b/dom/ipc/ContentParent.cpp
index d717e549b5f8..38d6e7278f13 100644
--- a/dom/ipc/ContentParent.cpp
+++ b/dom/ipc/ContentParent.cpp
@@ -57,7 +57,6 @@
 #include "mozilla/dom/PPresentationParent.h"
 #include "mozilla/dom/PushNotifier.h"
 #include "mozilla/dom/quota/QuotaManagerService.h"
-#include "mozilla/dom/time/DateCacheCleaner.h"
 #include "mozilla/dom/URLClassifierParent.h"
 #include "mozilla/embedding/printingui/PrintingParent.h"
 #include "mozilla/extensions/StreamFilterParent.h"
@@ -628,8 +627,6 @@ ContentParent::StartUp()
   // Note: This reporter measures all ContentParents.
   RegisterStrongMemoryReporter(new ContentParentsMemoryReporter());
 
-  mozilla::dom::time::InitializeDateCacheCleaner();
-
   BackgroundChild::Startup();
   ClientManager::Startup();
 
diff --git a/dom/moz.build b/dom/moz.build
index 379013efa8b7..f8614900b0e2 100644
--- a/dom/moz.build
+++ b/dom/moz.build
@@ -74,7 +74,6 @@ DIRS += [
     'security',
     'storage',
     'svg',
-    'time',
     'locales',
     'network',
     'permission',
diff --git a/dom/time/DateCacheCleaner.cpp b/dom/time/DateCacheCleaner.cpp
deleted file mode 100644
index 83d7102b98f4..000000000000
--- a/dom/time/DateCacheCleaner.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "DateCacheCleaner.h"
-
-#include "js/Date.h"
-#include "mozilla/dom/ScriptSettings.h"
-#include "mozilla/ClearOnShutdown.h"
-#include "mozilla/Hal.h"
-#include "mozilla/StaticPtr.h"
-
-using namespace mozilla::hal;
-
-namespace mozilla {
-namespace dom {
-namespace time {
-
-class DateCacheCleaner : public SystemTimezoneChangeObserver
-{
-public:
-  DateCacheCleaner()
-  {
-    RegisterSystemTimezoneChangeObserver(this);
-  }
-
-  ~DateCacheCleaner()
-  {
-    UnregisterSystemTimezoneChangeObserver(this);
-  }
-
-  void Notify(const SystemTimezoneChangeInformation& aSystemTimezoneChangeInfo) override
-  {
-    JS::ResetTimeZone();
-  }
-
-};
-
-StaticAutoPtr<DateCacheCleaner> sDateCacheCleaner;
-
-void
-InitializeDateCacheCleaner()
-{
-  if (!sDateCacheCleaner) {
-    sDateCacheCleaner = new DateCacheCleaner();
-    ClearOnShutdown(&sDateCacheCleaner);
-  }
-}
-
-} // namespace time
-} // namespace dom
-} // namespace mozilla
diff --git a/dom/time/DateCacheCleaner.h b/dom/time/DateCacheCleaner.h
deleted file mode 100644
index 7fb7a0692e6c..000000000000
--- a/dom/time/DateCacheCleaner.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-/*
- * InitializeDateCacheCleaner registers DateCacheCleaner to
- * SystemTimeChangeObserver. When time zone is changed, DateCacheCleaner calls
- * JS::ResetTimeZone to update the time zone information.
- */
-
-namespace mozilla {
-namespace dom {
-namespace time {
-void InitializeDateCacheCleaner();
-} //namespace time
-} //namespace dom
-} //namespace mozilla
diff --git a/dom/time/TimeChangeObserver.cpp b/dom/time/TimeChangeObserver.cpp
deleted file mode 100644
index 510153793901..000000000000
--- a/dom/time/TimeChangeObserver.cpp
+++ /dev/null
@@ -1,166 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "TimeChangeObserver.h"
-#include "mozilla/Hal.h"
-#include "mozilla/Observer.h"
-#include "mozilla/HalTypes.h"
-#include "nsWeakPtr.h"
-#include "nsTObserverArray.h"
-#include "mozilla/ClearOnShutdown.h"
-#include "mozilla/Services.h"
-#include "mozilla/StaticPtr.h"
-#include "nsPIDOMWindow.h"
-#include "nsContentUtils.h"
-#include "nsIObserverService.h"
-#include "nsIDocument.h"
-
-using namespace mozilla;
-using namespace mozilla::hal;
-using namespace mozilla::services;
-
-class nsSystemTimeChangeObserver : public SystemClockChangeObserver,
-                                   public SystemTimezoneChangeObserver
-{
-  typedef nsTObserverArray<nsWeakPtr> ListenerArray;
-public:
-  static nsSystemTimeChangeObserver* GetInstance();
-  virtual ~nsSystemTimeChangeObserver();
-
-  // Implementing hal::SystemClockChangeObserver::Notify()
-  void Notify(const int64_t& aClockDeltaMS) override;
-
-  // Implementing hal::SystemTimezoneChangeObserver::Notify()
-  void Notify(
-    const mozilla::hal::SystemTimezoneChangeInformation& aSystemTimezoneChangeInfo) override;
-
-  nsresult AddWindowListenerImpl(nsPIDOMWindowInner* aWindow);
-  nsresult RemoveWindowListenerImpl(nsPIDOMWindowInner* aWindow);
-
-private:
-  nsSystemTimeChangeObserver() { };
-  ListenerArray mWindowListeners;
-  void FireMozTimeChangeEvent();
-};
-
-StaticAutoPtr<nsSystemTimeChangeObserver> sObserver;
-
-nsSystemTimeChangeObserver* nsSystemTimeChangeObserver::GetInstance()
-{
-  if (!sObserver) {
-    sObserver = new nsSystemTimeChangeObserver();
-    ClearOnShutdown(&sObserver);
-  }
-  return sObserver;
-}
-
-nsSystemTimeChangeObserver::~nsSystemTimeChangeObserver()
-{
-  UnregisterSystemClockChangeObserver(this);
-  UnregisterSystemTimezoneChangeObserver(this);
-}
-
-void
-nsSystemTimeChangeObserver::FireMozTimeChangeEvent()
-{
-  ListenerArray::ForwardIterator iter(mWindowListeners);
-  while (iter.HasMore()) {
-    nsWeakPtr weakWindow = iter.GetNext();
-    nsCOMPtr<nsPIDOMWindowInner> innerWindow = do_QueryReferent(weakWindow);
-    nsCOMPtr<nsPIDOMWindowOuter> outerWindow;
-    nsCOMPtr<nsIDocument> document;
-    if (!innerWindow ||
-        !(document = innerWindow->GetExtantDoc()) ||
-        !(outerWindow = innerWindow->GetOuterWindow())) {
-      mWindowListeners.RemoveElement(weakWindow);
-      continue;
-    }
-
-    nsContentUtils::DispatchTrustedEvent(document, outerWindow,
-      NS_LITERAL_STRING("moztimechange"), /* bubbles = */ true,
-      /* canceable = */ false);
-  }
-}
-
-void
-nsSystemTimeChangeObserver::Notify(const int64_t& aClockDeltaMS)
-{
-  // Notify observers that the system clock has been adjusted.
-  nsCOMPtr<nsIObserverService> observerService = GetObserverService();
-  if (observerService) {
-    nsString dataStr;
-    dataStr.AppendFloat(static_cast<double>(aClockDeltaMS));
-    observerService->NotifyObservers(
-      nullptr, "system-clock-change", dataStr.get());
-  }
-
-  FireMozTimeChangeEvent();
-}
-
-void
-nsSystemTimeChangeObserver::Notify(
-  const SystemTimezoneChangeInformation& aSystemTimezoneChangeInfo)
-{
-  FireMozTimeChangeEvent();
-}
-
-nsresult
-mozilla::time::AddWindowListener(nsPIDOMWindowInner* aWindow)
-{
-  return nsSystemTimeChangeObserver::GetInstance()->AddWindowListenerImpl(aWindow);
-}
-
-nsresult
-nsSystemTimeChangeObserver::AddWindowListenerImpl(nsPIDOMWindowInner* aWindow)
-{
-  if (!aWindow) {
-    return NS_ERROR_ILLEGAL_VALUE;
-  }
-
-  nsWeakPtr windowWeakRef = do_GetWeakReference(aWindow);
-  NS_ASSERTION(windowWeakRef, "nsIDOMWindow implementations shuld support weak ref");
-
-  if (mWindowListeners.IndexOf(windowWeakRef) !=
-      ListenerArray::array_type::NoIndex) {
-    return NS_OK;
-  }
-
-  if (mWindowListeners.IsEmpty()) {
-    RegisterSystemClockChangeObserver(sObserver);
-    RegisterSystemTimezoneChangeObserver(sObserver);
-  }
-
-  mWindowListeners.AppendElement(windowWeakRef);
-  return NS_OK;
-}
-
-nsresult
-mozilla::time::RemoveWindowListener(nsPIDOMWindowInner* aWindow)
-{
-  if (!sObserver) {
-    return NS_OK;
-  }
-
-  return nsSystemTimeChangeObserver::GetInstance()->RemoveWindowListenerImpl(aWindow);
-}
-
-nsresult
-nsSystemTimeChangeObserver::RemoveWindowListenerImpl(nsPIDOMWindowInner* aWindow)
-{
-  if (!aWindow) {
-    return NS_OK;
-  }
-
-  nsWeakPtr windowWeakRef = do_GetWeakReference(aWindow);
-  mWindowListeners.RemoveElement(windowWeakRef);
-
-  if (mWindowListeners.IsEmpty()) {
-    UnregisterSystemClockChangeObserver(sObserver);
-    UnregisterSystemTimezoneChangeObserver(sObserver);
-  }
-
-  return NS_OK;
-}
diff --git a/dom/time/TimeChangeObserver.h b/dom/time/TimeChangeObserver.h
deleted file mode 100644
index 65945cbffb9b..000000000000
--- a/dom/time/TimeChangeObserver.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#ifndef _mozilla_time_change_observer_h_
-#define _mozilla_time_change_observer_h_
-
-#include "nscore.h"
-
-class nsPIDOMWindowInner;
-
-namespace mozilla {
-namespace time {
-
-nsresult AddWindowListener(nsPIDOMWindowInner* aWindow);
-nsresult RemoveWindowListener(nsPIDOMWindowInner* aWindow);
-
-} // namespace time
-} // namespace mozilla
-
-#endif //_mozilla_time_change_observer_h_
diff --git a/dom/time/TimeManager.cpp b/dom/time/TimeManager.cpp
deleted file mode 100644
index 0fb7f162327e..000000000000
--- a/dom/time/TimeManager.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "TimeManager.h"
-
-#include "mozilla/dom/Date.h"
-#include "mozilla/dom/MozTimeManagerBinding.h"
-#include "nsITimeService.h"
-#include "nsServiceManagerUtils.h"
-
-namespace mozilla {
-namespace dom {
-namespace time {
-
-NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TimeManager)
-  NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
-  NS_INTERFACE_MAP_ENTRY(nsISupports)
-NS_INTERFACE_MAP_END
-
-NS_IMPL_CYCLE_COLLECTING_ADDREF(TimeManager)
-NS_IMPL_CYCLE_COLLECTING_RELEASE(TimeManager)
-
-NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(TimeManager, mWindow)
-
-JSObject*
-TimeManager::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto)
-{
-  return MozTimeManagerBinding::Wrap(aCx, this, aGivenProto);
-}
-
-void
-TimeManager::Set(Date& aDate)
-{
-  Set(aDate.ToDouble());
-}
-
-void
-TimeManager::Set(double aTime)
-{
-  nsCOMPtr<nsITimeService> timeService = do_GetService(TIMESERVICE_CONTRACTID);
-  if (timeService) {
-    timeService->Set(aTime);
-  }
-}
-
-} // namespace time
-} // namespace dom
-} // namespace mozilla
diff --git a/dom/time/TimeManager.h b/dom/time/TimeManager.h
deleted file mode 100644
index 8c58342b96ba..000000000000
--- a/dom/time/TimeManager.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#ifndef mozilla_dom_time_TimeManager_h
-#define mozilla_dom_time_TimeManager_h
-
-#include "mozilla/Attributes.h"
-#include "nsISupports.h"
-#include "nsPIDOMWindow.h"
-#include "nsWrapperCache.h"
-
-namespace mozilla {
-namespace dom {
-
-class Date;
-
-namespace time {
-
-class TimeManager final : public nsISupports
-                        , public nsWrapperCache
-{
-public:
-  static bool PrefEnabled(JSContext* aCx, JSObject* aGlobal)
-  {
-#ifdef MOZ_TIME_MANAGER
-    return true;
-#else
-    return false;
-#endif
-  }
-
-  NS_DECL_CYCLE_COLLECTING_ISUPPORTS
-  NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(TimeManager)
-
-  explicit TimeManager(nsPIDOMWindowInner* aWindow)
-    : mWindow(aWindow)
-  {
-  }
-
-  nsPIDOMWindowInner* GetParentObject() const
-  {
-    return mWindow;
-  }
-  virtual JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
-
-  void Set(Date& aDate);
-  void Set(double aTime);
-
-private:
-  ~TimeManager() {}
-
-  nsCOMPtr<nsPIDOMWindowInner> mWindow;
-};
-
-} // namespace time
-} // namespace dom
-} // namespace mozilla
-
-#endif //mozilla_dom_time_TimeManager_h
diff --git a/dom/time/TimeService.cpp b/dom/time/TimeService.cpp
deleted file mode 100644
index edbd97d51296..000000000000
--- a/dom/time/TimeService.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "base/basictypes.h"
-#include "jsapi.h"
-#include "mozilla/ClearOnShutdown.h"
-#include "mozilla/Hal.h"
-#include "TimeService.h"
-
-namespace mozilla {
-namespace dom {
-namespace time {
-
-NS_IMPL_ISUPPORTS(TimeService, nsITimeService)
-
-/* static */ StaticRefPtr<TimeService> TimeService::sSingleton;
-
-/* static */ already_AddRefed<TimeService>
-TimeService::GetInstance()
-{
-  if (!sSingleton) {
-    sSingleton = new TimeService();
-    ClearOnShutdown(&sSingleton);
-  }
-  RefPtr<TimeService> service = sSingleton.get();
-  return service.forget();
-}
-
-NS_IMETHODIMP
-TimeService::Set(int64_t aTimeInMS) {
-  hal::AdjustSystemClock(aTimeInMS - (JS_Now() / PR_USEC_PER_MSEC));
-  return NS_OK;
-}
-
-} // namespace time
-} // namespace dom
-} // namespace mozilla
diff --git a/dom/time/TimeService.h b/dom/time/TimeService.h
deleted file mode 100644
index 4c802cf0580b..000000000000
--- a/dom/time/TimeService.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#ifndef mozilla_dom_time_TimeService_h
-#define mozilla_dom_time_TimeService_h
-
-#include "mozilla/StaticPtr.h"
-#include "nsITimeService.h"
-
-namespace mozilla {
-namespace dom {
-namespace time {
-
-/**
- * This class implements a service which lets us modify the system clock time.
- */
-class TimeService : public nsITimeService
-{
-public:
-  NS_DECL_ISUPPORTS
-  NS_DECL_NSITIMESERVICE
-
-  static already_AddRefed<TimeService> GetInstance();
-
-private:
-  virtual ~TimeService() {};
-
-  static StaticRefPtr<TimeService> sSingleton;
-};
-
-} // namespace time
-} // namespace dom
-} // namespace mozilla
-
-#endif //mozilla_dom_time_TimeService_h
diff --git a/dom/time/moz.build b/dom/time/moz.build
deleted file mode 100644
index 90db7c55b0e9..000000000000
--- a/dom/time/moz.build
+++ /dev/null
@@ -1,32 +0,0 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-with Files("**"):
-    BUG_COMPONENT = ("Core", "DOM: Device Interfaces")
-
-XPIDL_SOURCES += [
-    'nsITimeService.idl',
-]
-
-XPIDL_MODULE = 'dom_time'
-
-EXPORTS.mozilla.dom.time += [
-    'DateCacheCleaner.h',
-    'TimeChangeObserver.h',
-    'TimeManager.h',
-    'TimeService.h',
-]
-
-UNIFIED_SOURCES += [
-    'DateCacheCleaner.cpp',
-    'TimeChangeObserver.cpp',
-    'TimeManager.cpp',
-    'TimeService.cpp',
-]
-
-include('/ipc/chromium/chromium-config.mozbuild')
-
-FINAL_LIBRARY = 'xul'
diff --git a/dom/time/nsITimeService.idl b/dom/time/nsITimeService.idl
deleted file mode 100644
index 2bba628e948e..000000000000
--- a/dom/time/nsITimeService.idl
+++ /dev/null
@@ -1,20 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "nsISupports.idl"
-
-%{C++
-#define NS_TIMESERVICE_CID { 0x80d6f9cc, 0xf16d, 0x40c3, { 0xa5, 0x2e, 0xc4, 0xe6, 0x56, 0xe3, 0x65, 0xb4 } }
-#define TIMESERVICE_CONTRACTID "@mozilla.org/time/timeservice;1"
-%}
-
-[scriptable, builtinclass, uuid(1fc7fde2-0090-11e2-bdd6-0fea4b9f41f8)]
-interface nsITimeService : nsISupports
-{
-  /* Set the system time.
-   *
-   * The |aTimeInMS| argument is the time in milliseconds since the epoch.
-   */
-  void set(in int64_t aTimeInMS);
-};
diff --git a/dom/webidl/MozTimeManager.webidl b/dom/webidl/MozTimeManager.webidl
deleted file mode 100644
index 0308835fb562..000000000000
--- a/dom/webidl/MozTimeManager.webidl
+++ /dev/null
@@ -1,18 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-[Func="mozilla::dom::time::TimeManager::PrefEnabled"]
-interface MozTimeManager {
-  /* Set the system time.
-   *
-   * The |time| argument can be either a Date object or a number.
-   *
-   * - If |time| is a number, it's interpreted as milliseconds
-   *   since the epoch (midnight UTC on January 1, 1970).
-   * - If |time| is a Date object, |set(time)| is equivalent to
-   *   |set(time.getTime())|.
-   */
-  void set(Date time);
-  void set(double time);
-};
diff --git a/dom/webidl/Navigator.webidl b/dom/webidl/Navigator.webidl
index d9af6d027115..3ace32934ae4 100644
--- a/dom/webidl/Navigator.webidl
+++ b/dom/webidl/Navigator.webidl
@@ -234,14 +234,6 @@ partial interface Navigator {
   VRServiceTest requestVRServiceTest();
 };
 
-#ifdef MOZ_TIME_MANAGER
-// nsIDOMMozNavigatorTime
-partial interface Navigator {
-  [Throws, ChromeOnly]
-  readonly attribute MozTimeManager mozTime;
-};
-#endif // MOZ_TIME_MANAGER
-
 callback NavigatorUserMediaSuccessCallback = void (MediaStream stream);
 callback NavigatorUserMediaErrorCallback = void (MediaStreamError error);
 
diff --git a/dom/webidl/moz.build b/dom/webidl/moz.build
index 0ef18251feb8..09efa87f6bf9 100644
--- a/dom/webidl/moz.build
+++ b/dom/webidl/moz.build
@@ -211,9 +211,6 @@ with Files("Media*Track*"):
 with Files("Mouse*"):
     BUG_COMPONENT = ("Core", "DOM: Events")
 
-with Files("MozTimeManager.webidl"):
-    BUG_COMPONENT = ("Core", "DOM: Device Interfaces")
-
 with Files("MutationEvent.webidl"):
     BUG_COMPONENT = ("Core", "DOM: Events")
 
@@ -364,7 +361,6 @@ GENERATED_WEBIDL_FILES = [
 
 PREPROCESSED_WEBIDL_FILES = [
     'ChromeUtils.webidl',
-    'Navigator.webidl',
     'Node.webidl',
     'Window.webidl',
 ]
@@ -690,11 +686,11 @@ WEBIDL_FILES = [
     'MozStorageAsyncStatementParams.webidl',
     'MozStorageStatementParams.webidl',
     'MozStorageStatementRow.webidl',
-    'MozTimeManager.webidl',
     'MutationEvent.webidl',
     'MutationObserver.webidl',
     'NamedNodeMap.webidl',
     'NativeOSFileInternals.webidl',
+    'Navigator.webidl',
     'NetDashboard.webidl',
     'NetworkInformation.webidl',
     'NetworkOptions.webidl',
diff --git a/layout/build/nsLayoutModule.cpp b/layout/build/nsLayoutModule.cpp
index d85877b27930..06332901000a 100644
--- a/layout/build/nsLayoutModule.cpp
+++ b/layout/build/nsLayoutModule.cpp
@@ -163,7 +163,6 @@ static void Shutdown();
 #include "mozilla/net/WebSocketEventService.h"
 
 #include "mozilla/dom/power/PowerManagerService.h"
-#include "mozilla/dom/time/TimeService.h"
 
 #include "nsIPresentationService.h"
 
@@ -185,7 +184,6 @@ using mozilla::dom::quota::QuotaManagerService;
 using mozilla::dom::workers::ServiceWorkerManager;
 using mozilla::dom::workers::WorkerDebuggerManager;
 using mozilla::dom::UDPSocketChild;
-using mozilla::dom::time::TimeService;
 using mozilla::gmp::GeckoMediaPluginService;
 using mozilla::dom::NotificationTelemetryService;
 
@@ -261,8 +259,6 @@ NS_GENERIC_FACTORY_CONSTRUCTOR(nsHapticFeedback)
 NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(ThirdPartyUtil, Init)
 NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(nsIPowerManagerService,
                                          PowerManagerService::GetInstance)
-NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(nsITimeService,
-                                         TimeService::GetInstance)
 
 NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(nsIMediaManagerService,
                                          MediaManager::GetInstance)
@@ -649,7 +645,6 @@ NS_DEFINE_NAMED_CID(NS_HAPTICFEEDBACK_CID);
 NS_DEFINE_NAMED_CID(NS_POWERMANAGERSERVICE_CID);
 NS_DEFINE_NAMED_CID(OSFILECONSTANTSSERVICE_CID);
 NS_DEFINE_NAMED_CID(UDPSOCKETCHILD_CID);
-NS_DEFINE_NAMED_CID(NS_TIMESERVICE_CID);
 NS_DEFINE_NAMED_CID(NS_MEDIAMANAGERSERVICE_CID);
 #ifdef MOZ_WEBSPEECH_TEST_BACKEND
 NS_DEFINE_NAMED_CID(NS_FAKE_SPEECH_RECOGNITION_SERVICE_CID);
@@ -910,7 +905,6 @@ static const mozilla::Module::CIDEntry kLayoutCIDs[] = {
   { &kOSFILECONSTANTSSERVICE_CID, true, nullptr, OSFileConstantsServiceConstructor },
   { &kUDPSOCKETCHILD_CID, false, nullptr, UDPSocketChildConstructor },
   { &kGECKO_MEDIA_PLUGIN_SERVICE_CID, true, nullptr, GeckoMediaPluginServiceConstructor },
-  { &kNS_TIMESERVICE_CID, false, nullptr, nsITimeServiceConstructor },
   { &kNS_MEDIAMANAGERSERVICE_CID, false, nullptr, nsIMediaManagerServiceConstructor },
 #ifdef ACCESSIBILITY
   { &kNS_ACCESSIBILITY_SERVICE_CID, false, nullptr, CreateA11yService },
@@ -1029,7 +1023,6 @@ static const mozilla::Module::ContractIDEntry kLayoutContracts[] = {
   { POWERMANAGERSERVICE_CONTRACTID, &kNS_POWERMANAGERSERVICE_CID, Module::ALLOW_IN_GPU_PROCESS },
   { OSFILECONSTANTSSERVICE_CONTRACTID, &kOSFILECONSTANTSSERVICE_CID },
   { "@mozilla.org/udp-socket-child;1", &kUDPSOCKETCHILD_CID },
-  { TIMESERVICE_CONTRACTID, &kNS_TIMESERVICE_CID },
   { MEDIAMANAGERSERVICE_CONTRACTID, &kNS_MEDIAMANAGERSERVICE_CID },
 #ifdef ACCESSIBILITY
   { "@mozilla.org/accessibilityService;1", &kNS_ACCESSIBILITY_SERVICE_CID },
diff --git a/layout/build/nsLayoutStatics.cpp b/layout/build/nsLayoutStatics.cpp
index 7ead72fc3430..7a13e266be01 100644
--- a/layout/build/nsLayoutStatics.cpp
+++ b/layout/build/nsLayoutStatics.cpp
@@ -106,7 +106,6 @@
 #include "nsCookieService.h"
 #include "nsApplicationCacheService.h"
 #include "mozilla/dom/CustomElementRegistry.h"
-#include "mozilla/dom/time/DateCacheCleaner.h"
 #include "mozilla/EventDispatcher.h"
 #include "mozilla/IMEStateManager.h"
 #include "mozilla/dom/HTMLVideoElement.h"

From 0fef68ce31aa119a651c344108e83f2d3dcad239 Mon Sep 17 00:00:00 2001
From: Kyle Machulis <kyle@nonpolynomial.com>
Date: Mon, 22 Jan 2018 12:56:36 -0800
Subject: [PATCH 37/46] Bug 1432271 - Remove MOZ_TIME_MANAGER block from
 old-configure.in; r=nalexander

MozReview-Commit-ID: E9GBaJq8haT

--HG--
extra : rebase_source : 90f73ce10c8c52d8f9fea9c6a9adb75f8013e7a7
---
 old-configure.in | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/old-configure.in b/old-configure.in
index 5ad6e327a0fe..bb1d4ad162fa 100644
--- a/old-configure.in
+++ b/old-configure.in
@@ -1991,7 +1991,6 @@ NECKO_COOKIES=1
 MOZ_USE_NATIVE_POPUP_WINDOWS=
 MOZ_EXCLUDE_HYPHENATION_DICTIONARIES=
 ACCESSIBILITY=1
-MOZ_TIME_MANAGER=
 MOZ_CONTENT_SANDBOX=
 MOZ_GMP_SANDBOX=
 MOZ_SANDBOX=1
@@ -3806,14 +3805,6 @@ if test -n "$MOZ_DISABLE_STARTUPCACHE"; then
 fi
 AC_SUBST(MOZ_DISABLE_STARTUPCACHE)
 
-dnl ========================================================
-dnl = Enable Support for Time Manager API
-dnl ========================================================
-if test -n "$MOZ_TIME_MANAGER"; then
-    AC_DEFINE(MOZ_TIME_MANAGER)
-fi
-AC_SUBST(MOZ_TIME_MANAGER)
-
 dnl ========================================================
 dnl = Enable Support for Secure Element API
 dnl ========================================================

From 97a87e010b61454bfed9b7c6cf39ef3b50826a1a Mon Sep 17 00:00:00 2001
From: Noemi Erli <nerli@mozilla.com>
Date: Sat, 27 Jan 2018 02:49:25 +0200
Subject: [PATCH 38/46] Backed out 2 changesets (bug 1432271) for bustage in
 /builds/worker/workspace/build/src/obj-firefox/dist/include/mozilla/dom/Navigator.h:79:23
 on a CLOSED TREE

Backed out changeset 7af7827525c8 (bug 1432271)
Backed out changeset 79b9d49e08db (bug 1432271)
---
 dom/base/Navigator.cpp           |  23 +++++
 dom/base/Navigator.h             |  11 +-
 dom/base/moz.build               |   1 +
 dom/base/nsGlobalWindowInner.cpp |  13 +++
 dom/base/nsGlobalWindowInner.h   |   3 +
 dom/base/nsGlobalWindowOuter.cpp |   1 +
 dom/base/nsPIDOMWindow.h         |   3 +
 dom/bindings/Bindings.conf       |   4 +
 dom/ipc/ContentChild.cpp         |   4 +
 dom/ipc/ContentParent.cpp        |   3 +
 dom/moz.build                    |   1 +
 dom/time/DateCacheCleaner.cpp    |  54 ++++++++++
 dom/time/DateCacheCleaner.h      |  19 ++++
 dom/time/TimeChangeObserver.cpp  | 166 +++++++++++++++++++++++++++++++
 dom/time/TimeChangeObserver.h    |  23 +++++
 dom/time/TimeManager.cpp         |  51 ++++++++++
 dom/time/TimeManager.h           |  62 ++++++++++++
 dom/time/TimeService.cpp         |  40 ++++++++
 dom/time/TimeService.h           |  38 +++++++
 dom/time/moz.build               |  32 ++++++
 dom/time/nsITimeService.idl      |  20 ++++
 dom/webidl/MozTimeManager.webidl |  18 ++++
 dom/webidl/Navigator.webidl      |   8 ++
 dom/webidl/moz.build             |   6 +-
 layout/build/nsLayoutModule.cpp  |   7 ++
 layout/build/nsLayoutStatics.cpp |   1 +
 old-configure.in                 |   9 ++
 27 files changed, 618 insertions(+), 3 deletions(-)
 create mode 100644 dom/time/DateCacheCleaner.cpp
 create mode 100644 dom/time/DateCacheCleaner.h
 create mode 100644 dom/time/TimeChangeObserver.cpp
 create mode 100644 dom/time/TimeChangeObserver.h
 create mode 100644 dom/time/TimeManager.cpp
 create mode 100644 dom/time/TimeManager.h
 create mode 100644 dom/time/TimeService.cpp
 create mode 100644 dom/time/TimeService.h
 create mode 100644 dom/time/moz.build
 create mode 100644 dom/time/nsITimeService.idl
 create mode 100644 dom/webidl/MozTimeManager.webidl

diff --git a/dom/base/Navigator.cpp b/dom/base/Navigator.cpp
index bc5d4bb4ace3..d81ea94fe333 100644
--- a/dom/base/Navigator.cpp
+++ b/dom/base/Navigator.cpp
@@ -62,6 +62,7 @@
 #include "nsIStringStream.h"
 #include "nsIHttpChannel.h"
 #include "nsIHttpChannelInternal.h"
+#include "TimeManager.h"
 #include "nsStreamUtils.h"
 #include "WidgetUtils.h"
 #include "nsIPresentationService.h"
@@ -197,6 +198,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(Navigator)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mStorageManager)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCredentials)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaDevices)
+  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTimeManager)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mServiceWorkerContainer)
 
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow)
@@ -246,6 +248,10 @@ Navigator::Invalidate()
 
   mMediaDevices = nullptr;
 
+  if (mTimeManager) {
+    mTimeManager = nullptr;
+  }
+
   if (mPresentation) {
     mPresentation = nullptr;
   }
@@ -1496,6 +1502,23 @@ Navigator::GetConnection(ErrorResult& aRv)
   return mConnection;
 }
 
+#ifdef MOZ_TIME_MANAGER
+time::TimeManager*
+Navigator::GetMozTime(ErrorResult& aRv)
+{
+  if (!mWindow) {
+    aRv.Throw(NS_ERROR_UNEXPECTED);
+    return nullptr;
+  }
+
+  if (!mTimeManager) {
+    mTimeManager = new time::TimeManager(mWindow);
+  }
+
+  return mTimeManager;
+}
+#endif
+
 already_AddRefed<ServiceWorkerContainer>
 Navigator::ServiceWorker()
 {
diff --git a/dom/base/Navigator.h b/dom/base/Navigator.h
index 89c43192a754..51f429cd2d70 100644
--- a/dom/base/Navigator.h
+++ b/dom/base/Navigator.h
@@ -75,8 +75,11 @@ class VRDisplay;
 class VRServiceTest;
 class StorageManager;
 
-class Navigator final : public nsIDOMNavigator
-                      , public nsIMozNavigatorNetwork
+namespace time {
+class TimeManager;
+} // namespace time
+
+class Navigator final : public nsISupports
                       , public nsWrapperCache
 {
 public:
@@ -179,6 +182,9 @@ public:
   bool IsWebVRContentPresenting() const;
   void RequestVRPresentation(VRDisplay& aDisplay);
   nsINetworkProperties* GetNetworkProperties();
+#ifdef MOZ_TIME_MANAGER
+  time::TimeManager* GetMozTime(ErrorResult& aRv);
+#endif // MOZ_TIME_MANAGER
 
   Presentation* GetPresentation(ErrorResult& aRv);
 
@@ -265,6 +271,7 @@ private:
   RefPtr<network::Connection> mConnection;
   RefPtr<CredentialsContainer> mCredentials;
   RefPtr<MediaDevices> mMediaDevices;
+  RefPtr<time::TimeManager> mTimeManager;
   RefPtr<ServiceWorkerContainer> mServiceWorkerContainer;
   nsCOMPtr<nsPIDOMWindowInner> mWindow;
   RefPtr<Presentation> mPresentation;
diff --git a/dom/base/moz.build b/dom/base/moz.build
index 833dac114387..bf36fd6699da 100644
--- a/dom/base/moz.build
+++ b/dom/base/moz.build
@@ -426,6 +426,7 @@ LOCAL_INCLUDES += [
     '../events',
     '../media',
     '../network',
+    '../time',
     '/caps',
     '/docshell/base',
     '/dom/base',
diff --git a/dom/base/nsGlobalWindowInner.cpp b/dom/base/nsGlobalWindowInner.cpp
index 9d3135f92453..d307190b41e6 100644
--- a/dom/base/nsGlobalWindowInner.cpp
+++ b/dom/base/nsGlobalWindowInner.cpp
@@ -228,6 +228,7 @@
 #include "mozilla/DOMEventTargetHelper.h"
 #include "prrng.h"
 #include "nsSandboxFlags.h"
+#include "TimeChangeObserver.h"
 #include "mozilla/dom/AudioContext.h"
 #include "mozilla/dom/BrowserElementDictionariesBinding.h"
 #include "mozilla/dom/cache/CacheStorage.h"
@@ -6871,6 +6872,18 @@ nsGlobalWindowInner::IsVRContentPresenting() const
   return false;
 }
 
+void
+nsGlobalWindowInner::EnableTimeChangeNotifications()
+{
+  mozilla::time::AddWindowListener(this);
+}
+
+void
+nsGlobalWindowInner::DisableTimeChangeNotifications()
+{
+  mozilla::time::RemoveWindowListener(this);
+}
+
 void
 nsGlobalWindowInner::AddSizeOfIncludingThis(nsWindowSizes& aWindowSizes) const
 {
diff --git a/dom/base/nsGlobalWindowInner.h b/dom/base/nsGlobalWindowInner.h
index 46545884fceb..cac0f9f591fd 100644
--- a/dom/base/nsGlobalWindowInner.h
+++ b/dom/base/nsGlobalWindowInner.h
@@ -489,6 +489,9 @@ public:
   virtual void DisableOrientationChangeListener() override;
 #endif
 
+  virtual void EnableTimeChangeNotifications() override;
+  virtual void DisableTimeChangeNotifications() override;
+
   bool IsClosedOrClosing() {
     return mCleanedUp;
   }
diff --git a/dom/base/nsGlobalWindowOuter.cpp b/dom/base/nsGlobalWindowOuter.cpp
index 4fc5eb212acd..347bf61d72a6 100644
--- a/dom/base/nsGlobalWindowOuter.cpp
+++ b/dom/base/nsGlobalWindowOuter.cpp
@@ -226,6 +226,7 @@
 #include "mozilla/DOMEventTargetHelper.h"
 #include "prrng.h"
 #include "nsSandboxFlags.h"
+#include "TimeChangeObserver.h"
 #include "mozilla/dom/AudioContext.h"
 #include "mozilla/dom/BrowserElementDictionariesBinding.h"
 #include "mozilla/dom/cache/CacheStorage.h"
diff --git a/dom/base/nsPIDOMWindow.h b/dom/base/nsPIDOMWindow.h
index 3b198a473f16..6e1440965463 100644
--- a/dom/base/nsPIDOMWindow.h
+++ b/dom/base/nsPIDOMWindow.h
@@ -558,6 +558,9 @@ public:
   virtual void DisableOrientationChangeListener() = 0;
 #endif
 
+  virtual void EnableTimeChangeNotifications() = 0;
+  virtual void DisableTimeChangeNotifications() = 0;
+
   /**
    * Tell this window that there is an observer for gamepad input
    *
diff --git a/dom/bindings/Bindings.conf b/dom/bindings/Bindings.conf
index ad2f46fb2baf..2950e5f49d7c 100644
--- a/dom/bindings/Bindings.conf
+++ b/dom/bindings/Bindings.conf
@@ -601,6 +601,10 @@ DOMInterfaces = {
     'notflattened': True
 },
 
+'MozTimeManager': {
+    'nativeType': 'mozilla::dom::time::TimeManager',
+},
+
 'MozStorageAsyncStatementParams': {
     'headerFile': 'mozilla/storage/mozStorageAsyncStatementParams.h',
     'nativeType': 'mozilla::storage::AsyncStatementParams',
diff --git a/dom/ipc/ContentChild.cpp b/dom/ipc/ContentChild.cpp
index 1a51d1dafa93..b320f4bd9110 100644
--- a/dom/ipc/ContentChild.cpp
+++ b/dom/ipc/ContentChild.cpp
@@ -211,6 +211,7 @@
 #include "nsIPrincipal.h"
 #include "DomainPolicy.h"
 #include "mozilla/dom/ipc/StructuredCloneData.h"
+#include "mozilla/dom/time/DateCacheCleaner.h"
 #include "mozilla/ipc/CrashReporterClient.h"
 #include "mozilla/net/NeckoMessageUtils.h"
 #include "mozilla/widget/PuppetBidiKeyboard.h"
@@ -1221,6 +1222,9 @@ ContentChild::InitXPCOM(const XPCOMInitData& aXPCOMInit,
   nsCOMPtr<nsIURI> ucsURL = DeserializeURI(aXPCOMInit.userContentSheetURL());
   nsLayoutStylesheetCache::SetUserContentCSSURL(ucsURL);
 
+  // This will register cross-process observer.
+  mozilla::dom::time::InitializeDateCacheCleaner();
+
   GfxInfoBase::SetFeatureStatus(aXPCOMInit.gfxFeatureStatus());
 
   DataStorage::SetCachedStorageEntries(aXPCOMInit.dataStorage());
diff --git a/dom/ipc/ContentParent.cpp b/dom/ipc/ContentParent.cpp
index 38d6e7278f13..d717e549b5f8 100644
--- a/dom/ipc/ContentParent.cpp
+++ b/dom/ipc/ContentParent.cpp
@@ -57,6 +57,7 @@
 #include "mozilla/dom/PPresentationParent.h"
 #include "mozilla/dom/PushNotifier.h"
 #include "mozilla/dom/quota/QuotaManagerService.h"
+#include "mozilla/dom/time/DateCacheCleaner.h"
 #include "mozilla/dom/URLClassifierParent.h"
 #include "mozilla/embedding/printingui/PrintingParent.h"
 #include "mozilla/extensions/StreamFilterParent.h"
@@ -627,6 +628,8 @@ ContentParent::StartUp()
   // Note: This reporter measures all ContentParents.
   RegisterStrongMemoryReporter(new ContentParentsMemoryReporter());
 
+  mozilla::dom::time::InitializeDateCacheCleaner();
+
   BackgroundChild::Startup();
   ClientManager::Startup();
 
diff --git a/dom/moz.build b/dom/moz.build
index f8614900b0e2..379013efa8b7 100644
--- a/dom/moz.build
+++ b/dom/moz.build
@@ -74,6 +74,7 @@ DIRS += [
     'security',
     'storage',
     'svg',
+    'time',
     'locales',
     'network',
     'permission',
diff --git a/dom/time/DateCacheCleaner.cpp b/dom/time/DateCacheCleaner.cpp
new file mode 100644
index 000000000000..83d7102b98f4
--- /dev/null
+++ b/dom/time/DateCacheCleaner.cpp
@@ -0,0 +1,54 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "DateCacheCleaner.h"
+
+#include "js/Date.h"
+#include "mozilla/dom/ScriptSettings.h"
+#include "mozilla/ClearOnShutdown.h"
+#include "mozilla/Hal.h"
+#include "mozilla/StaticPtr.h"
+
+using namespace mozilla::hal;
+
+namespace mozilla {
+namespace dom {
+namespace time {
+
+class DateCacheCleaner : public SystemTimezoneChangeObserver
+{
+public:
+  DateCacheCleaner()
+  {
+    RegisterSystemTimezoneChangeObserver(this);
+  }
+
+  ~DateCacheCleaner()
+  {
+    UnregisterSystemTimezoneChangeObserver(this);
+  }
+
+  void Notify(const SystemTimezoneChangeInformation& aSystemTimezoneChangeInfo) override
+  {
+    JS::ResetTimeZone();
+  }
+
+};
+
+StaticAutoPtr<DateCacheCleaner> sDateCacheCleaner;
+
+void
+InitializeDateCacheCleaner()
+{
+  if (!sDateCacheCleaner) {
+    sDateCacheCleaner = new DateCacheCleaner();
+    ClearOnShutdown(&sDateCacheCleaner);
+  }
+}
+
+} // namespace time
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/time/DateCacheCleaner.h b/dom/time/DateCacheCleaner.h
new file mode 100644
index 000000000000..7fb7a0692e6c
--- /dev/null
+++ b/dom/time/DateCacheCleaner.h
@@ -0,0 +1,19 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * InitializeDateCacheCleaner registers DateCacheCleaner to
+ * SystemTimeChangeObserver. When time zone is changed, DateCacheCleaner calls
+ * JS::ResetTimeZone to update the time zone information.
+ */
+
+namespace mozilla {
+namespace dom {
+namespace time {
+void InitializeDateCacheCleaner();
+} //namespace time
+} //namespace dom
+} //namespace mozilla
diff --git a/dom/time/TimeChangeObserver.cpp b/dom/time/TimeChangeObserver.cpp
new file mode 100644
index 000000000000..510153793901
--- /dev/null
+++ b/dom/time/TimeChangeObserver.cpp
@@ -0,0 +1,166 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "TimeChangeObserver.h"
+#include "mozilla/Hal.h"
+#include "mozilla/Observer.h"
+#include "mozilla/HalTypes.h"
+#include "nsWeakPtr.h"
+#include "nsTObserverArray.h"
+#include "mozilla/ClearOnShutdown.h"
+#include "mozilla/Services.h"
+#include "mozilla/StaticPtr.h"
+#include "nsPIDOMWindow.h"
+#include "nsContentUtils.h"
+#include "nsIObserverService.h"
+#include "nsIDocument.h"
+
+using namespace mozilla;
+using namespace mozilla::hal;
+using namespace mozilla::services;
+
+class nsSystemTimeChangeObserver : public SystemClockChangeObserver,
+                                   public SystemTimezoneChangeObserver
+{
+  typedef nsTObserverArray<nsWeakPtr> ListenerArray;
+public:
+  static nsSystemTimeChangeObserver* GetInstance();
+  virtual ~nsSystemTimeChangeObserver();
+
+  // Implementing hal::SystemClockChangeObserver::Notify()
+  void Notify(const int64_t& aClockDeltaMS) override;
+
+  // Implementing hal::SystemTimezoneChangeObserver::Notify()
+  void Notify(
+    const mozilla::hal::SystemTimezoneChangeInformation& aSystemTimezoneChangeInfo) override;
+
+  nsresult AddWindowListenerImpl(nsPIDOMWindowInner* aWindow);
+  nsresult RemoveWindowListenerImpl(nsPIDOMWindowInner* aWindow);
+
+private:
+  nsSystemTimeChangeObserver() { };
+  ListenerArray mWindowListeners;
+  void FireMozTimeChangeEvent();
+};
+
+StaticAutoPtr<nsSystemTimeChangeObserver> sObserver;
+
+nsSystemTimeChangeObserver* nsSystemTimeChangeObserver::GetInstance()
+{
+  if (!sObserver) {
+    sObserver = new nsSystemTimeChangeObserver();
+    ClearOnShutdown(&sObserver);
+  }
+  return sObserver;
+}
+
+nsSystemTimeChangeObserver::~nsSystemTimeChangeObserver()
+{
+  UnregisterSystemClockChangeObserver(this);
+  UnregisterSystemTimezoneChangeObserver(this);
+}
+
+void
+nsSystemTimeChangeObserver::FireMozTimeChangeEvent()
+{
+  ListenerArray::ForwardIterator iter(mWindowListeners);
+  while (iter.HasMore()) {
+    nsWeakPtr weakWindow = iter.GetNext();
+    nsCOMPtr<nsPIDOMWindowInner> innerWindow = do_QueryReferent(weakWindow);
+    nsCOMPtr<nsPIDOMWindowOuter> outerWindow;
+    nsCOMPtr<nsIDocument> document;
+    if (!innerWindow ||
+        !(document = innerWindow->GetExtantDoc()) ||
+        !(outerWindow = innerWindow->GetOuterWindow())) {
+      mWindowListeners.RemoveElement(weakWindow);
+      continue;
+    }
+
+    nsContentUtils::DispatchTrustedEvent(document, outerWindow,
+      NS_LITERAL_STRING("moztimechange"), /* bubbles = */ true,
+      /* canceable = */ false);
+  }
+}
+
+void
+nsSystemTimeChangeObserver::Notify(const int64_t& aClockDeltaMS)
+{
+  // Notify observers that the system clock has been adjusted.
+  nsCOMPtr<nsIObserverService> observerService = GetObserverService();
+  if (observerService) {
+    nsString dataStr;
+    dataStr.AppendFloat(static_cast<double>(aClockDeltaMS));
+    observerService->NotifyObservers(
+      nullptr, "system-clock-change", dataStr.get());
+  }
+
+  FireMozTimeChangeEvent();
+}
+
+void
+nsSystemTimeChangeObserver::Notify(
+  const SystemTimezoneChangeInformation& aSystemTimezoneChangeInfo)
+{
+  FireMozTimeChangeEvent();
+}
+
+nsresult
+mozilla::time::AddWindowListener(nsPIDOMWindowInner* aWindow)
+{
+  return nsSystemTimeChangeObserver::GetInstance()->AddWindowListenerImpl(aWindow);
+}
+
+nsresult
+nsSystemTimeChangeObserver::AddWindowListenerImpl(nsPIDOMWindowInner* aWindow)
+{
+  if (!aWindow) {
+    return NS_ERROR_ILLEGAL_VALUE;
+  }
+
+  nsWeakPtr windowWeakRef = do_GetWeakReference(aWindow);
+  NS_ASSERTION(windowWeakRef, "nsIDOMWindow implementations shuld support weak ref");
+
+  if (mWindowListeners.IndexOf(windowWeakRef) !=
+      ListenerArray::array_type::NoIndex) {
+    return NS_OK;
+  }
+
+  if (mWindowListeners.IsEmpty()) {
+    RegisterSystemClockChangeObserver(sObserver);
+    RegisterSystemTimezoneChangeObserver(sObserver);
+  }
+
+  mWindowListeners.AppendElement(windowWeakRef);
+  return NS_OK;
+}
+
+nsresult
+mozilla::time::RemoveWindowListener(nsPIDOMWindowInner* aWindow)
+{
+  if (!sObserver) {
+    return NS_OK;
+  }
+
+  return nsSystemTimeChangeObserver::GetInstance()->RemoveWindowListenerImpl(aWindow);
+}
+
+nsresult
+nsSystemTimeChangeObserver::RemoveWindowListenerImpl(nsPIDOMWindowInner* aWindow)
+{
+  if (!aWindow) {
+    return NS_OK;
+  }
+
+  nsWeakPtr windowWeakRef = do_GetWeakReference(aWindow);
+  mWindowListeners.RemoveElement(windowWeakRef);
+
+  if (mWindowListeners.IsEmpty()) {
+    UnregisterSystemClockChangeObserver(sObserver);
+    UnregisterSystemTimezoneChangeObserver(sObserver);
+  }
+
+  return NS_OK;
+}
diff --git a/dom/time/TimeChangeObserver.h b/dom/time/TimeChangeObserver.h
new file mode 100644
index 000000000000..65945cbffb9b
--- /dev/null
+++ b/dom/time/TimeChangeObserver.h
@@ -0,0 +1,23 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _mozilla_time_change_observer_h_
+#define _mozilla_time_change_observer_h_
+
+#include "nscore.h"
+
+class nsPIDOMWindowInner;
+
+namespace mozilla {
+namespace time {
+
+nsresult AddWindowListener(nsPIDOMWindowInner* aWindow);
+nsresult RemoveWindowListener(nsPIDOMWindowInner* aWindow);
+
+} // namespace time
+} // namespace mozilla
+
+#endif //_mozilla_time_change_observer_h_
diff --git a/dom/time/TimeManager.cpp b/dom/time/TimeManager.cpp
new file mode 100644
index 000000000000..0fb7f162327e
--- /dev/null
+++ b/dom/time/TimeManager.cpp
@@ -0,0 +1,51 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "TimeManager.h"
+
+#include "mozilla/dom/Date.h"
+#include "mozilla/dom/MozTimeManagerBinding.h"
+#include "nsITimeService.h"
+#include "nsServiceManagerUtils.h"
+
+namespace mozilla {
+namespace dom {
+namespace time {
+
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TimeManager)
+  NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
+  NS_INTERFACE_MAP_ENTRY(nsISupports)
+NS_INTERFACE_MAP_END
+
+NS_IMPL_CYCLE_COLLECTING_ADDREF(TimeManager)
+NS_IMPL_CYCLE_COLLECTING_RELEASE(TimeManager)
+
+NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(TimeManager, mWindow)
+
+JSObject*
+TimeManager::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto)
+{
+  return MozTimeManagerBinding::Wrap(aCx, this, aGivenProto);
+}
+
+void
+TimeManager::Set(Date& aDate)
+{
+  Set(aDate.ToDouble());
+}
+
+void
+TimeManager::Set(double aTime)
+{
+  nsCOMPtr<nsITimeService> timeService = do_GetService(TIMESERVICE_CONTRACTID);
+  if (timeService) {
+    timeService->Set(aTime);
+  }
+}
+
+} // namespace time
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/time/TimeManager.h b/dom/time/TimeManager.h
new file mode 100644
index 000000000000..8c58342b96ba
--- /dev/null
+++ b/dom/time/TimeManager.h
@@ -0,0 +1,62 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_time_TimeManager_h
+#define mozilla_dom_time_TimeManager_h
+
+#include "mozilla/Attributes.h"
+#include "nsISupports.h"
+#include "nsPIDOMWindow.h"
+#include "nsWrapperCache.h"
+
+namespace mozilla {
+namespace dom {
+
+class Date;
+
+namespace time {
+
+class TimeManager final : public nsISupports
+                        , public nsWrapperCache
+{
+public:
+  static bool PrefEnabled(JSContext* aCx, JSObject* aGlobal)
+  {
+#ifdef MOZ_TIME_MANAGER
+    return true;
+#else
+    return false;
+#endif
+  }
+
+  NS_DECL_CYCLE_COLLECTING_ISUPPORTS
+  NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(TimeManager)
+
+  explicit TimeManager(nsPIDOMWindowInner* aWindow)
+    : mWindow(aWindow)
+  {
+  }
+
+  nsPIDOMWindowInner* GetParentObject() const
+  {
+    return mWindow;
+  }
+  virtual JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
+
+  void Set(Date& aDate);
+  void Set(double aTime);
+
+private:
+  ~TimeManager() {}
+
+  nsCOMPtr<nsPIDOMWindowInner> mWindow;
+};
+
+} // namespace time
+} // namespace dom
+} // namespace mozilla
+
+#endif //mozilla_dom_time_TimeManager_h
diff --git a/dom/time/TimeService.cpp b/dom/time/TimeService.cpp
new file mode 100644
index 000000000000..edbd97d51296
--- /dev/null
+++ b/dom/time/TimeService.cpp
@@ -0,0 +1,40 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "base/basictypes.h"
+#include "jsapi.h"
+#include "mozilla/ClearOnShutdown.h"
+#include "mozilla/Hal.h"
+#include "TimeService.h"
+
+namespace mozilla {
+namespace dom {
+namespace time {
+
+NS_IMPL_ISUPPORTS(TimeService, nsITimeService)
+
+/* static */ StaticRefPtr<TimeService> TimeService::sSingleton;
+
+/* static */ already_AddRefed<TimeService>
+TimeService::GetInstance()
+{
+  if (!sSingleton) {
+    sSingleton = new TimeService();
+    ClearOnShutdown(&sSingleton);
+  }
+  RefPtr<TimeService> service = sSingleton.get();
+  return service.forget();
+}
+
+NS_IMETHODIMP
+TimeService::Set(int64_t aTimeInMS) {
+  hal::AdjustSystemClock(aTimeInMS - (JS_Now() / PR_USEC_PER_MSEC));
+  return NS_OK;
+}
+
+} // namespace time
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/time/TimeService.h b/dom/time/TimeService.h
new file mode 100644
index 000000000000..4c802cf0580b
--- /dev/null
+++ b/dom/time/TimeService.h
@@ -0,0 +1,38 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_time_TimeService_h
+#define mozilla_dom_time_TimeService_h
+
+#include "mozilla/StaticPtr.h"
+#include "nsITimeService.h"
+
+namespace mozilla {
+namespace dom {
+namespace time {
+
+/**
+ * This class implements a service which lets us modify the system clock time.
+ */
+class TimeService : public nsITimeService
+{
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSITIMESERVICE
+
+  static already_AddRefed<TimeService> GetInstance();
+
+private:
+  virtual ~TimeService() {};
+
+  static StaticRefPtr<TimeService> sSingleton;
+};
+
+} // namespace time
+} // namespace dom
+} // namespace mozilla
+
+#endif //mozilla_dom_time_TimeService_h
diff --git a/dom/time/moz.build b/dom/time/moz.build
new file mode 100644
index 000000000000..90db7c55b0e9
--- /dev/null
+++ b/dom/time/moz.build
@@ -0,0 +1,32 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files("**"):
+    BUG_COMPONENT = ("Core", "DOM: Device Interfaces")
+
+XPIDL_SOURCES += [
+    'nsITimeService.idl',
+]
+
+XPIDL_MODULE = 'dom_time'
+
+EXPORTS.mozilla.dom.time += [
+    'DateCacheCleaner.h',
+    'TimeChangeObserver.h',
+    'TimeManager.h',
+    'TimeService.h',
+]
+
+UNIFIED_SOURCES += [
+    'DateCacheCleaner.cpp',
+    'TimeChangeObserver.cpp',
+    'TimeManager.cpp',
+    'TimeService.cpp',
+]
+
+include('/ipc/chromium/chromium-config.mozbuild')
+
+FINAL_LIBRARY = 'xul'
diff --git a/dom/time/nsITimeService.idl b/dom/time/nsITimeService.idl
new file mode 100644
index 000000000000..2bba628e948e
--- /dev/null
+++ b/dom/time/nsITimeService.idl
@@ -0,0 +1,20 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+%{C++
+#define NS_TIMESERVICE_CID { 0x80d6f9cc, 0xf16d, 0x40c3, { 0xa5, 0x2e, 0xc4, 0xe6, 0x56, 0xe3, 0x65, 0xb4 } }
+#define TIMESERVICE_CONTRACTID "@mozilla.org/time/timeservice;1"
+%}
+
+[scriptable, builtinclass, uuid(1fc7fde2-0090-11e2-bdd6-0fea4b9f41f8)]
+interface nsITimeService : nsISupports
+{
+  /* Set the system time.
+   *
+   * The |aTimeInMS| argument is the time in milliseconds since the epoch.
+   */
+  void set(in int64_t aTimeInMS);
+};
diff --git a/dom/webidl/MozTimeManager.webidl b/dom/webidl/MozTimeManager.webidl
new file mode 100644
index 000000000000..0308835fb562
--- /dev/null
+++ b/dom/webidl/MozTimeManager.webidl
@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+[Func="mozilla::dom::time::TimeManager::PrefEnabled"]
+interface MozTimeManager {
+  /* Set the system time.
+   *
+   * The |time| argument can be either a Date object or a number.
+   *
+   * - If |time| is a number, it's interpreted as milliseconds
+   *   since the epoch (midnight UTC on January 1, 1970).
+   * - If |time| is a Date object, |set(time)| is equivalent to
+   *   |set(time.getTime())|.
+   */
+  void set(Date time);
+  void set(double time);
+};
diff --git a/dom/webidl/Navigator.webidl b/dom/webidl/Navigator.webidl
index 3ace32934ae4..d9af6d027115 100644
--- a/dom/webidl/Navigator.webidl
+++ b/dom/webidl/Navigator.webidl
@@ -234,6 +234,14 @@ partial interface Navigator {
   VRServiceTest requestVRServiceTest();
 };
 
+#ifdef MOZ_TIME_MANAGER
+// nsIDOMMozNavigatorTime
+partial interface Navigator {
+  [Throws, ChromeOnly]
+  readonly attribute MozTimeManager mozTime;
+};
+#endif // MOZ_TIME_MANAGER
+
 callback NavigatorUserMediaSuccessCallback = void (MediaStream stream);
 callback NavigatorUserMediaErrorCallback = void (MediaStreamError error);
 
diff --git a/dom/webidl/moz.build b/dom/webidl/moz.build
index 09efa87f6bf9..0ef18251feb8 100644
--- a/dom/webidl/moz.build
+++ b/dom/webidl/moz.build
@@ -211,6 +211,9 @@ with Files("Media*Track*"):
 with Files("Mouse*"):
     BUG_COMPONENT = ("Core", "DOM: Events")
 
+with Files("MozTimeManager.webidl"):
+    BUG_COMPONENT = ("Core", "DOM: Device Interfaces")
+
 with Files("MutationEvent.webidl"):
     BUG_COMPONENT = ("Core", "DOM: Events")
 
@@ -361,6 +364,7 @@ GENERATED_WEBIDL_FILES = [
 
 PREPROCESSED_WEBIDL_FILES = [
     'ChromeUtils.webidl',
+    'Navigator.webidl',
     'Node.webidl',
     'Window.webidl',
 ]
@@ -686,11 +690,11 @@ WEBIDL_FILES = [
     'MozStorageAsyncStatementParams.webidl',
     'MozStorageStatementParams.webidl',
     'MozStorageStatementRow.webidl',
+    'MozTimeManager.webidl',
     'MutationEvent.webidl',
     'MutationObserver.webidl',
     'NamedNodeMap.webidl',
     'NativeOSFileInternals.webidl',
-    'Navigator.webidl',
     'NetDashboard.webidl',
     'NetworkInformation.webidl',
     'NetworkOptions.webidl',
diff --git a/layout/build/nsLayoutModule.cpp b/layout/build/nsLayoutModule.cpp
index 06332901000a..d85877b27930 100644
--- a/layout/build/nsLayoutModule.cpp
+++ b/layout/build/nsLayoutModule.cpp
@@ -163,6 +163,7 @@ static void Shutdown();
 #include "mozilla/net/WebSocketEventService.h"
 
 #include "mozilla/dom/power/PowerManagerService.h"
+#include "mozilla/dom/time/TimeService.h"
 
 #include "nsIPresentationService.h"
 
@@ -184,6 +185,7 @@ using mozilla::dom::quota::QuotaManagerService;
 using mozilla::dom::workers::ServiceWorkerManager;
 using mozilla::dom::workers::WorkerDebuggerManager;
 using mozilla::dom::UDPSocketChild;
+using mozilla::dom::time::TimeService;
 using mozilla::gmp::GeckoMediaPluginService;
 using mozilla::dom::NotificationTelemetryService;
 
@@ -259,6 +261,8 @@ NS_GENERIC_FACTORY_CONSTRUCTOR(nsHapticFeedback)
 NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(ThirdPartyUtil, Init)
 NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(nsIPowerManagerService,
                                          PowerManagerService::GetInstance)
+NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(nsITimeService,
+                                         TimeService::GetInstance)
 
 NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(nsIMediaManagerService,
                                          MediaManager::GetInstance)
@@ -645,6 +649,7 @@ NS_DEFINE_NAMED_CID(NS_HAPTICFEEDBACK_CID);
 NS_DEFINE_NAMED_CID(NS_POWERMANAGERSERVICE_CID);
 NS_DEFINE_NAMED_CID(OSFILECONSTANTSSERVICE_CID);
 NS_DEFINE_NAMED_CID(UDPSOCKETCHILD_CID);
+NS_DEFINE_NAMED_CID(NS_TIMESERVICE_CID);
 NS_DEFINE_NAMED_CID(NS_MEDIAMANAGERSERVICE_CID);
 #ifdef MOZ_WEBSPEECH_TEST_BACKEND
 NS_DEFINE_NAMED_CID(NS_FAKE_SPEECH_RECOGNITION_SERVICE_CID);
@@ -905,6 +910,7 @@ static const mozilla::Module::CIDEntry kLayoutCIDs[] = {
   { &kOSFILECONSTANTSSERVICE_CID, true, nullptr, OSFileConstantsServiceConstructor },
   { &kUDPSOCKETCHILD_CID, false, nullptr, UDPSocketChildConstructor },
   { &kGECKO_MEDIA_PLUGIN_SERVICE_CID, true, nullptr, GeckoMediaPluginServiceConstructor },
+  { &kNS_TIMESERVICE_CID, false, nullptr, nsITimeServiceConstructor },
   { &kNS_MEDIAMANAGERSERVICE_CID, false, nullptr, nsIMediaManagerServiceConstructor },
 #ifdef ACCESSIBILITY
   { &kNS_ACCESSIBILITY_SERVICE_CID, false, nullptr, CreateA11yService },
@@ -1023,6 +1029,7 @@ static const mozilla::Module::ContractIDEntry kLayoutContracts[] = {
   { POWERMANAGERSERVICE_CONTRACTID, &kNS_POWERMANAGERSERVICE_CID, Module::ALLOW_IN_GPU_PROCESS },
   { OSFILECONSTANTSSERVICE_CONTRACTID, &kOSFILECONSTANTSSERVICE_CID },
   { "@mozilla.org/udp-socket-child;1", &kUDPSOCKETCHILD_CID },
+  { TIMESERVICE_CONTRACTID, &kNS_TIMESERVICE_CID },
   { MEDIAMANAGERSERVICE_CONTRACTID, &kNS_MEDIAMANAGERSERVICE_CID },
 #ifdef ACCESSIBILITY
   { "@mozilla.org/accessibilityService;1", &kNS_ACCESSIBILITY_SERVICE_CID },
diff --git a/layout/build/nsLayoutStatics.cpp b/layout/build/nsLayoutStatics.cpp
index 7a13e266be01..7ead72fc3430 100644
--- a/layout/build/nsLayoutStatics.cpp
+++ b/layout/build/nsLayoutStatics.cpp
@@ -106,6 +106,7 @@
 #include "nsCookieService.h"
 #include "nsApplicationCacheService.h"
 #include "mozilla/dom/CustomElementRegistry.h"
+#include "mozilla/dom/time/DateCacheCleaner.h"
 #include "mozilla/EventDispatcher.h"
 #include "mozilla/IMEStateManager.h"
 #include "mozilla/dom/HTMLVideoElement.h"
diff --git a/old-configure.in b/old-configure.in
index bb1d4ad162fa..5ad6e327a0fe 100644
--- a/old-configure.in
+++ b/old-configure.in
@@ -1991,6 +1991,7 @@ NECKO_COOKIES=1
 MOZ_USE_NATIVE_POPUP_WINDOWS=
 MOZ_EXCLUDE_HYPHENATION_DICTIONARIES=
 ACCESSIBILITY=1
+MOZ_TIME_MANAGER=
 MOZ_CONTENT_SANDBOX=
 MOZ_GMP_SANDBOX=
 MOZ_SANDBOX=1
@@ -3805,6 +3806,14 @@ if test -n "$MOZ_DISABLE_STARTUPCACHE"; then
 fi
 AC_SUBST(MOZ_DISABLE_STARTUPCACHE)
 
+dnl ========================================================
+dnl = Enable Support for Time Manager API
+dnl ========================================================
+if test -n "$MOZ_TIME_MANAGER"; then
+    AC_DEFINE(MOZ_TIME_MANAGER)
+fi
+AC_SUBST(MOZ_TIME_MANAGER)
+
 dnl ========================================================
 dnl = Enable Support for Secure Element API
 dnl ========================================================

From 082c64aa0c42ae86e216b682a3c117a22e5d4e44 Mon Sep 17 00:00:00 2001
From: Masayuki Nakano <masayuki@d-toybox.com>
Date: Thu, 25 Jan 2018 23:27:07 +0900
Subject: [PATCH 39/46] Bug 1433101 - part 1: Add new pref which disables
 keypress event for non-printable keys only for the default event group in web
 content r=smaug

UI Events declares that keypress event should be fired when the keypress event
causes some text input.  However, we're keeping our traditional behavior for
historical reasons because our internal event handlers (including event
handlers of Thunderbird) handles keypress events for any keys.  Therefore,
for minimizing the side effect, we should stop kicking keypress event handlers
in the default event group in web content.

This patch adds new pref for enabling the standard behavior in web content.

Additionally, creates WidgetKeyboardEvent::IsInputtingText() for sharing the
check logic between TextEventDispatcher and TextEditor/HTMLEditor.

MozReview-Commit-ID: 3rtXdLBPeVC

--HG--
extra : rebase_source : 2fc3c9a09840d0d03800c9a42bb83ca76a8db2d5
---
 editor/libeditor/HTMLEditor.cpp |  6 +-----
 editor/libeditor/TextEditor.cpp |  6 +-----
 modules/libpref/init/all.js     |  5 +++++
 widget/TextEventDispatcher.cpp  | 13 +++++++++++++
 widget/TextEventDispatcher.h    |  3 +++
 widget/TextEvents.h             | 13 +++++++++++++
 6 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/editor/libeditor/HTMLEditor.cpp b/editor/libeditor/HTMLEditor.cpp
index f0b04277344e..ded58bd1f32a 100644
--- a/editor/libeditor/HTMLEditor.cpp
+++ b/editor/libeditor/HTMLEditor.cpp
@@ -679,11 +679,7 @@ HTMLEditor::HandleKeyPressEvent(WidgetKeyboardEvent* aKeyboardEvent)
       return TypedText(EmptyString(), eTypedBreak);
   }
 
-  // NOTE: On some keyboard layout, some characters are inputted with Control
-  // key or Alt key, but at that time, widget sets FALSE to these keys.
-  if (!aKeyboardEvent->mCharCode || aKeyboardEvent->IsControl() ||
-      aKeyboardEvent->IsAlt() || aKeyboardEvent->IsMeta() ||
-      aKeyboardEvent->IsOS()) {
+  if (!aKeyboardEvent->IsInputtingText()) {
     // we don't PreventDefault() here or keybindings like control-x won't work
     return NS_OK;
   }
diff --git a/editor/libeditor/TextEditor.cpp b/editor/libeditor/TextEditor.cpp
index 322b6d1fb980..4db2eab03cec 100644
--- a/editor/libeditor/TextEditor.cpp
+++ b/editor/libeditor/TextEditor.cpp
@@ -384,11 +384,7 @@ TextEditor::HandleKeyPressEvent(WidgetKeyboardEvent* aKeyboardEvent)
       return TypedText(EmptyString(), eTypedBreak);
   }
 
-  // NOTE: On some keyboard layout, some characters are inputted with Control
-  // key or Alt key, but at that time, widget sets FALSE to these keys.
-  if (!aKeyboardEvent->mCharCode || aKeyboardEvent->IsControl() ||
-      aKeyboardEvent->IsAlt() || aKeyboardEvent->IsMeta() ||
-      aKeyboardEvent->IsOS()) {
+  if (!aKeyboardEvent->IsInputtingText()) {
     // we don't PreventDefault() here or keybindings like control-x won't work
     return NS_OK;
   }
diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
index 34dd59673945..e46ae18d9268 100644
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -206,6 +206,11 @@ pref("dom.gamepad.haptic_feedback.enabled", true);
 // even if this is true).
 pref("dom.keyboardevent.dispatch_during_composition", false);
 
+// If this is true, TextEventDispatcher dispatches keypress event with setting
+// WidgetEvent::mFlags::mOnlySystemGroupDispatchInContent to true if it won't
+// cause inputting printable character.
+pref("dom.keyboardevent.keypress.dispatch_non_printable_keys_only_system_group_in_content", false);
+
 // Whether to run add-on code in different compartments from browser code. This
 // causes a separate compartment for each (addon, global) combination, which may
 // significantly increase the number of compartments in the system.
diff --git a/widget/TextEventDispatcher.cpp b/widget/TextEventDispatcher.cpp
index a300115562a3..8b68ea4d8b6e 100644
--- a/widget/TextEventDispatcher.cpp
+++ b/widget/TextEventDispatcher.cpp
@@ -22,6 +22,8 @@ namespace widget {
  *****************************************************************************/
 
 bool TextEventDispatcher::sDispatchKeyEventsDuringComposition = false;
+bool TextEventDispatcher::sDispatchKeyPressEventsOnlySystemGroupInContent =
+       false;
 
 TextEventDispatcher::TextEventDispatcher(nsIWidget* aWidget)
   : mWidget(aWidget)
@@ -39,6 +41,11 @@ TextEventDispatcher::TextEventDispatcher(nsIWidget* aWidget)
       &sDispatchKeyEventsDuringComposition,
       "dom.keyboardevent.dispatch_during_composition",
       false);
+    Preferences::AddBoolVarCache(
+      &sDispatchKeyPressEventsOnlySystemGroupInContent,
+      "dom.keyboardevent.keypress."
+        "dispatch_non_printable_keys_only_system_group_in_content",
+      false);
     sInitialized = true;
   }
 
@@ -687,6 +694,12 @@ TextEventDispatcher::DispatchKeyboardEventInternal(
     }
   }
 
+  if (sDispatchKeyPressEventsOnlySystemGroupInContent &&
+      keyEvent.mMessage == eKeyPress &&
+      !keyEvent.IsInputtingText()) {
+    keyEvent.mFlags.mOnlySystemGroupDispatchInContent = true;
+  }
+
   DispatchInputEvent(mWidget, keyEvent, aStatus);
   return true;
 }
diff --git a/widget/TextEventDispatcher.h b/widget/TextEventDispatcher.h
index fc9c15e509e4..cc20acd1ec74 100644
--- a/widget/TextEventDispatcher.h
+++ b/widget/TextEventDispatcher.h
@@ -454,6 +454,9 @@ private:
   // If this is true, keydown and keyup events are dispatched even when there
   // is a composition.
   static bool sDispatchKeyEventsDuringComposition;
+  // If this is true, keypress events for non-printable keys are dispatched only
+  // for event listeners of the system event group in web content.
+  static bool sDispatchKeyPressEventsOnlySystemGroupInContent;
 
   nsresult BeginInputTransactionInternal(
              TextEventDispatcherListener* aListener,
diff --git a/widget/TextEvents.h b/widget/TextEvents.h
index 3d43c6652d58..5d8812c89bc2 100644
--- a/widget/TextEvents.h
+++ b/widget/TextEvents.h
@@ -228,6 +228,19 @@ public:
     return IsKeyEventOnPlugin(mMessage);
   }
 
+  bool IsInputtingText() const
+  {
+    // NOTE: On some keyboard layout, some characters are inputted with Control
+    //       key or Alt key, but at that time, widget unset the modifier flag
+    //       from eKeyPress event.
+    return mMessage == eKeyPress &&
+           mCharCode &&
+           !(mModifiers & (MODIFIER_ALT |
+                           MODIFIER_CONTROL |
+                           MODIFIER_META |
+                           MODIFIER_OS));
+  }
+
   virtual WidgetEvent* Duplicate() const override
   {
     MOZ_ASSERT(mClass == eKeyboardEventClass,

From e43e597a9eabf9f439c6f9e23a925ca2117a572d Mon Sep 17 00:00:00 2001
From: Masayuki Nakano <masayuki@d-toybox.com>
Date: Thu, 25 Jan 2018 23:59:20 +0900
Subject: [PATCH 40/46] Bug 1433101 - part 2: Treat Enter and Shift+Enter as
 printable key r=smaug
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Chromium dispatches a keypress event when pressing Enter or Shift+Enter.
Actually, when user press them in <pre> element of HTML editor, ﾂ･n is inserted.
It makes sense to treat the key combinations as inputting text.

MozReview-Commit-ID: Hvx87MZtZkn

--HG--
extra : rebase_source : 196b63843ebcb6e4b398f6b21a4f5f1d020b8db3
---
 editor/libeditor/HTMLEditor.cpp |  3 +--
 editor/libeditor/TextEditor.cpp |  4 +---
 widget/TextEventDispatcher.cpp  |  2 +-
 widget/TextEvents.h             | 10 ++++++++++
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/editor/libeditor/HTMLEditor.cpp b/editor/libeditor/HTMLEditor.cpp
index ded58bd1f32a..4f0585c937af 100644
--- a/editor/libeditor/HTMLEditor.cpp
+++ b/editor/libeditor/HTMLEditor.cpp
@@ -666,8 +666,7 @@ HTMLEditor::HandleKeyPressEvent(WidgetKeyboardEvent* aKeyboardEvent)
       return TypedText(NS_LITERAL_STRING("\t"), eTypedText);
     }
     case NS_VK_RETURN:
-      if (aKeyboardEvent->IsControl() || aKeyboardEvent->IsAlt() ||
-          aKeyboardEvent->IsMeta() || aKeyboardEvent->IsOS()) {
+      if (!aKeyboardEvent->IsInputtingLineBreak()) {
         return NS_OK;
       }
       aKeyboardEvent->PreventDefault(); // consumed
diff --git a/editor/libeditor/TextEditor.cpp b/editor/libeditor/TextEditor.cpp
index 4db2eab03cec..b3fecc9591bc 100644
--- a/editor/libeditor/TextEditor.cpp
+++ b/editor/libeditor/TextEditor.cpp
@@ -375,9 +375,7 @@ TextEditor::HandleKeyPressEvent(WidgetKeyboardEvent* aKeyboardEvent)
       return TypedText(NS_LITERAL_STRING("\t"), eTypedText);
     }
     case NS_VK_RETURN:
-      if (IsSingleLineEditor() || aKeyboardEvent->IsControl() ||
-          aKeyboardEvent->IsAlt() || aKeyboardEvent->IsMeta() ||
-          aKeyboardEvent->IsOS()) {
+      if (IsSingleLineEditor() || !aKeyboardEvent->IsInputtingLineBreak()) {
         return NS_OK;
       }
       aKeyboardEvent->PreventDefault();
diff --git a/widget/TextEventDispatcher.cpp b/widget/TextEventDispatcher.cpp
index 8b68ea4d8b6e..c323eb580e64 100644
--- a/widget/TextEventDispatcher.cpp
+++ b/widget/TextEventDispatcher.cpp
@@ -696,7 +696,7 @@ TextEventDispatcher::DispatchKeyboardEventInternal(
 
   if (sDispatchKeyPressEventsOnlySystemGroupInContent &&
       keyEvent.mMessage == eKeyPress &&
-      !keyEvent.IsInputtingText()) {
+      !keyEvent.IsInputtingText() && !keyEvent.IsInputtingLineBreak()) {
     keyEvent.mFlags.mOnlySystemGroupDispatchInContent = true;
   }
 
diff --git a/widget/TextEvents.h b/widget/TextEvents.h
index 5d8812c89bc2..7b766bde47d5 100644
--- a/widget/TextEvents.h
+++ b/widget/TextEvents.h
@@ -241,6 +241,16 @@ public:
                            MODIFIER_OS));
   }
 
+  bool IsInputtingLineBreak() const
+  {
+    return mMessage == eKeyPress &&
+           mKeyNameIndex == KEY_NAME_INDEX_Enter &&
+           !(mModifiers & (MODIFIER_ALT |
+                           MODIFIER_CONTROL |
+                           MODIFIER_META |
+                           MODIFIER_OS));
+  }
+
   virtual WidgetEvent* Duplicate() const override
   {
     MOZ_ASSERT(mClass == eKeyboardEventClass,

From 11cd13abd8f0da990c867e3146a062b8f3f8e9e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Tue, 23 Jan 2018 17:01:16 +0100
Subject: [PATCH 41/46] Bug 1432563: Fetch rustfmt when repackaging rust.
 r=rillian

I tested this on automation and the build went on, though I couldn't test
the bindgen output because the build right now is busted on one dependent crate
with rust beta, which is the first toolchain that has this package, and will go
to release shortly.

This should work though! If I need more changes I'll adjust them in bug 1432153.

You can test the repackage manually with repack_rust.py --toolchain beta, for
example.

MozReview-Commit-ID: GI2f6vGVqTe
---
 taskcluster/scripts/misc/repack_rust.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/taskcluster/scripts/misc/repack_rust.py b/taskcluster/scripts/misc/repack_rust.py
index 54afcaa2c830..e0744c1f6be3 100644
--- a/taskcluster/scripts/misc/repack_rust.py
+++ b/taskcluster/scripts/misc/repack_rust.py
@@ -231,6 +231,14 @@ def fetch_std(manifest, targets):
     return stds
 
 
+def fetch_optional(manifest, pkg, host):
+    try:
+        return fetch_package(manifest, pkg, host)
+    except KeyError:
+        # The package is not available, oh well!
+        return None
+
+
 def tar_for_host(host):
     if 'linux' in host:
         tar_options = 'cJf'
@@ -274,6 +282,7 @@ def repack(host, targets, channel='stable', cargo_channel=None):
     rustc = fetch_package(manifest, 'rustc', host)
     cargo = fetch_package(cargo_manifest, 'cargo', host)
     stds = fetch_std(manifest, targets)
+    rustfmt = fetch_optional(manifest, 'rustfmt-preview', host)
 
     log('Installing packages...')
     install_dir = 'rustc'
@@ -285,6 +294,8 @@ def repack(host, targets, channel='stable', cargo_channel=None):
             raise
     install(os.path.basename(rustc['url']), install_dir)
     install(os.path.basename(cargo['url']), install_dir)
+    if rustfmt:
+        install(os.path.basename(rustfmt['url']), install_dir)
     for std in stds:
         install(os.path.basename(std['url']), install_dir)
         pass

From 8d7187be91bb9c71a626827fea9ad09aef2858a1 Mon Sep 17 00:00:00 2001
From: Kit Cambridge <kit@yakshaving.ninja>
Date: Sat, 6 Jan 2018 07:44:33 -0800
Subject: [PATCH 42/46] Bug 1305563 - Add a bookmark mirror and two-way merger
 for synced bookmarks. r=mak,markh,rnewman,tcsc

The mirror matches the complete bookmark tree stored on the Sync
server, stores new bookmarks changed on the server since the last
sync, merges the local and remote trees, applies the resulting merged
tree back to Places, fires observer notifications for all items changed
in the merge, and stages locally changed bookmarks for upload.

MozReview-Commit-ID: MbeFQUargt

--HG--
extra : rebase_source : b635fd65ef10ab9ce3a2f9af2e527d342a790f6e
---
 services/sync/modules/engines/bookmarks.js    |    2 +-
 toolkit/components/places/PlacesSyncUtils.jsm |   14 +-
 .../places/SyncedBookmarksMirror.jsm          | 3909 +++++++++++++++++
 toolkit/components/places/moz.build           |    1 +
 .../places/tests/PlacesTestUtils.jsm          |   38 +
 toolkit/components/places/tests/moz.build     |    1 +
 .../components/places/tests/sync/head_sync.js |  190 +
 .../components/places/tests/sync/livemark.xml |   17 +
 .../tests/sync/test_bookmark_corruption.js    | 1000 +++++
 .../tests/sync/test_bookmark_deduping.js      |  444 ++
 .../tests/sync/test_bookmark_deletion.js      |  587 +++
 .../places/tests/sync/test_bookmark_kinds.js  |  287 ++
 .../sync/test_bookmark_structure_changes.js   |  742 ++++
 .../tests/sync/test_bookmark_value_changes.js | 1202 +++++
 .../components/places/tests/sync/xpcshell.ini |   11 +
 tools/lint/eslint/modules.json                |    1 +
 16 files changed, 8442 insertions(+), 4 deletions(-)
 create mode 100644 toolkit/components/places/SyncedBookmarksMirror.jsm
 create mode 100644 toolkit/components/places/tests/sync/head_sync.js
 create mode 100644 toolkit/components/places/tests/sync/livemark.xml
 create mode 100644 toolkit/components/places/tests/sync/test_bookmark_corruption.js
 create mode 100644 toolkit/components/places/tests/sync/test_bookmark_deduping.js
 create mode 100644 toolkit/components/places/tests/sync/test_bookmark_deletion.js
 create mode 100644 toolkit/components/places/tests/sync/test_bookmark_kinds.js
 create mode 100644 toolkit/components/places/tests/sync/test_bookmark_structure_changes.js
 create mode 100644 toolkit/components/places/tests/sync/test_bookmark_value_changes.js
 create mode 100644 toolkit/components/places/tests/sync/xpcshell.ini

diff --git a/services/sync/modules/engines/bookmarks.js b/services/sync/modules/engines/bookmarks.js
index e60d8e4065a1..c023859c941a 100644
--- a/services/sync/modules/engines/bookmarks.js
+++ b/services/sync/modules/engines/bookmarks.js
@@ -128,7 +128,7 @@ PlacesItem.prototype = {
     };
     let dateAdded = PlacesSyncUtils.bookmarks.ratchetTimestampBackwards(
       this.dateAdded, +this.modified * 1000);
-    if (dateAdded !== undefined) {
+    if (dateAdded > 0) {
       result.dateAdded = dateAdded;
     }
     return result;
diff --git a/toolkit/components/places/PlacesSyncUtils.jsm b/toolkit/components/places/PlacesSyncUtils.jsm
index 0edf0968444c..d7b1cbd3e895 100644
--- a/toolkit/components/places/PlacesSyncUtils.jsm
+++ b/toolkit/components/places/PlacesSyncUtils.jsm
@@ -564,6 +564,7 @@ const BookmarkSyncUtils = PlacesSyncUtils.bookmarks = Object.freeze({
     return PlacesUtils.withConnectionWrapper(
       "BookmarkSyncUtils: pushChanges", async function(db) {
         let skippedCount = 0;
+        let weakCount = 0;
         let updateParams = [];
 
         for (let recordId in changeRecords) {
@@ -577,6 +578,12 @@ const BookmarkSyncUtils = PlacesSyncUtils.bookmarks = Object.freeze({
             }
           );
 
+          // Skip weakly uploaded records.
+          if (!changeRecord.counter) {
+            weakCount++;
+            continue;
+          }
+
           // Sync sets the `synced` flag for reconciled or successfully
           // uploaded items. If upload failed, ignore the change; we'll
           // try again on the next sync.
@@ -617,7 +624,8 @@ const BookmarkSyncUtils = PlacesSyncUtils.bookmarks = Object.freeze({
         }
 
         BookmarkSyncLog.debug(`pushChanges: Processed change records`,
-                              { skipped: skippedCount,
+                              { weak: weakCount,
+                                skipped: skippedCount,
                                 updated: updateParams.length });
       }
     );
@@ -1024,14 +1032,14 @@ const BookmarkSyncUtils = PlacesSyncUtils.bookmarks = Object.freeze({
   },
 
   /**
-   * Returns `undefined` if no sensible timestamp could be found.
+   * Returns `0` if no sensible timestamp could be found.
    * Otherwise, returns the earliest sensible timestamp between `existingMillis`
    * and `serverMillis`.
    */
   ratchetTimestampBackwards(existingMillis, serverMillis, lowerBound = BookmarkSyncUtils.EARLIEST_BOOKMARK_TIMESTAMP) {
     const possible = [+existingMillis, +serverMillis].filter(n => !isNaN(n) && n > lowerBound);
     if (!possible.length) {
-      return undefined;
+      return 0;
     }
     return Math.min(...possible);
   },
diff --git a/toolkit/components/places/SyncedBookmarksMirror.jsm b/toolkit/components/places/SyncedBookmarksMirror.jsm
new file mode 100644
index 000000000000..170f65381ea7
--- /dev/null
+++ b/toolkit/components/places/SyncedBookmarksMirror.jsm
@@ -0,0 +1,3909 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * This file implements a mirror and two-way merger for synced bookmarks. The
+ * mirror matches the complete tree stored on the Sync server, and stages new
+ * bookmarks changed on the server since the last sync. The merger walks the
+ * local tree in Places and the mirrored remote tree, produces a new merged
+ * tree, then updates the local tree to reflect the merged tree.
+ *
+ * Let's start with an overview of the different classes, and how they fit
+ * together.
+ *
+ * - `SyncedBookmarksMirror` sets up the database, validates and upserts new
+ *   incoming records, attaches to Places, and applies the changed records.
+ *   During application, we fetch the local and remote bookmark trees, merge
+ *   them, and update Places to match. Merging and application happen in a
+ *   single transaction, so applying the merged tree won't collide with local
+ *   changes. A failure at this point aborts the merge and leaves Places
+ *   unchanged.
+ *
+ * - A `BookmarkTree` is a fully rooted tree that also notes deletions. A
+ *   `BookmarkNode` represents a local item in Places, or a remote item in the
+ *   mirror.
+ *
+ * - A `MergedBookmarkNode` holds a local node, a remote node, and a
+ *   `MergeState` that indicates which node to prefer when updating Places and
+ *   the server to match the merged tree.
+ *
+ * - `BookmarkObserverRecorder` records all changes made to Places during the
+ *   merge, then dispatches `nsINavBookmarkObserver` notifications. Places uses
+ *   these notifications to update the UI and internal caches. We can't dispatch
+ *   during the merge because observers won't see the changes until the merge
+ *   transaction commits and the database is consistent again.
+ *
+ * - After application, we flag all applied incoming items as merged, create
+ *   Sync records for the locally new and updated items in Places, and upload
+ *   the records to the server. At this point, all outgoing items are flagged as
+ *   changed in Places, so the next sync can resume cleanly if the upload is
+ *   interrupted or fails.
+ *
+ * - Once upload succeeds, we update the mirror with the uploaded records, so
+ *   that the mirror matches the server again. An interruption or error here
+ *   will leave the uploaded items flagged as changed in Places, so we'll merge
+ *   them again on the next sync. This is redundant work, but shouldn't cause
+ *   issues.
+ */
+
+this.EXPORTED_SYMBOLS = ["SyncedBookmarksMirror"];
+
+const { utils: Cu, interfaces: Ci } = Components;
+
+Cu.importGlobalProperties(["URL"]);
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  AsyncShutdown: "resource://gre/modules/AsyncShutdown.jsm",
+  Log: "resource://gre/modules/Log.jsm",
+  OS: "resource://gre/modules/osfile.jsm",
+  PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.jsm",
+  PlacesUtils: "resource://gre/modules/PlacesUtils.jsm",
+  Sqlite: "resource://gre/modules/Sqlite.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(this, "MirrorLog", () =>
+  Log.repository.getLogger("Sync.Engine.Bookmarks.Mirror")
+);
+
+// These can be removed once they're exposed in a central location (bug
+// 1375896).
+const DB_URL_LENGTH_MAX = 65536;
+const DB_TITLE_LENGTH_MAX = 4096;
+const DB_DESCRIPTION_LENGTH_MAX = 256;
+
+const SQLITE_MAX_VARIABLE_NUMBER = 999;
+
+// The current mirror database schema version. Bump for migrations, then add
+// migration code to `migrateMirrorSchema`.
+const MIRROR_SCHEMA_VERSION = 1;
+
+/**
+ * A mirror maintains a copy of the complete tree as stored on the Sync server.
+ * It is persistent.
+ *
+ * The mirror schema is a hybrid of how Sync and Places represent bookmarks.
+ * The `items` table contains item attributes (title, kind, description,
+ * URL, etc.), while the `structure` table stores parent-child relationships and
+ * position. This is similar to how iOS encodes "value" and "structure" state,
+ * though we handle these differently when merging. See `BookmarkMerger` for
+ * details.
+ *
+ * There's no guarantee that the remote state is consistent. We might be missing
+ * parents or children, or a bookmark and its parent might disagree about where
+ * it belongs. This means we need a strategy to handle missing parents and
+ * children.
+ *
+ * We treat the `children` of the last parent we see as canonical, and ignore
+ * the child's `parentid` entirely. We also ignore missing children, and
+ * temporarily reparent bookmarks with missing parents to "unfiled". When we
+ * eventually see the missing items, either during a later sync or as part of
+ * repair, we'll fill in the mirror's gaps and fix up the local tree.
+ *
+ * During merging, we won't intentionally try to fix inconsistencies on the
+ * server, and opt to build as complete a tree as we can from the remote state,
+ * even if we diverge from what's in the mirror. See bug 1433512 for context.
+ *
+ * If a sync is interrupted, we resume downloading from the server collection
+ * last modified time, or the server last modified time of the most recent
+ * record if newer. New incoming records always replace existing records in the
+ * mirror.
+ *
+ * We delete the mirror database on client reset, including when the sync ID
+ * changes on the server, and when the user is node reassigned, disables the
+ * bookmarks engine, or signs out.
+ */
+class SyncedBookmarksMirror {
+  constructor(db, { recordTelemetryEvent, finalizeAt =
+                      AsyncShutdown.profileBeforeChange } = {}) {
+    this.db = db;
+    this.recordTelemetryEvent = recordTelemetryEvent;
+
+    // Automatically close the database connection on shutdown.
+    this.finalizeAt = finalizeAt;
+    this.finalizeBound = () => this.finalize();
+    this.finalizeAt.addBlocker("SyncedBookmarksMirror: finalize",
+                               this.finalizeBound);
+  }
+
+  /**
+   * Sets up the mirror database connection and upgrades the mirror to the
+   * newest schema version. Automatically recreates the mirror if it's corrupt;
+   * throws on failure.
+   *
+   * @param  {String} options.path
+   *         The full path to the mirror database file.
+   * @param  {Function} options.recordTelemetryEvent
+   *         A function with the signature `(object: String, method: String,
+   *         value: String?, extra: Object?)`, used to emit telemetry events.
+   * @param  {AsyncShutdown.Barrier} [options.finalizeAt]
+   *         A shutdown phase, barrier, or barrier client that should
+   *         automatically finalize the mirror when triggered. Exposed for
+   *         testing.
+   * @return {SyncedBookmarksMirror}
+   *         A mirror ready for use.
+   */
+  static async open(options) {
+    let db = await Sqlite.cloneStorageConnection({
+      connection: PlacesUtils.history.DBConnection,
+      readOnly: false,
+    });
+    try {
+      try {
+        await db.execute(`ATTACH :mirrorPath AS mirror`,
+                         { mirrorPath: options.path });
+      } catch (ex) {
+        if (ex.errors && isDatabaseCorrupt(ex.errors[0])) {
+          MirrorLog.warn("Error attaching mirror to Places; removing and " +
+                         "recreating mirror", ex);
+          options.recordTelemetryEvent("mirror", "open", "error",
+                                       { why: "corrupt" });
+          await OS.File.remove(options.path);
+          await db.execute(`ATTACH :mirrorPath AS mirror`,
+                           { mirrorPath: options.path });
+        } else {
+          MirrorLog.warn("Unrecoverable error attaching mirror to Places", ex);
+          throw ex;
+        }
+      }
+      await db.execute(`PRAGMA foreign_keys = ON`);
+      await migrateMirrorSchema(db);
+      await initializeTempMirrorEntities(db);
+    } catch (ex) {
+      options.recordTelemetryEvent("mirror", "open", "error",
+                                   { why: "initialize" });
+      await db.close();
+      throw ex;
+    }
+    return new SyncedBookmarksMirror(db, options);
+  }
+
+  /**
+   * Returns the newer of the bookmarks collection last modified time, or the
+   * server modified time of the newest record. The bookmarks engine uses this
+   * timestamp as the "high water mark" for all downloaded records. Each sync
+   * downloads and stores records that are strictly newer than this time.
+   *
+   * @return {Number}
+   *         The high water mark time, in seconds.
+   */
+  async getCollectionHighWaterMark() {
+    // The first case, where we have records with server modified times newer
+    // than the collection last modified time, occurs when a sync is interrupted
+    // before we call `setCollectionLastModified`. We subtract one second, the
+    // maximum time precision guaranteed by the server, so that we don't miss
+    // other records with the same time as the newest one we downloaded.
+    let rows = await this.db.execute(`
+      SELECT MAX(
+        IFNULL((SELECT MAX(serverModified) - 1000 FROM items), 0),
+        IFNULL((SELECT CAST(value AS INTEGER) FROM meta
+                WHERE key = :modifiedKey), 0)
+      ) AS highWaterMark`,
+      { modifiedKey: SyncedBookmarksMirror.META.MODIFIED });
+    let highWaterMark = rows[0].getResultByName("highWaterMark");
+    return highWaterMark / 1000;
+  }
+
+  /**
+   * Updates the bookmarks collection last modified time. Note that this may
+   * be newer than the modified time of the most recent record.
+   *
+   * @param {Number|String} lastModifiedSeconds
+   *        The collection last modified time, in seconds.
+   */
+  async setCollectionLastModified(lastModifiedSeconds) {
+    let lastModified = lastModifiedSeconds * 1000;
+    if (!Number.isFinite(lastModified)) {
+      throw new TypeError("Invalid collection last modified time");
+    }
+    await this.db.executeBeforeShutdown(
+      "SyncedBookmarksMirror: setCollectionLastModified",
+      db => db.execute(`
+        REPLACE INTO meta(key, value)
+        VALUES(:modifiedKey, :lastModified)`,
+        { modifiedKey: SyncedBookmarksMirror.META.MODIFIED, lastModified })
+    );
+  }
+
+  /**
+   * Stores incoming or uploaded Sync records in the mirror. Rejects if any
+   * records are invalid.
+   *
+   * @param {PlacesItem[]} records
+   *        Sync records to store in the mirror.
+   * @param {Boolean} [options.needsMerge]
+   *        Indicates if the records were changed remotely since the last sync,
+   *        and should be merged into the local tree. This option is set to
+   *       `true` for incoming records, and `false` for successfully uploaded
+   *        records. Tests can also pass `false` to set up an existing mirror.
+   */
+  async store(records, { needsMerge = true } = {}) {
+    let options = { needsMerge };
+    await this.db.executeBeforeShutdown(
+      "SyncedBookmarksMirror: store",
+      db => db.executeTransaction(async () => {
+        for (let record of records) {
+          switch (record.type) {
+            case "bookmark":
+              MirrorLog.trace("Storing bookmark in mirror", record.cleartext);
+              await this.storeRemoteBookmark(record, options);
+              continue;
+
+            case "query":
+              MirrorLog.trace("Storing query in mirror", record.cleartext);
+              await this.storeRemoteQuery(record, options);
+              continue;
+
+            case "folder":
+              MirrorLog.trace("Storing folder in mirror", record.cleartext);
+              await this.storeRemoteFolder(record, options);
+              continue;
+
+            case "livemark":
+              MirrorLog.trace("Storing livemark in mirror", record.cleartext);
+              await this.storeRemoteLivemark(record, options);
+              continue;
+
+            case "separator":
+              MirrorLog.trace("Storing separator in mirror", record.cleartext);
+              await this.storeRemoteSeparator(record, options);
+              continue;
+
+            default:
+              if (record.deleted) {
+                MirrorLog.trace("Storing tombstone in mirror",
+                                record.cleartext);
+                await this.storeRemoteTombstone(record, options);
+                continue;
+              }
+          }
+          MirrorLog.warn("Ignoring record with unknown type", record.type);
+          this.recordTelemetryEvent("mirror", "ignore", "unknown",
+                                    { why: "kind" });
+        }
+      })
+    );
+  }
+
+  /**
+   * Builds a complete merged tree from the local and remote trees, resolves
+   * value and structure conflicts, dedupes local items, applies the merged
+   * tree back to Places, and notifies observers about the changes.
+   *
+   * Merging and application happen in a transaction, meaning code that uses the
+   * main Places connection, including the UI, will fail to write to the
+   * database until the transaction commits. Asynchronous consumers will retry
+   * on `SQLITE_BUSY`; synchronous consumers will fail after waiting for 100ms.
+   * See bug 1305563, comment 122 for details.
+   *
+   * @param  {Number} [options.localTimeSeconds]
+   *         The current local time, in seconds.
+   * @param  {Number} [options.remoteTimeSeconds]
+   *         The current server time, in seconds.
+   * @return {Object.<String, BookmarkChangeRecord>}
+   *         A changeset containing locally changed and reconciled records to
+   *         upload to the server, and to store in the mirror once upload
+   *         succeeds.
+   */
+  async apply({ localTimeSeconds = Date.now() / 1000,
+                remoteTimeSeconds = 0 } = {}) {
+    // We intentionally don't use `executeBeforeShutdown` in this function,
+    // since merging can take a while for large trees, and we don't want to
+    // block shutdown. Since all new items are in the mirror, we'll just try
+    // to merge again on the next sync.
+    let { missingParents, missingChildren } = await this.fetchRemoteOrphans();
+    if (missingParents.length) {
+      MirrorLog.debug("Temporarily reparenting remote items with missing " +
+                      "parents to unfiled", missingParents);
+      this.recordTelemetryEvent("mirror", "orphans", "parents",
+                                { count: String(missingParents.length) });
+    }
+    if (missingChildren.length) {
+      MirrorLog.debug("Remote tree missing items", missingChildren);
+      this.recordTelemetryEvent("mirror", "orphans", "children",
+                                { count: String(missingChildren.length) });
+    }
+
+    // It's safe to build the remote tree outside the transaction because
+    // `fetchRemoteTree` doesn't join to Places, only Sync writes to the
+    // mirror, and we're holding the Sync lock at this point.
+    MirrorLog.debug("Building remote tree from mirror");
+    let remoteTree = await this.fetchRemoteTree(remoteTimeSeconds);
+    MirrorLog.trace("Built remote tree from mirror", remoteTree);
+
+    let observersToNotify = new BookmarkObserverRecorder(this.db);
+
+    let changeRecords = await this.db.executeTransaction(async () => {
+      MirrorLog.debug("Building local tree from Places");
+      let localTree = await this.fetchLocalTree(localTimeSeconds);
+      MirrorLog.trace("Built local tree from Places", localTree);
+
+      MirrorLog.debug("Fetching content info for new mirror items");
+      let newRemoteContents = await this.fetchNewRemoteContents();
+
+      MirrorLog.debug("Fetching content info for new Places items");
+      let newLocalContents = await this.fetchNewLocalContents();
+
+      MirrorLog.debug("Building complete merged tree");
+      let merger = new BookmarkMerger(localTree, newLocalContents,
+                                      remoteTree, newRemoteContents);
+      let mergedRoot = merger.merge();
+      for (let { value, extra } of merger.telemetryEvents) {
+        this.recordTelemetryEvent("mirror", "merge", value, extra);
+      }
+
+      if (MirrorLog.level <= Log.Level.Trace) {
+        let newTreeRoot = mergedRoot.toBookmarkNode();
+        MirrorLog.trace("Built new merged tree", newTreeRoot);
+      }
+
+      // The merged tree should know about all items mentioned in the local
+      // and remote trees. Otherwise, it's incomplete, and we'll corrupt
+      // Places or lose data on the server if we try to apply it.
+      if (!merger.subsumes(localTree)) {
+        throw new SyncedBookmarksMirror.ConsistencyError(
+          "Merged tree doesn't mention all items from local tree");
+      }
+      if (!merger.subsumes(remoteTree)) {
+        throw new SyncedBookmarksMirror.ConsistencyError(
+          "Merged tree doesn't mention all items from remote tree");
+      }
+
+      MirrorLog.debug("Applying merged tree");
+      let localDeletions = Array.from(merger.deleteLocally).map(guid =>
+        ({ guid, level: localTree.levelForGuid(guid) })
+      );
+      let remoteDeletions = Array.from(merger.deleteRemotely);
+      await this.updateLocalItemsInPlaces(mergedRoot, localDeletions,
+                                          remoteDeletions);
+
+      // At this point, the database is consistent, and we can fetch info to
+      // pass to observers. Note that we can't fetch observer info in the
+      // triggers above, because the structure might not be complete yet. An
+      // incomplete structure might cause us to miss or record wrong parents and
+      // positions.
+
+      MirrorLog.debug("Recording observer notifications");
+      await this.noteObserverChanges(observersToNotify);
+
+      MirrorLog.debug("Staging locally changed items for upload");
+      await this.stageItemsToUpload();
+
+      MirrorLog.debug("Fetching records for local items to upload");
+      let changeRecords = await this.fetchLocalChangeRecords();
+
+      await this.db.execute(`DELETE FROM mergeStates`);
+      await this.db.execute(`DELETE FROM itemsAdded`);
+      await this.db.execute(`DELETE FROM guidsChanged`);
+      await this.db.execute(`DELETE FROM itemsChanged`);
+      await this.db.execute(`DELETE FROM itemsRemoved`);
+      await this.db.execute(`DELETE FROM itemsMoved`);
+      await this.db.execute(`DELETE FROM annosChanged`);
+      await this.db.execute(`DELETE FROM keywordsChanged`);
+      await this.db.execute(`DELETE FROM itemsToUpload`);
+
+      return changeRecords;
+    }, this.db.TRANSACTION_IMMEDIATE);
+
+    MirrorLog.debug("Replaying recorded observer notifications");
+    try {
+      await observersToNotify.notifyAll();
+    } catch (ex) {
+      MirrorLog.error("Error notifying Places observers", ex);
+    }
+
+    return changeRecords;
+  }
+
+  /**
+   * Discards the mirror contents. This is called when the user is node
+   * reassigned, disables the bookmarks engine, or signs out.
+   */
+  async reset() {
+    await this.db.executeBeforeShutdown(
+      "SyncedBookmarksMirror: reset",
+      async function(db) {
+        await db.executeTransaction(async function() {
+          await db.execute(`DELETE FROM meta`);
+          await db.execute(`DELETE FROM items`);
+          await db.execute(`DELETE FROM urls`);
+
+          // Since we need to reset the modified times for the syncable roots,
+          // we simply delete and recreate them.
+          await createMirrorRoots(db);
+        });
+      }
+    );
+  }
+
+  /**
+   * Fetches the GUIDs of all items in the remote tree that need to be merged
+   * into the local tree.
+   *
+   * @return {String[]}
+   *         Remotely changed GUIDs that need to be merged into Places.
+   */
+  async fetchUnmergedGuids() {
+    let rows = await this.db.execute(`SELECT guid FROM items WHERE needsMerge`);
+    return rows.map(row => row.getResultByName("guid"));
+  }
+
+  async storeRemoteBookmark(record, { needsMerge }) {
+    let guid = validateGuid(record.id);
+    if (!guid) {
+      MirrorLog.warn("Ignoring bookmark with invalid ID", record.id);
+      this.recordTelemetryEvent("mirror", "ignore", "bookmark",
+                                { why: "id" });
+      return;
+    }
+
+    let url = validateURL(record.bmkUri);
+    if (!url) {
+      MirrorLog.trace("Ignoring bookmark ${guid} with invalid URL ${url}",
+                      { guid, url: record.bmkUri });
+      this.recordTelemetryEvent("mirror", "ignore", "bookmark",
+                                { why: "url" });
+      return;
+    }
+
+    await this.maybeStoreRemoteURL(url);
+
+    let serverModified = determineServerModified(record);
+    let dateAdded = determineDateAdded(record);
+    let title = validateTitle(record.title);
+    let keyword = validateKeyword(record.keyword);
+    let description = validateDescription(record.description);
+    let loadInSidebar = record.loadInSidebar === true ? "1" : null;
+
+    await this.db.executeCached(`
+      REPLACE INTO items(guid, serverModified, needsMerge, kind,
+                         dateAdded, title, keyword,
+                         urlId, description, loadInSidebar)
+      VALUES(:guid, :serverModified, :needsMerge, :kind,
+             :dateAdded, NULLIF(:title, ""), :keyword,
+             (SELECT id FROM urls
+              WHERE hash = hash(:url) AND
+                    url = :url),
+             :description, :loadInSidebar)`,
+      { guid, serverModified, needsMerge,
+        kind: SyncedBookmarksMirror.KIND.BOOKMARK, dateAdded, title, keyword,
+        url: url.href, description, loadInSidebar });
+
+    let tags = record.tags;
+    if (tags && Array.isArray(tags)) {
+      for (let rawTag of tags) {
+        let tag = validateTag(rawTag);
+        if (!tag) {
+          continue;
+        }
+        await this.db.executeCached(`
+          INSERT INTO tags(itemId, tag)
+          SELECT id, :tag FROM items
+          WHERE guid = :guid`,
+          { tag, guid });
+      }
+    }
+  }
+
+  async storeRemoteQuery(record, { needsMerge }) {
+    let guid = validateGuid(record.id);
+    if (!guid) {
+      MirrorLog.warn("Ignoring query with invalid ID", record.id);
+      this.recordTelemetryEvent("mirror", "ignore", "query",
+                                { why: "id" });
+      return;
+    }
+
+    let url = validateURL(record.bmkUri);
+    if (!url) {
+      MirrorLog.trace("Ignoring query ${guid} with invalid URL ${url}",
+                      { guid, url: record.bmkUri });
+      this.recordTelemetryEvent("mirror", "ignore", "query",
+                                { why: "url" });
+      return;
+    }
+
+    await this.maybeStoreRemoteURL(url);
+
+    let serverModified = determineServerModified(record);
+    let dateAdded = determineDateAdded(record);
+    let title = validateTitle(record.title);
+    let tagFolderName = validateTag(record.folderName);
+    let description = validateDescription(record.description);
+    let smartBookmarkName = typeof record.queryId == "string" ?
+                            record.queryId : null;
+
+    await this.db.executeCached(`
+      REPLACE INTO items(guid, serverModified, needsMerge, kind,
+                         dateAdded, title, tagFolderName,
+                         urlId, description, smartBookmarkName)
+      VALUES(:guid, :serverModified, :needsMerge, :kind,
+             :dateAdded, NULLIF(:title, ""), :tagFolderName,
+             (SELECT id FROM urls
+              WHERE hash = hash(:url) AND
+                    url = :url),
+             :description, :smartBookmarkName)`,
+      { guid, serverModified, needsMerge,
+        kind: SyncedBookmarksMirror.KIND.QUERY, dateAdded, title, tagFolderName,
+        url: url.href, description, smartBookmarkName });
+  }
+
+  async storeRemoteFolder(record, { needsMerge }) {
+    let guid = validateGuid(record.id);
+    if (!guid) {
+      MirrorLog.warn("Ignoring folder with invalid ID", record.id);
+      this.recordTelemetryEvent("mirror", "ignore", "folder",
+                                { why: "id" });
+      return;
+    }
+    if (guid == PlacesUtils.bookmarks.rootGuid) {
+      // The Places root shouldn't be synced at all.
+      MirrorLog.warn("Ignoring Places root record", record.cleartext);
+      this.recordTelemetryEvent("mirror", "ignore", "folder",
+                                { why: "root" });
+      return;
+    }
+
+    let serverModified = determineServerModified(record);
+    let dateAdded = determineDateAdded(record);
+    let title = validateTitle(record.title);
+    let description = validateDescription(record.description);
+
+    await this.db.executeCached(`
+      REPLACE INTO items(guid, serverModified, needsMerge, kind,
+                         dateAdded, title, description)
+      VALUES(:guid, :serverModified, :needsMerge, :kind,
+             :dateAdded, NULLIF(:title, ""),
+             :description)`,
+      { guid, serverModified, needsMerge, kind: SyncedBookmarksMirror.KIND.FOLDER,
+        dateAdded, title, description });
+
+    let children = record.children;
+    if (children && Array.isArray(children)) {
+      for (let position = 0; position < children.length; ++position) {
+        let childRecordId = children[position];
+        let childGuid = validateGuid(childRecordId);
+        if (!childGuid) {
+          MirrorLog.warn("Ignoring child of folder ${parentGuid} with " +
+                         "invalid ID ${childRecordId}", { parentGuid: guid,
+                                                          childRecordId });
+          this.recordTelemetryEvent("mirror", "ignore", "child",
+                                    { why: "id" });
+          continue;
+        }
+        await this.db.executeCached(`
+          REPLACE INTO structure(guid, parentGuid, position)
+          VALUES(:childGuid, :parentGuid, :position)`,
+          { childGuid, parentGuid: guid, position });
+      }
+    }
+  }
+
+  async storeRemoteLivemark(record, { needsMerge }) {
+    let guid = validateGuid(record.id);
+    if (!guid) {
+      MirrorLog.warn("Ignoring livemark with invalid ID", record.id);
+      this.recordTelemetryEvent("mirror", "ignore", "livemark",
+                                { why: "id" });
+      return;
+    }
+
+    let feedURL = validateURL(record.feedUri);
+    if (!feedURL) {
+      MirrorLog.trace("Ignoring livemark ${guid} with invalid feed URL ${url}",
+                      { guid, url: record.feedUri });
+      this.recordTelemetryEvent("mirror", "ignore", "livemark",
+                                { why: "feed" });
+      return;
+    }
+
+    let serverModified = determineServerModified(record);
+    let dateAdded = determineDateAdded(record);
+    let title = validateTitle(record.title);
+    let description = validateDescription(record.description);
+    let siteURL = validateURL(record.siteUri);
+
+    await this.db.executeCached(`
+      REPLACE INTO items(guid, serverModified, needsMerge, kind, dateAdded,
+                         title, description, feedURL, siteURL)
+      VALUES(:guid, :serverModified, :needsMerge, :kind, :dateAdded,
+             NULLIF(:title, ""), :description, :feedURL, :siteURL)`,
+      { guid, serverModified, needsMerge,
+        kind: SyncedBookmarksMirror.KIND.LIVEMARK,
+        dateAdded, title, description, feedURL: feedURL.href,
+        siteURL: siteURL ? siteURL.href : null });
+  }
+
+  async storeRemoteSeparator(record, { needsMerge }) {
+    let guid = validateGuid(record.id);
+    if (!guid) {
+      MirrorLog.warn("Ignoring separator with invalid ID", record.id);
+      this.recordTelemetryEvent("mirror", "ignore", "separator",
+                                { why: "id" });
+      return;
+    }
+
+    let serverModified = determineServerModified(record);
+    let dateAdded = determineDateAdded(record);
+
+    await this.db.executeCached(`
+      REPLACE INTO items(guid, serverModified, needsMerge, kind,
+                         dateAdded)
+      VALUES(:guid, :serverModified, :needsMerge, :kind,
+             :dateAdded)`,
+      { guid, serverModified, needsMerge, kind: SyncedBookmarksMirror.KIND.SEPARATOR,
+        dateAdded });
+  }
+
+  async storeRemoteTombstone(record, { needsMerge }) {
+    let guid = validateGuid(record.id);
+    if (!guid) {
+      MirrorLog.warn("Ignoring tombstone with invalid ID", record.id);
+      this.recordTelemetryEvent("mirror", "ignore", "tombstone",
+                                { why: "id" });
+      return;
+    }
+
+    if (PlacesUtils.bookmarks.userContentRoots.includes(guid)) {
+      MirrorLog.warn("Ignoring tombstone for syncable root", guid);
+      this.recordTelemetryEvent("mirror", "ignore", "tombstone",
+                                { why: "root" });
+      return;
+    }
+
+    await this.db.executeCached(`
+      REPLACE INTO items(guid, serverModified, needsMerge, isDeleted)
+      VALUES(:guid, :serverModified, :needsMerge, 1)`,
+      { guid, serverModified: determineServerModified(record), needsMerge });
+  }
+
+  async maybeStoreRemoteURL(url) {
+    await this.db.executeCached(`
+      INSERT OR IGNORE INTO urls(guid, url, hash, revHost)
+      VALUES(IFNULL((SELECT guid FROM urls
+                     WHERE hash = hash(:url) AND
+                                  url = :url),
+                    GENERATE_GUID()), :url, hash(:url), :revHost)`,
+      { url: url.href, revHost: PlacesUtils.getReversedHost(url) });
+  }
+
+  async fetchRemoteOrphans() {
+    let infos = {
+      missingParents: [],
+      missingChildren: [],
+    };
+
+    let orphanRows = await this.db.execute(`
+      SELECT v.guid AS guid, 1 AS missingParent, 0 AS missingChild
+      FROM items v
+      LEFT JOIN structure s ON s.guid = v.guid
+      WHERE NOT v.isDeleted AND
+            s.guid IS NULL
+      UNION ALL
+      SELECT s.guid AS guid, 0 AS missingParent, 1 AS missingChild
+      FROM structure s
+      LEFT JOIN items v ON v.guid = s.guid
+      WHERE v.guid IS NULL`);
+
+    for (let row of orphanRows) {
+      let guid = row.getResultByName("guid");
+      let missingParent = row.getResultByName("missingParent");
+      if (missingParent) {
+        infos.missingParents.push(guid);
+      }
+      let missingChild = row.getResultByName("missingChild");
+      if (missingChild) {
+        infos.missingChildren.push(guid);
+      }
+    }
+
+    return infos;
+  }
+
+  /**
+   * Builds a fully rooted, consistent tree from the items and tombstones in the
+   * mirror.
+   *
+   * @param  {Number} remoteTimeSeconds
+   *         The current server time, in seconds.
+   * @return {BookmarkTree}
+   *         The remote bookmark tree.
+   */
+  async fetchRemoteTree(remoteTimeSeconds) {
+    let remoteTree = new BookmarkTree(BookmarkNode.root());
+    let startTime = Cu.now();
+
+    // First, build a flat mapping of parents to children. The `LEFT JOIN`
+    // includes items orphaned by an interrupted upload on another device.
+    // We keep the orphans in "unfiled" until the other device returns and
+    // uploads the missing parent.
+    let itemRows = await this.db.execute(`
+      SELECT v.guid, IFNULL(s.parentGuid, :unfiledGuid) AS parentGuid,
+             IFNULL(s.position, -1) AS position, v.serverModified, v.kind,
+             v.needsMerge
+      FROM items v
+      LEFT JOIN structure s ON s.guid = v.guid
+      WHERE NOT v.isDeleted AND
+            v.guid <> :rootGuid
+      ORDER BY parentGuid, position = -1, position, v.guid`,
+      { rootGuid: PlacesUtils.bookmarks.rootGuid,
+        unfiledGuid: PlacesUtils.bookmarks.unfiledGuid });
+
+    let pseudoTree = new Map();
+    for (let row of itemRows) {
+      let parentGuid = row.getResultByName("parentGuid");
+      let node = BookmarkNode.fromRemoteRow(row, remoteTimeSeconds);
+      if (pseudoTree.has(parentGuid)) {
+        let nodes = pseudoTree.get(parentGuid);
+        nodes.push(node);
+      } else {
+        pseudoTree.set(parentGuid, [node]);
+      }
+    }
+
+    // Second, build a complete tree from the pseudo-tree. We could do these
+    // two steps in SQL, but it's extremely inefficient. An equivalent
+    // recursive query, with joins in the base and recursive clauses, takes
+    // 10 seconds for a mirror with 5k items. Building the pseudo-tree and
+    // the pseudo-tree and recursing in JS takes 30ms for 5k items.
+    inflateTree(remoteTree, pseudoTree, PlacesUtils.bookmarks.rootGuid);
+
+    // Note tombstones for remotely deleted items.
+    let tombstoneRows = await this.db.execute(`
+      SELECT guid FROM items
+      WHERE isDeleted AND
+            needsMerge`);
+
+    for (let row of tombstoneRows) {
+      let guid = row.getResultByName("guid");
+      remoteTree.noteDeleted(guid);
+    }
+
+    let elapsedTime = Cu.now() - startTime;
+    let totalRows = itemRows.length + tombstoneRows.length;
+    this.recordTelemetryEvent("mirror", "fetch", "remoteTree",
+                              { time: String(elapsedTime),
+                                count: String(totalRows) });
+
+    return remoteTree;
+  }
+
+  /**
+   * Fetches content info for all items in the mirror that changed since the
+   * last sync and don't exist locally.
+   *
+   * @return {Map.<String, BookmarkContent>}
+   *         Changed items in the mirror that don't exist in Places, keyed by
+   *         their GUIDs.
+   */
+  async fetchNewRemoteContents() {
+    let newRemoteContents = new Map();
+    let startTime = Cu.now();
+
+    let rows = await this.db.execute(`
+      SELECT v.guid, IFNULL(v.title, "") AS title, u.url, v.smartBookmarkName,
+             IFNULL(s.position, -1) AS position
+      FROM items v
+      LEFT JOIN urls u ON u.id = v.urlId
+      LEFT JOIN structure s ON s.guid = v.guid
+      LEFT JOIN moz_bookmarks b ON b.guid = v.guid
+      WHERE NOT v.isDeleted AND
+            v.needsMerge AND
+            b.guid IS NULL AND
+            IFNULL(s.parentGuid, :unfiledGuid) <> :rootGuid`,
+      { unfiledGuid: PlacesUtils.bookmarks.unfiledGuid,
+        rootGuid: PlacesUtils.bookmarks.rootGuid });
+
+    for (let row of rows) {
+      let guid = row.getResultByName("guid");
+      let content = BookmarkContent.fromRow(row);
+      newRemoteContents.set(guid, content);
+    }
+
+    let elapsedTime = Cu.now() - startTime;
+    this.recordTelemetryEvent("mirror", "fetch", "newRemoteContents",
+                              { time: String(elapsedTime),
+                                count: String(rows.length) });
+
+    return newRemoteContents;
+  }
+
+  /**
+   * Builds a fully rooted, consistent tree from the items and tombstones in
+   * Places.
+   *
+   * @param  {Number} localTimeSeconds
+   *         The current local time, in seconds.
+   * @return {BookmarkTree}
+   *         The local bookmark tree.
+   */
+  async fetchLocalTree(localTimeSeconds) {
+    let localTree = new BookmarkTree(BookmarkNode.root());
+    let startTime = Cu.now();
+
+    // This unsightly query collects all descendants and maps their Places types
+    // to the Sync record kinds. We start with the roots, and work our way down.
+    // The list of roots in `syncedItems` should be updated if we add new
+    // syncable roots to Places.
+    let itemRows = await this.db.execute(`
+      WITH RECURSIVE
+      syncedItems(id, level) AS (
+        SELECT b.id, 0 AS level FROM moz_bookmarks b
+        WHERE b.guid IN (:menuGuid, :toolbarGuid, :unfiledGuid, :mobileGuid)
+        UNION ALL
+        SELECT b.id, s.level + 1 AS level FROM moz_bookmarks b
+        JOIN syncedItems s ON s.id = b.parent
+      )
+      SELECT b.id, b.guid, p.guid AS parentGuid,
+             /* Map Places item types to Sync record kinds. */
+             (CASE b.type
+                WHEN :bookmarkType THEN (
+                  CASE SUBSTR((SELECT h.url FROM moz_places h
+                               WHERE h.id = b.fk), 1, 6)
+                  /* Queries are bookmarks with a "place:" URL scheme. */
+                  WHEN 'place:' THEN :queryKind
+                  ELSE :bookmarkKind END)
+                WHEN :folderType THEN (
+                  CASE WHEN EXISTS(
+                    /* Livemarks are folders with a feed URL annotation. */
+                    SELECT 1 FROM moz_items_annos a
+                    JOIN moz_anno_attributes n ON n.id = a.anno_attribute_id
+                    WHERE a.item_id = b.id AND
+                          n.name = :feedURLAnno
+                  ) THEN :livemarkKind
+                  ELSE :folderKind END)
+                ELSE :separatorKind END) AS kind,
+             b.lastModified, b.syncChangeCounter
+      FROM moz_bookmarks b
+      JOIN moz_bookmarks p ON p.id = b.parent
+      JOIN syncedItems s ON s.id = b.id
+      ORDER BY s.level, b.parent, b.position`,
+      { menuGuid: PlacesUtils.bookmarks.menuGuid,
+        toolbarGuid: PlacesUtils.bookmarks.toolbarGuid,
+        unfiledGuid: PlacesUtils.bookmarks.unfiledGuid,
+        mobileGuid: PlacesUtils.bookmarks.mobileGuid,
+        bookmarkType: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        queryKind: SyncedBookmarksMirror.KIND.QUERY,
+        bookmarkKind: SyncedBookmarksMirror.KIND.BOOKMARK,
+        folderType: PlacesUtils.bookmarks.TYPE_FOLDER,
+        feedURLAnno: PlacesUtils.LMANNO_FEEDURI,
+        livemarkKind: SyncedBookmarksMirror.KIND.LIVEMARK,
+        folderKind: SyncedBookmarksMirror.KIND.FOLDER,
+        separatorKind: SyncedBookmarksMirror.KIND.SEPARATOR });
+
+    for (let row of itemRows) {
+      let parentGuid = row.getResultByName("parentGuid");
+      let node = BookmarkNode.fromLocalRow(row, localTimeSeconds);
+      localTree.insert(parentGuid, node);
+    }
+
+    // Note tombstones for locally deleted items.
+    let tombstoneRows = await this.db.execute(`
+      SELECT guid FROM moz_bookmarks_deleted`);
+
+    for (let row of tombstoneRows) {
+      let guid = row.getResultByName("guid");
+      localTree.noteDeleted(guid);
+    }
+
+    let elapsedTime = Cu.now() - startTime;
+    let totalRows = itemRows.length + tombstoneRows.length;
+    this.recordTelemetryEvent("mirror", "fetch", "localTree",
+                              { time: String(elapsedTime),
+                                count: String(totalRows) });
+
+    return localTree;
+  }
+
+  /**
+   * Fetches content info for all NEW local items that don't exist in the
+   * mirror. We'll try to dedupe them to changed items with similar contents and
+   * different GUIDs in the mirror.
+   *
+   * @return {Map.<String, BookmarkContent>}
+   *         New items in Places that don't exist in the mirror, keyed by their
+   *         GUIDs.
+   */
+  async fetchNewLocalContents() {
+    let newLocalContents = new Map();
+    let startTime = Cu.now();
+
+    let rows = await this.db.execute(`
+      SELECT b.guid, IFNULL(b.title, "") AS title, h.url,
+             (SELECT a.content FROM moz_items_annos a
+              JOIN moz_anno_attributes n ON n.id = a.anno_attribute_id
+              WHERE a.item_id = b.id AND
+                    n.name = :smartBookmarkAnno) AS smartBookmarkName,
+             b.position
+      FROM moz_bookmarks b
+      JOIN moz_bookmarks p ON p.id = b.parent
+      LEFT JOIN moz_places h ON h.id = b.fk
+      LEFT JOIN items v ON v.guid = b.guid
+      WHERE v.guid IS NULL AND
+            p.guid <> :rootGuid AND
+            b.syncStatus <> :syncStatus`,
+      { smartBookmarkAnno: PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO,
+        rootGuid: PlacesUtils.bookmarks.rootGuid,
+        syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL });
+
+    for (let row of rows) {
+      let guid = row.getResultByName("guid");
+      let content = BookmarkContent.fromRow(row);
+      newLocalContents.set(guid, content);
+    }
+
+    let elapsedTime = Cu.now() - startTime;
+    this.recordTelemetryEvent("mirror", "fetch", "newLocalContents",
+                              { time: String(elapsedTime),
+                                count: String(rows.length) });
+
+    return newLocalContents;
+  }
+
+  /**
+   * Builds a temporary table with the merge states of all nodes in the merged
+   * tree, rewrites tag queries, and updates Places to match the merged tree.
+   *
+   * Conceptually, we examine the merge state of each item, and either keep the
+   * complete local state, take the complete remote state, or apply a new
+   * structure state and flag the item for reupload.
+   *
+   * Note that we update Places and flag items *before* upload, while iOS
+   * updates the mirror *after* a successful upload. This simplifies our
+   * implementation, though we lose idempotent merges. If upload is interrupted,
+   * the next sync won't distinguish between new merge states from the previous
+   * sync, and local changes. Since this is how Desktop behaved before
+   * structured application, that's OK. In the future, we can make this more
+   * like iOS.
+   *
+   * @param {MergedBookmarkNode} mergedRoot
+   *        The root of the merged bookmark tree.
+   * @param {Object[]} localDeletions
+   *        `{ guid, level }` tuples for items to remove from Places and flag as
+   *        merged.
+   * @param {String[]} remoteDeletions
+   *        Remotely deleted GUIDs that should be flagged as merged.
+   */
+  async updateLocalItemsInPlaces(mergedRoot, localDeletions, remoteDeletions) {
+    MirrorLog.debug("Setting up merge states table");
+    let mergeStatesParams = Array.from(mergedRoot.mergeStatesParams());
+    if (mergeStatesParams.length) {
+      await this.db.execute(`
+        INSERT INTO mergeStates(localGuid, mergedGuid, parentGuid, level,
+                                position, valueState, structureState)
+        VALUES(IFNULL(:localGuid, :mergedGuid), :mergedGuid, :parentGuid,
+               :level, :position, :valueState, :structureState)`,
+        mergeStatesParams);
+    }
+
+    MirrorLog.debug("Rewriting tag queries in mirror");
+    await this.rewriteRemoteTagQueries();
+
+    MirrorLog.debug("Inserting new URLs into Places");
+    await this.db.execute(`
+      INSERT OR IGNORE INTO moz_places(url, url_hash, rev_host, hidden,
+                                       frecency, guid)
+      SELECT u.url, u.hash, u.revHost, 0,
+             (CASE SUBSTR(u.url, 1, 6) WHEN 'place:' THEN 0 ELSE -1 END),
+             IFNULL(h.guid, u.guid)
+      FROM items v
+      JOIN urls u ON u.id = v.urlId
+      LEFT JOIN moz_places h ON h.url_hash = u.hash AND
+                                h.url = u.url
+      JOIN mergeStates r ON r.mergedGuid = v.guid
+      WHERE r.valueState = :valueState`,
+      { valueState: BookmarkMergeState.TYPE.REMOTE });
+    await this.db.execute(`DELETE FROM moz_updatehostsinsert_temp`);
+
+    // Deleting from `newRemoteItems` fires the `insertNewLocalItems` and
+    // `updateExistingLocalItems` triggers.
+    MirrorLog.debug("Updating value states for local bookmarks");
+    await this.db.execute(`DELETE FROM newRemoteItems`);
+
+    // Update the structure. The mirror stores structure info in a separate
+    // table, like iOS, while Places stores structure info on children. We don't
+    // check the parent's merge state here because our merged tree might
+    // diverge from the server if we're missing children, or moved children
+    // without parents to "unfiled". In that case, we *don't* want to reupload
+    // the new local structure to the server.
+    MirrorLog.debug("Updating structure states for local bookmarks");
+    await this.db.execute(`DELETE FROM newRemoteStructure`);
+
+    MirrorLog.debug("Removing remotely deleted items from Places");
+    for (let chunk of PlacesSyncUtils.chunkArray(localDeletions,
+      SQLITE_MAX_VARIABLE_NUMBER)) {
+
+      let guids = chunk.map(({ guid }) => guid);
+
+      // Record item removed notifications.
+      await this.db.execute(`
+        WITH
+        guidsWithLevelsToDelete(guid, level) AS (
+          VALUES ${chunk.map(({ level }) => `(?, ${level})`).join(",")}
+        )
+        INSERT INTO itemsRemoved(itemId, parentId, position, type, placeId,
+                                 guid, parentGuid, level)
+        SELECT b.id, b.parent, b.position, b.type, b.fk, b.guid, p.guid,
+               o.level
+        FROM moz_bookmarks b
+        JOIN moz_bookmarks p ON p.id = b.parent
+        JOIN guidsWithLevelsToDelete o ON o.guid = b.guid`,
+        guids);
+
+      let guidsParams = new Array(guids.length).fill("?").join(",");
+
+      // Recalculate frecencies.
+      await this.db.execute(`
+        UPDATE moz_places SET
+          frecency = -1
+        WHERE id IN (SELECT fk FROM moz_bookmarks
+                     WHERE guid IN (${guidsParams}))`,
+        guids);
+
+      // Remove annos for the deleted items.
+      await this.db.execute(`
+        DELETE FROM moz_items_annos
+        WHERE item_id = (SELECT id FROM moz_bookmarks
+                         WHERE guid IN (${guidsParams}))`,
+        guids);
+
+      // Remove any local tombstones for deleted items.
+      await this.db.execute(`
+        DELETE FROM moz_bookmarks_deleted
+        WHERE guid IN (${guidsParams})`,
+        guids);
+
+      await this.db.execute(`
+        DELETE FROM moz_bookmarks WHERE guid IN (${guidsParams})`,
+        guids);
+
+      // Flag locally deleted items as merged.
+      await this.db.execute(`
+        UPDATE items SET
+          needsMerge = 0
+        WHERE needsMerge AND
+              guid IN (${guidsParams})`,
+        guids);
+    }
+
+    MirrorLog.debug("Flagging remotely deleted items as merged");
+    for (let chunk of PlacesSyncUtils.chunkArray(remoteDeletions,
+      SQLITE_MAX_VARIABLE_NUMBER)) {
+
+      await this.db.execute(`
+        UPDATE items SET
+          needsMerge = 0
+        WHERE needsMerge AND
+              guid IN (${new Array(chunk.length).fill("?").join(",")})`,
+        chunk);
+    }
+  }
+
+  /**
+   * Creates local tag folders mentioned in remotely changed tag queries, then
+   * rewrites the query URLs in the mirror to point to the new local folders.
+   *
+   * This can be removed once bug 1293445 lands.
+   */
+  async rewriteRemoteTagQueries() {
+    // Create local tag folders that don't already exist. This fires the
+    // `tagLocalPlace` trigger.
+    await this.db.execute(`
+      INSERT INTO localTags(tag)
+      SELECT v.tagFolderName FROM items v
+      JOIN mergeStates r ON r.mergedGuid = v.guid
+      WHERE r.valueState = :valueState AND
+            v.tagFolderName NOT NULL`,
+      { valueState: BookmarkMergeState.TYPE.REMOTE });
+
+    let queryRows = await this.db.execute(`
+      SELECT u.id AS urlId, u.url, b.id AS newTagFolderId FROM urls u
+      JOIN items v ON v.urlId = u.id
+      JOIN mergeStates r ON r.mergedGuid = v.guid
+      JOIN moz_bookmarks b ON b.title = v.tagFolderName
+      JOIN moz_bookmarks p ON p.id = b.parent
+      WHERE p.guid = :tagsGuid AND
+            r.valueState = :valueState AND
+            v.kind = :queryKind AND
+            v.tagFolderName NOT NULL`,
+      { tagsGuid: PlacesUtils.bookmarks.tagsGuid,
+        valueState: BookmarkMergeState.TYPE.REMOTE,
+        queryKind: SyncedBookmarksMirror.KIND.QUERY });
+
+    let urlsParams = [];
+    for (let row of queryRows) {
+      let url = new URL(row.getResultByName("url"));
+      let tagQueryParams = new URLSearchParams(url.pathname);
+      let type = Number(tagQueryParams.get("type"));
+      if (type != Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) {
+        continue;
+      }
+
+      // Rewrite the query URL to point to the new folder.
+      let newTagFolderId = row.getResultByName("newTagFolderId");
+      tagQueryParams.set("folder", newTagFolderId);
+
+      let newURLHref = url.protocol + tagQueryParams;
+      urlsParams.push({
+        urlId: row.getResultByName("urlId"),
+        url: newURLHref,
+      });
+    }
+
+    if (urlsParams.length) {
+      await this.db.execute(`
+        UPDATE urls SET
+          url = :url,
+          hash = hash(:url)
+        WHERE id = :urlId`,
+        urlsParams);
+    }
+  }
+
+  /**
+   * Records Places observer notifications for removed, added, moved, and
+   * changed items.
+   *
+   * @param {BookmarkObserverRecorder} observersToNotify
+   */
+  async noteObserverChanges(observersToNotify) {
+    MirrorLog.debug("Recording observer notifications for removed items");
+    // `ORDER BY v.level DESC` sorts deleted children before parents, to ensure
+    // that we update caches in the correct order (bug 1297941). We also order
+    // by parent and position so that the notifications are well-ordered for
+    // tests.
+    let removedItemRows = await this.db.execute(`
+      SELECT v.itemId AS id, v.parentId, v.parentGuid, v.position, v.type,
+             h.url, v.guid, v.isUntagging
+      FROM itemsRemoved v
+      LEFT JOIN moz_places h ON h.id = v.placeId
+      ORDER BY v.level DESC, v.parentId, v.position`);
+    for (let row of removedItemRows) {
+      let info = {
+        id: row.getResultByName("id"),
+        parentId: row.getResultByName("parentId"),
+        position: row.getResultByName("position"),
+        type: row.getResultByName("type"),
+        urlHref: row.getResultByName("url"),
+        guid: row.getResultByName("guid"),
+        parentGuid: row.getResultByName("parentGuid"),
+        isUntagging: row.getResultByName("isUntagging"),
+      };
+      observersToNotify.noteItemRemoved(info);
+    }
+
+    MirrorLog.debug("Recording observer notifications for changed GUIDs");
+    let changedGuidRows = await this.db.execute(`
+      SELECT b.id, b.lastModified, b.type, b.guid AS newGuid,
+             c.oldGuid, p.id AS parentId, p.guid AS parentGuid
+      FROM guidsChanged c
+      JOIN moz_bookmarks b ON b.id = c.itemId
+      JOIN moz_bookmarks p ON p.id = b.parent
+      JOIN mergeStates r ON r.mergedGuid = b.guid
+      ORDER BY r.level, p.id, b.position`);
+    for (let row of changedGuidRows) {
+      let info = {
+        id: row.getResultByName("id"),
+        lastModified: row.getResultByName("lastModified"),
+        type: row.getResultByName("type"),
+        newGuid: row.getResultByName("newGuid"),
+        oldGuid: row.getResultByName("oldGuid"),
+        parentId: row.getResultByName("parentId"),
+        parentGuid: row.getResultByName("parentGuid"),
+      };
+      observersToNotify.noteGuidChanged(info);
+    }
+
+    MirrorLog.debug("Recording observer notifications for new items");
+    // We `LEFT JOIN` to `mergeStates` because `itemsAdded` may include tag
+    // folders and entries, which are not part of the merged tree structure, and
+    // so don't exist in `mergeStates`.
+    let newItemRows = await this.db.execute(`
+      SELECT b.id, p.id AS parentId, b.position, b.type, h.url,
+             IFNULL(b.title, "") AS title, b.dateAdded, b.guid,
+             p.guid AS parentGuid, n.isTagging
+      FROM itemsAdded n
+      JOIN moz_bookmarks b ON b.guid = n.guid
+      JOIN moz_bookmarks p ON p.id = b.parent
+      LEFT JOIN moz_places h ON h.id = b.fk
+      LEFT JOIN mergeStates r ON r.mergedGuid = b.guid
+      ORDER BY r.level, p.id, b.position`);
+    for (let row of newItemRows) {
+      let info = {
+        id: row.getResultByName("id"),
+        parentId: row.getResultByName("parentId"),
+        position: row.getResultByName("position"),
+        type: row.getResultByName("type"),
+        urlHref: row.getResultByName("url"),
+        title: row.getResultByName("title"),
+        dateAdded: row.getResultByName("dateAdded"),
+        guid: row.getResultByName("guid"),
+        parentGuid: row.getResultByName("parentGuid"),
+        isTagging: row.getResultByName("isTagging"),
+      };
+      observersToNotify.noteItemAdded(info);
+    }
+
+    MirrorLog.debug("Recording observer notifications for moved items");
+    let movedItemRows = await this.db.execute(`
+      SELECT b.id, b.guid, b.type, p.id AS newParentId, c.oldParentId,
+             p.guid AS newParentGuid, c.oldParentGuid,
+             b.position AS newPosition, c.oldPosition
+      FROM itemsMoved c
+      JOIN moz_bookmarks b ON b.id = c.itemId
+      JOIN moz_bookmarks p ON p.id = b.parent
+      JOIN mergeStates r ON r.mergedGuid = b.guid
+      ORDER BY r.level, newParentId, newPosition`);
+    for (let row of movedItemRows) {
+      let info = {
+        id: row.getResultByName("id"),
+        guid: row.getResultByName("guid"),
+        type: row.getResultByName("type"),
+        newParentId: row.getResultByName("newParentId"),
+        oldParentId: row.getResultByName("oldParentId"),
+        newParentGuid: row.getResultByName("newParentGuid"),
+        oldParentGuid: row.getResultByName("oldParentGuid"),
+        newPosition: row.getResultByName("newPosition"),
+        oldPosition: row.getResultByName("oldPosition"),
+      };
+      observersToNotify.noteItemMoved(info);
+    }
+
+    MirrorLog.debug("Recording observer notifications for changed items");
+    let changedItemRows = await this.db.execute(`
+      SELECT b.id, b.guid, b.lastModified, b.type,
+             IFNULL(b.title, "") AS newTitle,
+             IFNULL(c.oldTitle, "") AS oldTitle,
+             h.url AS newURL, i.url AS oldURL,
+             p.id AS parentId, p.guid AS parentGuid
+      FROM itemsChanged c
+      JOIN moz_bookmarks b ON b.id = c.itemId
+      JOIN moz_bookmarks p ON p.id = b.parent
+      JOIN mergeStates r ON r.mergedGuid = b.guid
+      LEFT JOIN moz_places h ON h.id = b.fk
+      LEFT JOIN moz_places i ON i.id = c.oldPlaceId
+      ORDER BY r.level, p.id, b.position`);
+    for (let row of changedItemRows) {
+      let info = {
+        id: row.getResultByName("id"),
+        guid: row.getResultByName("guid"),
+        lastModified: row.getResultByName("lastModified"),
+        type: row.getResultByName("type"),
+        newTitle: row.getResultByName("newTitle"),
+        oldTitle: row.getResultByName("oldTitle"),
+        newURLHref: row.getResultByName("newURL"),
+        oldURLHref: row.getResultByName("oldURL"),
+        parentId: row.getResultByName("parentId"),
+        parentGuid: row.getResultByName("parentGuid"),
+      };
+      observersToNotify.noteItemChanged(info);
+    }
+
+    MirrorLog.debug("Recording observer notifications for changed annos");
+    let annoRows = await this.db.execute(`
+      SELECT itemId, annoName, wasRemoved FROM annosChanged
+      ORDER BY itemId`);
+    for (let row of annoRows) {
+      let id = row.getResultByName("itemId");
+      let name = row.getResultByName("annoName");
+      if (row.getResultByName("wasRemoved")) {
+        observersToNotify.noteAnnoRemoved(id, name);
+      } else {
+        observersToNotify.noteAnnoSet(id, name);
+      }
+    }
+
+    MirrorLog.debug("Recording notifications for changed keywords");
+    // `ORDER BY k.ROWID` replays additions and deletions for the same keyword
+    // or URL in order.
+    let changedKeywordRows = await this.db.execute(`
+      SELECT b.id, IFNULL(k.keyword, "") AS keyword, b.lastModified, b.type,
+             p.id AS parentId, b.guid, p.guid AS parentGuid, h.url
+      FROM keywordsChanged k
+      JOIN moz_bookmarks b ON b.id = k.itemId
+      JOIN moz_bookmarks p ON p.id = b.parent
+      JOIN moz_places h ON h.id = k.placeId
+      ORDER BY k.ROWID`);
+    for (let row of changedKeywordRows) {
+      let info = {
+        id: row.getResultByName("id"),
+        keyword: row.getResultByName("keyword"),
+        lastModified: row.getResultByName("lastModified"),
+        type: row.getResultByName("type"),
+        parentId: row.getResultByName("parentId"),
+        guid: row.getResultByName("guid"),
+        parentGuid: row.getResultByName("parentGuid"),
+        urlHref: row.getResultByName("url"),
+      };
+      observersToNotify.noteKeywordChanged(info);
+    }
+  }
+
+  /**
+   * Stores a snapshot of all locally changed items in a temporary table for
+   * upload. This is called from within the merge transaction, to ensure that
+   * structure changes made during the sync don't cause us to upload an
+   * inconsistent tree.
+   *
+   * For an example of why we use a temporary table instead of reading directly
+   * from Places, consider a user adding a bookmark, then changing its parent
+   * folder. We first add the bookmark to the default folder, bump the change
+   * counter of the new bookmark and the default folder, then trigger a sync.
+   * Depending on how quickly the user picks the new parent, we might upload
+   * a record for the default folder, commit the move, then upload the bookmark.
+   * We'll still upload the new parent on the next sync, but, in the meantime,
+   * we've introduced a parent-child disagreement. This can also happen if the
+   * user moves many items between folders.
+   *
+   * Conceptually, `itemsToUpload` is a transient "view" of locally changed
+   * items. The change counter in Places is the persistent record of items that
+   * we need to upload, so, if upload is interrupted or fails, we'll stage the
+   * items again on the next sync.
+   */
+  async stageItemsToUpload() {
+    // Stage all locally changed items for upload, along with any remotely
+    // changed records with older local creation dates. These are tracked
+    // "weakly", in the in-memory table only. If the upload is interrupted
+    // or fails, we won't reupload the record on the next sync.
+    await this.db.execute(`
+      WITH RECURSIVE
+      syncedItems(id, level) AS (
+        SELECT b.id, 0 AS level FROM moz_bookmarks b
+        WHERE b.guid IN (:menuGuid, :toolbarGuid, :unfiledGuid, :mobileGuid)
+        UNION ALL
+        SELECT b.id, s.level + 1 AS level FROM moz_bookmarks b
+        JOIN syncedItems s ON s.id = b.parent
+      ),
+      annos(itemId, name, content) AS (
+        SELECT a.item_id, n.name, a.content FROM moz_items_annos a
+        JOIN moz_anno_attributes n ON n.id = a.anno_attribute_id
+      )
+      INSERT INTO itemsToUpload(guid, syncChangeCounter, parentGuid,
+                                parentTitle, dateAdded, type, title, isQuery,
+                                url, tags, description, loadInSidebar,
+                                smartBookmarkName, keyword, feedURL, siteURL,
+                                position)
+      SELECT b.guid, b.syncChangeCounter, p.guid, p.title, b.dateAdded, b.type,
+             b.title, IFNULL(SUBSTR(h.url, 1, 6) = 'place:', 0), h.url,
+             (SELECT GROUP_CONCAT(t.title, ',') FROM moz_bookmarks e
+              JOIN moz_bookmarks t ON t.id = e.parent
+              JOIN moz_bookmarks r ON r.id = t.parent
+              WHERE r.guid = :tagsGuid AND
+                    e.fk = h.id),
+             (SELECT content FROM annos WHERE itemId = b.id AND
+                                              name = :descriptionAnno),
+             IFNULL((SELECT content FROM annos WHERE itemId = b.id AND
+                                                     name = :sidebarAnno), 0),
+             (SELECT content FROM annos WHERE itemId = b.id AND
+                                              name = :smartBookmarkAnno),
+             (SELECT keyword FROM moz_keywords WHERE place_id = h.id),
+             (SELECT content FROM annos WHERE itemId = b.id AND
+                                              name = :feedURLAnno),
+             (SELECT content FROM annos WHERE itemId = b.id AND
+                                              name = :siteURLAnno),
+             b.position
+      FROM moz_bookmarks b
+      JOIN moz_bookmarks p ON p.id = b.parent
+      JOIN syncedItems s ON s.id = b.id
+      LEFT JOIN moz_places h ON h.id = b.fk
+      JOIN mergeStates r ON r.mergedGuid = b.guid
+      LEFT JOIN items v ON v.guid = r.mergedGuid
+      WHERE b.syncChangeCounter >= 1 OR
+            (r.valueState = :valueState AND
+              b.dateAdded < v.dateAdded)`,
+      { menuGuid: PlacesUtils.bookmarks.menuGuid,
+        toolbarGuid: PlacesUtils.bookmarks.toolbarGuid,
+        unfiledGuid: PlacesUtils.bookmarks.unfiledGuid,
+        mobileGuid: PlacesUtils.bookmarks.mobileGuid,
+        tagsGuid: PlacesUtils.bookmarks.tagsGuid,
+        descriptionAnno: PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO,
+        sidebarAnno: PlacesSyncUtils.bookmarks.SIDEBAR_ANNO,
+        smartBookmarkAnno: PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO,
+        feedURLAnno: PlacesUtils.LMANNO_FEEDURI,
+        siteURLAnno: PlacesUtils.LMANNO_SITEURI,
+        valueState: BookmarkMergeState.TYPE.REMOTE });
+
+    // Record tag folder names for tag queries. Parsing query URLs one by one
+    // is inefficient, but queries aren't common today, and we can remove this
+    // logic entirely once bug 1293445 lands.
+    let queryRows = await this.db.execute(`
+      SELECT guid, url FROM itemsToUpload
+      WHERE isQuery`);
+
+    let tagFolderNameParams = [];
+    for (let row of queryRows) {
+      let url = new URL(row.getResultByName("url"));
+      let tagQueryParams = new URLSearchParams(url.pathname);
+      let type = Number(tagQueryParams.get("type"));
+      if (type == Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) {
+        continue;
+      }
+      let tagFolderId = Number(tagQueryParams.get("folder"));
+      tagFolderNameParams.push({
+        guid: row.getResultByName("guid"),
+        tagFolderId,
+        folderType: PlacesUtils.bookmarks.TYPE_FOLDER,
+      });
+    }
+
+    if (tagFolderNameParams.length) {
+      await this.db.execute(`
+        UPDATE itemsToUpload SET
+          tagFolderName = (SELECT b.title FROM moz_bookmarks b
+                           WHERE b.id = :tagFolderId AND
+                                 b.type = :folderType)
+        WHERE guid = :guid`);
+    }
+
+    // Record the child GUIDs of locally changed folders, which we use to
+    // populate the `children` array in the record.
+    await this.db.execute(`
+      INSERT INTO structureToUpload(guid, parentGuid, position)
+      SELECT b.guid, p.guid, b.position FROM moz_bookmarks b
+      JOIN moz_bookmarks p ON p.id = b.parent
+      JOIN itemsToUpload o ON o.guid = p.guid`);
+
+    // Finally, stage tombstones for deleted items. Ignore conflicts if we have
+    // tombstones for undeleted items; Places Maintenance should clean these up.
+    await this.db.execute(`
+      INSERT OR IGNORE INTO itemsToUpload(guid, syncChangeCounter, isDeleted,
+                                          dateAdded)
+      SELECT guid, 1, 1, dateRemoved FROM moz_bookmarks_deleted`);
+  }
+
+  /**
+   * Inflates Sync records for all staged outgoing items.
+   *
+   * @return {Object.<String, BookmarkChangeRecord>}
+   *         A changeset containing Sync record cleartexts for outgoing items
+   *         and tombstones, keyed by their Sync record IDs.
+   */
+  async fetchLocalChangeRecords() {
+    let changeRecords = {};
+
+    let itemRows = await this.db.execute(`
+      SELECT syncChangeCounter, guid, isDeleted, type, isQuery,
+             smartBookmarkName, IFNULL(tagFolderName, "") AS tagFolderName,
+             loadInSidebar, keyword, tags, url, IFNULL(title, "") AS title,
+             description, feedURL, siteURL, position, parentGuid,
+             IFNULL(parentTitle, "") AS parentTitle, dateAdded
+      FROM itemsToUpload`);
+
+    for (let row of itemRows) {
+      let syncChangeCounter = row.getResultByName("syncChangeCounter");
+
+      let guid = row.getResultByName("guid");
+      let recordId = PlacesSyncUtils.bookmarks.guidToRecordId(guid);
+
+      // Tombstones don't carry additional properties.
+      let isDeleted = row.getResultByName("isDeleted");
+      if (isDeleted) {
+        changeRecords[recordId] = new BookmarkChangeRecord(syncChangeCounter, {
+          id: recordId,
+          deleted: true,
+        });
+        continue;
+      }
+
+      let parentGuid = row.getResultByName("parentGuid");
+      let parentRecordId = PlacesSyncUtils.bookmarks.guidToRecordId(parentGuid);
+      let dateAdded = PlacesUtils.toDate(
+        row.getResultByName("dateAdded")).getTime();
+
+      let type = row.getResultByName("type");
+      switch (type) {
+        case PlacesUtils.bookmarks.TYPE_BOOKMARK: {
+          let isQuery = row.getResultByName("isQuery");
+          if (isQuery) {
+            let queryCleartext = {
+              id: recordId,
+              type: "query",
+              // We ignore `parentid` and use the parent's `children`, but older
+              // Desktops and Android use `parentid` as the canonical parent.
+              // iOS is stricter and requires both `children` and `parentid` to
+              // match.
+              parentid: parentRecordId,
+              // Older Desktops use `hasDupe` and `parentName` for deduping.
+              hasDupe: false,
+              parentName: row.getResultByName("parentTitle"),
+              dateAdded,
+              bmkUri: row.getResultByName("url"),
+              title: row.getResultByName("title"),
+              queryId: row.getResultByName("smartBookmarkName"),
+              folderName: row.getResultByName("tagFolderName"),
+            };
+            let description = row.getResultByName("description");
+            if (description) {
+              queryCleartext.description = description;
+            }
+            changeRecords[recordId] = new BookmarkChangeRecord(
+              syncChangeCounter, queryCleartext);
+            continue;
+          }
+
+          let bookmarkCleartext = {
+            id: recordId,
+            type: "bookmark",
+            parentid: parentRecordId,
+            hasDupe: false,
+            parentName: row.getResultByName("parentTitle"),
+            dateAdded,
+            bmkUri: row.getResultByName("url"),
+            title: row.getResultByName("title"),
+          };
+          let description = row.getResultByName("description");
+          if (description) {
+            bookmarkCleartext.description = description;
+          }
+          let loadInSidebar = row.getResultByName("loadInSidebar");
+          if (loadInSidebar) {
+            bookmarkCleartext.loadInSidebar = true;
+          }
+          let keyword = row.getResultByName("keyword");
+          if (keyword) {
+            bookmarkCleartext.keyword = keyword;
+          }
+          let tags = row.getResultByName("tags");
+          if (tags) {
+            bookmarkCleartext.tags = tags.split(",");
+          }
+          changeRecords[recordId] = new BookmarkChangeRecord(
+            syncChangeCounter, bookmarkCleartext);
+          continue;
+        }
+
+        case PlacesUtils.bookmarks.TYPE_FOLDER: {
+          let feedURLHref = row.getResultByName("feedURL");
+          if (feedURLHref) {
+            // Places stores livemarks as folders with feed and site URL annos.
+            // See bug 1072833 for discussion about changing them to queries.
+            let livemarkCleartext = {
+              id: recordId,
+              type: "livemark",
+              parentid: parentRecordId,
+              hasDupe: false,
+              parentName: row.getResultByName("parentTitle"),
+              dateAdded,
+              title: row.getResultByName("title"),
+              feedUri: feedURLHref,
+            };
+            let description = row.getResultByName("description");
+            if (description) {
+              livemarkCleartext.description = description;
+            }
+            let siteURLHref = row.getResultByName("siteURL");
+            if (siteURLHref) {
+              livemarkCleartext.siteUri = siteURLHref;
+            }
+            changeRecords[recordId] = new BookmarkChangeRecord(
+              syncChangeCounter, livemarkCleartext);
+            continue;
+          }
+
+          let folderCleartext = {
+            id: recordId,
+            type: "folder",
+            parentid: parentRecordId,
+            hasDupe: false,
+            parentName: row.getResultByName("parentTitle"),
+            dateAdded,
+            title: row.getResultByName("title"),
+          };
+          let description = row.getResultByName("description");
+          if (description) {
+            folderCleartext.description = description;
+          }
+          let childGuidRows = await this.db.executeCached(`
+            SELECT guid FROM structureToUpload
+            WHERE parentGuid = :guid
+            ORDER BY position`,
+            { guid });
+          folderCleartext.children = childGuidRows.map(row => {
+            let childGuid = row.getResultByName("guid");
+            return PlacesSyncUtils.bookmarks.guidToRecordId(childGuid);
+          });
+          changeRecords[recordId] = new BookmarkChangeRecord(
+            syncChangeCounter, folderCleartext);
+          continue;
+        }
+
+        case PlacesUtils.bookmarks.TYPE_SEPARATOR: {
+          let separatorCleartext = {
+            id: recordId,
+            type: "separator",
+            parentid: parentRecordId,
+            hasDupe: false,
+            parentName: row.getResultByName("parentTitle"),
+            dateAdded,
+            // Older Desktops use `pos` for deduping.
+            pos: row.getResultByName("position"),
+          };
+          changeRecords[recordId] = new BookmarkChangeRecord(
+            syncChangeCounter, separatorCleartext);
+          continue;
+        }
+
+        default:
+          throw new TypeError("Can't create record for unknown Places item");
+      }
+    }
+
+    return changeRecords;
+  }
+
+  /**
+   * Closes the mirror database connection. This is called automatically on
+   * shutdown, but may also be called explicitly when the mirror is no longer
+   * needed.
+   */
+  finalize() {
+    if (!this.finalizePromise) {
+      this.finalizePromise = (async () => {
+        await this.db.close();
+        this.finalizeAt.removeBlocker(this.finalizeBound);
+      })();
+    }
+    return this.finalizePromise;
+  }
+}
+
+this.SyncedBookmarksMirror = SyncedBookmarksMirror;
+
+/** Synced item kinds. Each corresponds to a Sync record type. */
+SyncedBookmarksMirror.KIND = {
+  BOOKMARK: 1,
+  QUERY: 2,
+  FOLDER: 3,
+  LIVEMARK: 4,
+  SEPARATOR: 5,
+};
+
+/** Valid key types for the key-value `meta` table. */
+SyncedBookmarksMirror.META = {
+  MODIFIED: 1,
+};
+
+/**
+ * An error thrown when the merge can't proceed because the local or remote
+ * tree is inconsistent.
+ */
+SyncedBookmarksMirror.ConsistencyError =
+  class ConsistencyError extends Error {};
+
+// Indicates if the mirror should be replaced because the database file is
+// corrupt.
+function isDatabaseCorrupt(error) {
+  return error instanceof Ci.mozIStorageError &&
+         (error.result == Ci.mozIStorageError.CORRUPT ||
+          error.result == Ci.mozIStorageError.NOTADB);
+}
+
+/**
+ * Migrates the mirror database schema to the latest version.
+ *
+ * @param {Sqlite.OpenedConnection} db
+ *        The mirror database connection.
+ */
+function migrateMirrorSchema(db) {
+  return db.executeTransaction(async function() {
+    let currentSchemaVersion = await db.getSchemaVersion("mirror");
+    if (currentSchemaVersion < 1) {
+      await initializeMirrorDatabase(db);
+    }
+    // Downgrading from a newer profile to an older profile rolls back the
+    // schema version, but leaves all new columns in place. We'll run the
+    // migration logic again on the next upgrade.
+    await db.setSchemaVersion(MIRROR_SCHEMA_VERSION, "mirror");
+  });
+}
+
+/**
+ * Initializes a new mirror database, creating persistent tables, indexes, and
+ * roots.
+ *
+ * @param {Sqlite.OpenedConnection} db
+ *        The mirror database connection.
+ */
+async function initializeMirrorDatabase(db) {
+  // Key-value metadata table. Currently stores just the server collection
+  // last modified time.
+  await db.execute(`CREATE TABLE mirror.meta(
+    key INTEGER PRIMARY KEY,
+    value NOT NULL
+    CHECK(key = ${SyncedBookmarksMirror.META.MODIFIED})
+  )`);
+
+  await db.execute(`CREATE TABLE mirror.items(
+    id INTEGER PRIMARY KEY,
+    guid TEXT UNIQUE NOT NULL,
+    /* The server modified time, in milliseconds. */
+    serverModified INTEGER NOT NULL DEFAULT 0,
+    needsMerge BOOLEAN NOT NULL DEFAULT 0,
+    isDeleted BOOLEAN NOT NULL DEFAULT 0,
+    kind INTEGER NOT NULL DEFAULT -1,
+    /* The creation date, in microseconds. */
+    dateAdded INTEGER NOT NULL DEFAULT 0,
+    title TEXT,
+    urlId INTEGER REFERENCES urls(id)
+                  ON DELETE SET NULL,
+    keyword TEXT,
+    tagFolderName TEXT,
+    description TEXT,
+    loadInSidebar BOOLEAN,
+    smartBookmarkName TEXT,
+    feedURL TEXT,
+    siteURL TEXT,
+    /* Only bookmarks and queries must have URLs. */
+    CHECK(CASE WHEN kind IN (${[
+                      SyncedBookmarksMirror.KIND.BOOKMARK,
+                      SyncedBookmarksMirror.KIND.QUERY,
+                    ].join(",")}) THEN urlId NOT NULL
+               ELSE urlId IS NULL END)
+  )`);
+
+  await db.execute(`CREATE TABLE mirror.structure(
+    guid TEXT NOT NULL PRIMARY KEY,
+    parentGuid TEXT NOT NULL REFERENCES items(guid)
+                             ON DELETE CASCADE,
+    position INTEGER NOT NULL
+  ) WITHOUT ROWID`);
+
+  await db.execute(`CREATE TABLE mirror.urls(
+    id INTEGER PRIMARY KEY,
+    guid TEXT NOT NULL,
+    url TEXT NOT NULL,
+    hash INTEGER NOT NULL,
+    revHost TEXT NOT NULL
+  )`);
+
+  await db.execute(`CREATE TABLE mirror.tags(
+    itemId INTEGER NOT NULL REFERENCES items(id)
+                            ON DELETE CASCADE,
+    tag TEXT NOT NULL
+  )`);
+
+  await db.execute(`CREATE INDEX mirror.urlHashes ON urls(hash)`);
+
+  await db.execute(`CREATE INDEX mirror.locations ON structure(
+    parentGuid,
+    position
+  )`);
+
+  await createMirrorRoots(db);
+}
+
+/**
+ * Sets up the syncable roots. All items in the mirror should descend from these
+ * roots. If we ever add new syncable roots to Places, this function should also
+ * be updated to create them in the mirror.
+ *
+ * @param {Sqlite.OpenedConnection} db
+ *        The mirror database connection.
+ */
+async function createMirrorRoots(db) {
+  const syncableRoots = [{
+    guid: PlacesUtils.bookmarks.rootGuid,
+    // The Places root is its own parent, to satisfy the foreign key and
+    // `NOT NULL` constraints on `structure`.
+    parentGuid: PlacesUtils.bookmarks.rootGuid,
+    position: -1,
+  }, {
+    guid: PlacesUtils.bookmarks.menuGuid,
+    parentGuid: PlacesUtils.bookmarks.rootGuid,
+    position: 0,
+  }, {
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    parentGuid: PlacesUtils.bookmarks.rootGuid,
+    position: 1,
+  }, {
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    parentGuid: PlacesUtils.bookmarks.rootGuid,
+    position: 2,
+  }, {
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    parentGuid: PlacesUtils.bookmarks.rootGuid,
+    position: 3,
+  }];
+  for (let info of syncableRoots) {
+    await db.executeCached(`
+      INSERT INTO items(guid, kind)
+      VALUES(:guid, :kind)`,
+      { guid: info.guid, kind: SyncedBookmarksMirror.KIND.FOLDER });
+
+    await db.executeCached(`
+      INSERT INTO structure(guid, parentGuid, position)
+      VALUES(:guid, :parentGuid, :position)`,
+      info);
+  }
+}
+
+/**
+ * Creates temporary tables, views, and triggers to apply the mirror to Places.
+ *
+ * The bulk of the logic to apply all remotely changed items is defined in
+ * `INSTEAD OF DELETE` triggers on the `newRemoteItems` and `newRemoteStructure`
+ * views. When we execute `DELETE FROM newRemote{Items, Structure}`, SQLite
+ * fires the triggers for each row in the view. This is equivalent to, but more
+ * efficient than, issuing `SELECT * FROM newRemote{Items, Structure}`,
+ * followed by separate `INSERT` and `UPDATE` statements.
+ *
+ * Using triggers to execute all these statements avoids the overhead of passing
+ * results between the storage and main threads, and wrapping each result row in
+ * a `mozStorageRow` object.
+ *
+ * @param {Sqlite.OpenedConnection} db
+ *        The mirror database connection.
+ */
+async function initializeTempMirrorEntities(db) {
+  // Columns in `items` that correspond to annos stored in `moz_items_annos`.
+  // We use this table to build SQL fragments for the `insertNewLocalItems` and
+  // `updateExistingLocalItems` triggers below.
+  const syncedAnnoTriggers = [{
+    annoName: PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO,
+    columnName: "description",
+    type: PlacesUtils.annotations.TYPE_STRING,
+  }, {
+    annoName: PlacesSyncUtils.bookmarks.SIDEBAR_ANNO,
+    columnName: "loadInSidebar",
+    type: PlacesUtils.annotations.TYPE_INT32,
+  }, {
+    annoName: PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO,
+    columnName: "smartBookmarkName",
+    type: PlacesUtils.annotations.TYPE_STRING,
+  }, {
+    annoName: PlacesUtils.LMANNO_FEEDURI,
+    columnName: "feedURL",
+    type: PlacesUtils.annotations.TYPE_STRING,
+  }, {
+    annoName: PlacesUtils.LMANNO_SITEURI,
+    columnName: "siteURL",
+    type: PlacesUtils.annotations.TYPE_STRING,
+  }];
+
+  // Stores the value and structure states of all nodes in the merged tree.
+  await db.execute(`CREATE TEMP TABLE mergeStates(
+    localGuid TEXT NOT NULL,
+    mergedGuid TEXT NOT NULL,
+    parentGuid TEXT NOT NULL,
+    level INTEGER NOT NULL,
+    position INTEGER NOT NULL,
+    valueState INTEGER NOT NULL,
+    structureState INTEGER NOT NULL,
+    PRIMARY KEY(localGuid, mergedGuid)
+  ) WITHOUT ROWID`);
+
+  // A view of the value states for all bookmarks in the mirror. We use triggers
+  // on this view to update Places. Note that we can't just `REPLACE INTO
+  // moz_bookmarks`, because `REPLACE` doesn't fire the `AFTER DELETE` triggers
+  // that Places uses to maintain schema coherency.
+  await db.execute(`
+    CREATE TEMP VIEW newRemoteItems(localId, remoteId, localGuid, mergedGuid,
+                                    needsUpdate, type, dateAdded, title,
+                                    oldPlaceId, newPlaceId, newKeyword,
+                                    description, loadInSidebar,
+                                    smartBookmarkName, feedURL, siteURL,
+                                    syncChangeCounter) AS
+    SELECT b.id, v.id, r.localGuid, r.mergedGuid,
+           r.valueState = ${BookmarkMergeState.TYPE.REMOTE},
+           (CASE WHEN v.kind IN (${[
+                        SyncedBookmarksMirror.KIND.BOOKMARK,
+                        SyncedBookmarksMirror.KIND.QUERY,
+                      ].join(",")}) THEN ${PlacesUtils.bookmarks.TYPE_BOOKMARK}
+                 WHEN v.kind IN (${[
+                        SyncedBookmarksMirror.KIND.FOLDER,
+                        SyncedBookmarksMirror.KIND.LIVEMARK,
+                      ].join(",")}) THEN ${PlacesUtils.bookmarks.TYPE_FOLDER}
+                 ELSE ${PlacesUtils.bookmarks.TYPE_SEPARATOR} END),
+           (CASE WHEN b.dateAdded < v.dateAdded THEN b.dateAdded
+                 ELSE v.dateAdded END),
+           v.title, h.id, u.newPlaceId, v.keyword, v.description,
+           v.loadInSidebar, v.smartBookmarkName, v.feedURL, v.siteURL,
+           (CASE r.structureState WHEN ${BookmarkMergeState.TYPE.REMOTE} THEN 0
+            ELSE 1 END)
+    FROM items v
+    JOIN mergeStates r ON r.mergedGuid = v.guid
+    LEFT JOIN moz_bookmarks b ON b.guid = r.localGuid
+    LEFT JOIN moz_places h ON h.id = b.fk
+    LEFT JOIN (
+      SELECT h.id AS newPlaceId, u.id AS urlId
+      FROM urls u
+      JOIN moz_places h ON h.url_hash = u.hash AND
+                           h.url = u.url
+    ) u ON u.urlId = v.urlId
+    WHERE r.mergedGuid <> '${PlacesUtils.bookmarks.rootGuid}'`);
+
+  // Changes local GUIDs to remote GUIDs, drops local tombstones for revived
+  // remote items, and flags remote items as merged. In the trigger body, `OLD`
+  // refers to the row for the unmerged item in `newRemoteItems`.
+  await db.execute(`
+    CREATE TEMP TRIGGER mergeGuids
+    INSTEAD OF DELETE ON newRemoteItems
+    BEGIN
+      /* We update GUIDs here, instead of in the "updateExistingLocalItems"
+         trigger, because deduped items where we're keeping the local value
+         state won't have "needsMerge" set. */
+      UPDATE moz_bookmarks SET
+        guid = OLD.mergedGuid
+      WHERE OLD.localGuid <> OLD.mergedGuid AND
+            guid = OLD.localGuid;
+
+      /* Record item changed notifications for the updated GUIDs. */
+      INSERT INTO guidsChanged(itemId, oldGuid)
+      SELECT OLD.localId, OLD.localGuid
+      WHERE OLD.localGuid <> OLD.mergedGuid;
+
+      DELETE FROM moz_bookmarks_deleted WHERE guid = OLD.mergedGuid;
+
+      /* Flag the remote item as merged. */
+      UPDATE items SET
+        needsMerge = 0
+      WHERE needsMerge AND
+            guid = OLD.mergedGuid;
+    END`);
+
+  // Inserts items from the mirror that don't exist locally.
+  await db.execute(`
+    CREATE TEMP TRIGGER insertNewLocalItems
+    INSTEAD OF DELETE ON newRemoteItems WHEN OLD.localId IS NULL
+    BEGIN
+      /* Sync associates keywords with bookmarks, and doesn't sync POST data;
+         Places associates keywords with (URL, POST data) pairs, and multiple
+         bookmarks may have the same URL. For simplicity, we bump the change
+         counter for all local bookmarks with the remote URL (bug 1328737),
+         then remove all local keywords from remote URLs, and the remote keyword
+         from local URLs. */
+      UPDATE moz_bookmarks SET
+        syncChangeCounter = syncChangeCounter + 1
+      WHERE fk IN (
+        /* We intentionally use "place_id = OLD.newPlaceId" in the subquery,
+           instead of "fk = OLD.newPlaceId OR fk IN (...)" in the WHERE clause
+           above, because we only want to bump the counter if the URL has
+           keywords. */
+        SELECT place_id FROM moz_keywords
+        WHERE place_id = OLD.newPlaceId OR
+              keyword = OLD.newKeyword);
+
+      /* Record item changed notifications for existing items with the new
+         keyword and URL. */
+      INSERT INTO keywordsChanged(itemId, placeId, keyword)
+      SELECT b.id, b.fk, NULL FROM moz_bookmarks b
+      JOIN moz_keywords k ON k.place_id = b.fk
+      WHERE k.place_id = OLD.newPlaceId OR
+            k.keyword = OLD.newKeyword;
+
+      /* Remove the new keyword from existing items, and all keywords from the
+         new URL. */
+      DELETE FROM moz_keywords WHERE place_id = OLD.newPlaceId OR
+                                     keyword = OLD.newKeyword;
+
+      /* Remove existing tags for the new URL. */
+      DELETE FROM localTags WHERE placeId = OLD.newPlaceId;
+
+      /* Insert the new item, using "-1" as the placeholder parent and
+         position. We'll update these later, in the "updateLocalStructure"
+         trigger. */
+      INSERT INTO moz_bookmarks(guid, parent, position, type, fk, title,
+                                dateAdded, lastModified, syncStatus,
+                                syncChangeCounter)
+      VALUES(OLD.mergedGuid, -1, -1, OLD.type, OLD.newPlaceId, OLD.title,
+             OLD.dateAdded, STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000,
+             ${PlacesUtils.bookmarks.SYNC_STATUS.NORMAL},
+             OLD.syncChangeCounter);
+
+      /* Record an item added notification for the new item. */
+      INSERT INTO itemsAdded(guid)
+      VALUES(OLD.mergedGuid);
+
+      /* Insert new keywords after the item, so that "noteKeywordAdded" can find
+         the new item by Place ID. */
+      INSERT INTO moz_keywords(keyword, place_id)
+      SELECT OLD.newKeyword, OLD.newPlaceId
+      WHERE OLD.newKeyword NOT NULL;
+
+      /* Record item changed notifications for the new keyword. */
+      INSERT INTO keywordsChanged(itemId, placeId, keyword)
+      SELECT b.id, OLD.newPlaceId, OLD.newKeyword FROM moz_bookmarks b
+      WHERE b.guid = OLD.mergedGuid AND
+            OLD.newKeyword NOT NULL;
+
+      /* Insert new tags for the URL. */
+      INSERT INTO localTags(tag, placeId)
+      SELECT t.tag, OLD.newPlaceId FROM tags t
+      WHERE t.itemId = OLD.remoteId;
+
+      /* Insert new synced annos. These are almost identical to the statements
+         for updates, except we need an additional subquery to fetch the new
+         item's ID. We can also skip removing existing annos. */
+      INSERT OR IGNORE INTO moz_anno_attributes(name)
+      VALUES ${syncedAnnoTriggers.map(annoTrigger =>
+        `('${annoTrigger.annoName}')`
+      ).join(",")};
+
+      ${syncedAnnoTriggers.map(annoTrigger => `
+        INSERT INTO moz_items_annos(item_id, anno_attribute_id, content, flags,
+                                    expiration, type, lastModified, dateAdded)
+        SELECT (SELECT id FROM moz_bookmarks
+                WHERE guid = OLD.mergedGuid),
+               (SELECT id FROM moz_anno_attributes
+                WHERE name = '${annoTrigger.annoName}'),
+               OLD.${annoTrigger.columnName}, 0,
+               ${PlacesUtils.annotations.EXPIRE_NEVER}, ${annoTrigger.type},
+               STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000,
+               STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000
+        WHERE OLD.${annoTrigger.columnName} NOT NULL;
+
+        /* Record an anno set notification for the new synced anno. */
+        REPLACE INTO annosChanged(itemId, annoName, wasRemoved)
+        SELECT b.id, '${annoTrigger.annoName}', 0 FROM moz_bookmarks b
+        WHERE b.guid = OLD.mergedGuid AND
+              OLD.${annoTrigger.columnName} NOT NULL;
+      `).join("")}
+    END`);
+
+  // Updates existing items with new values from the mirror.
+  await db.execute(`
+    CREATE TEMP TRIGGER updateExistingLocalItems
+    INSTEAD OF DELETE ON newRemoteItems WHEN OLD.needsUpdate AND
+                                             OLD.localId NOT NULL
+    BEGIN
+      /* Record item changed notifications for the title and URL. */
+      INSERT INTO itemsChanged(itemId, oldTitle, oldPlaceId)
+      SELECT id, title, OLD.oldPlaceId FROM moz_bookmarks
+      WHERE id = OLD.localId;
+
+      UPDATE moz_bookmarks SET
+        title = OLD.title,
+        dateAdded = OLD.dateAdded,
+        lastModified = STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000,
+        syncStatus = ${PlacesUtils.bookmarks.SYNC_STATUS.NORMAL},
+        syncChangeCounter = OLD.syncChangeCounter
+      WHERE id = OLD.localId;
+
+      /* Bump the change counter for items with the old URL, new URL, and new
+         keyword. */
+      UPDATE moz_bookmarks SET
+        syncChangeCounter = syncChangeCounter + 1
+      WHERE fk IN (SELECT place_id FROM moz_keywords
+                   WHERE place_id IN (OLD.oldPlaceId, OLD.newPlaceId) OR
+                         keyword = OLD.newKeyword);
+
+      /* Record change observer notifications for items with the old URL, new
+         URL, and new keyword. This is identical to the subquery above; we just
+         query the item ID instead of updating by the Place ID. */
+      INSERT INTO keywordsChanged(itemId, placeId, keyword)
+      SELECT b.id, b.fk, NULL FROM moz_bookmarks b
+      JOIN moz_keywords k ON k.place_id = b.fk
+      WHERE k.place_id IN (OLD.oldPlaceId, OLD.newPlaceId) OR
+            k.keyword = OLD.newKeyword;
+
+      /* Remove the new keyword from existing items, and all keywords from the
+         old and new URLs. */
+      DELETE FROM moz_keywords WHERE place_id IN (OLD.oldPlaceId,
+                                                  OLD.newPlaceId) OR
+                                     keyword = OLD.newKeyword;
+
+      /* Remove existing tags. */
+      DELETE FROM localTags WHERE placeId IN (OLD.oldPlaceId, OLD.newPlaceId);
+
+      /* Update the URL and recalculate frecency. It's important we do this
+         *after* removing old keywords and *before* inserting new ones, so that
+         the above statements select the correct affected items. */
+      UPDATE moz_bookmarks SET
+        fk = OLD.newPlaceId
+      WHERE OLD.oldPlaceId <> OLD.newPlaceId AND
+            id = OLD.localId;
+
+      UPDATE moz_places SET
+        frecency = -1
+      WHERE OLD.oldPlaceId <> OLD.newPlaceId AND
+            id IN (OLD.oldPlaceId, OLD.newPlaceId);
+
+      /* Insert a new keyword for the new URL, if one is set. */
+      INSERT INTO moz_keywords(keyword, place_id)
+      SELECT OLD.newKeyword, OLD.newPlaceId
+      WHERE OLD.newKeyword NOT NULL;
+
+      /* Record an item changed notification for the new keyword. */
+      INSERT INTO keywordsChanged(itemId, placeId, keyword)
+      SELECT OLD.localId, OLD.newPlaceId, OLD.newKeyword
+      WHERE OLD.newKeyword NOT NULL;
+
+      /* Insert new tags for the new URL. */
+      INSERT INTO localTags(tag, placeId)
+      SELECT t.tag, OLD.newPlaceId FROM tags t
+      WHERE t.itemId = OLD.remoteId;
+
+      /* Record anno removed notifications for the synced annos. */
+      REPLACE INTO annosChanged(itemId, annoName, wasRemoved)
+      SELECT a.item_id, n.name, 1 FROM moz_items_annos a
+      JOIN moz_anno_attributes n ON n.id = a.anno_attribute_id
+      WHERE item_id = OLD.localId AND
+            anno_attribute_id IN (SELECT id FROM moz_anno_attributes
+                                  WHERE name IN (${syncedAnnoTriggers.map(
+                                    annoTrigger => `'${annoTrigger.annoName}'`
+                                  ).join(",")}));
+
+      /* Remove existing synced annos. */
+      DELETE FROM moz_items_annos
+      WHERE item_id = OLD.localId AND
+            anno_attribute_id IN (SELECT id FROM moz_anno_attributes
+                                  WHERE name IN (${syncedAnnoTriggers.map(
+                                    annoTrigger => `'${annoTrigger.annoName}'`
+                                  ).join(",")}));
+
+      /* Insert new synced annos. */
+      INSERT OR IGNORE INTO moz_anno_attributes(name)
+      VALUES ${syncedAnnoTriggers.map(annoTrigger =>
+        `('${annoTrigger.annoName}')`
+      ).join(",")};
+
+      ${syncedAnnoTriggers.map(annoTrigger => `
+        INSERT INTO moz_items_annos(item_id, anno_attribute_id, content, flags,
+                                    expiration, type, lastModified, dateAdded)
+        SELECT OLD.localId, (SELECT id FROM moz_anno_attributes
+                             WHERE name = '${annoTrigger.annoName}'),
+               OLD.${annoTrigger.columnName}, 0,
+               ${PlacesUtils.annotations.EXPIRE_NEVER}, ${annoTrigger.type},
+               STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000,
+               STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000
+         WHERE OLD.${annoTrigger.columnName} NOT NULL;
+
+         /* Record an anno set notification for the new synced anno. */
+         REPLACE INTO annosChanged(itemId, annoName, wasRemoved)
+         SELECT OLD.localId, '${annoTrigger.annoName}', 0
+         WHERE OLD.${annoTrigger.columnName} NOT NULL;
+      `).join("")}
+    END`);
+
+  // A view of the new structure state for all items in the merged tree. The
+  // mirror stores structure info in a separate table, like iOS, while Places
+  // stores structure info on children. Unlike iOS, we can't simply check the
+  // parent's merge state to know if its children changed. This is because our
+  // merged tree might diverge from the mirror if we're missing children, or if
+  // we temporarily reparented children without parents to "unfiled". In that
+  // case, we want to keep syncing, but *don't* want to reupload the new local
+  // structure to the server.
+  await db.execute(`
+    CREATE TEMP VIEW newRemoteStructure(localId, oldParentId, newParentId,
+                                        oldPosition, newPosition) AS
+    SELECT b.id, b.parent, p.id, b.position, r.position
+    FROM moz_bookmarks b
+    JOIN mergeStates r ON r.mergedGuid = b.guid
+    JOIN moz_bookmarks p ON p.guid = r.parentGuid
+    WHERE r.parentGuid <> '${PlacesUtils.bookmarks.rootGuid}'`);
+
+  // Updates all parents and positions to reflect the merged tree.
+  await db.execute(`
+    CREATE TEMP TRIGGER updateLocalStructure
+    INSTEAD OF DELETE ON newRemoteStructure
+    BEGIN
+      UPDATE moz_bookmarks SET
+        parent = OLD.newParentId
+      WHERE id = OLD.localId AND
+            parent <> OLD.newParentId;
+
+      UPDATE moz_bookmarks SET
+        position = OLD.newPosition
+      WHERE id = OLD.localId AND
+            position <> OLD.newPosition;
+
+      /* Record observer notifications for moved items. We ignore items that
+         didn't move, and items with placeholder parents and positions of "-1",
+         since they're new. */
+      INSERT INTO itemsMoved(itemId, oldParentId, oldParentGuid, oldPosition)
+      SELECT OLD.localId, OLD.oldParentId, p.guid, OLD.oldPosition
+      FROM moz_bookmarks p
+      WHERE p.id = OLD.oldParentId AND
+            -1 NOT IN (OLD.oldParentId, OLD.oldPosition) AND
+            (OLD.oldParentId <> OLD.newParentId OR
+             OLD.oldPosition <> OLD.newPosition);
+    END`);
+
+  // A view of local bookmark tags. Tags, like keywords, are associated with
+  // URLs, so two bookmarks with the same URL should have the same tags. Unlike
+  // keywords, one tag may be associated with many different URLs. Tags are also
+  // different because they're implemented as bookmarks under the hood. Each tag
+  // is stored as a folder under the tags root, and tagged URLs are stored as
+  // untitled bookmarks under these folders. This complexity, along with tag
+  // query rewriting, can be removed once bug 1293445 lands.
+  await db.execute(`
+    CREATE TEMP VIEW localTags(tagEntryId, tagEntryGuid, tagFolderId,
+                               tagFolderGuid, tagEntryPosition, tagEntryType,
+                               tag, placeId) AS
+    SELECT b.id, b.guid, p.id, p.guid, b.position, b.type, p.title, b.fk
+    FROM moz_bookmarks b
+    JOIN moz_bookmarks p ON p.id = b.parent
+    JOIN moz_bookmarks r ON r.id = p.parent
+    WHERE b.type = ${PlacesUtils.bookmarks.TYPE_BOOKMARK} AND
+          r.guid = '${PlacesUtils.bookmarks.tagsGuid}'`);
+
+  // Untags a URL by removing its tag entry.
+  await db.execute(`
+    CREATE TEMP TRIGGER untagLocalPlace
+    INSTEAD OF DELETE ON localTags
+    BEGIN
+      /* Record an item removed notification for the tag entry. */
+      INSERT INTO itemsRemoved(itemId, parentId, position, type, placeId, guid,
+                               parentGuid, isUntagging)
+      VALUES(OLD.tagEntryId, OLD.tagFolderId, OLD.tagEntryPosition,
+             OLD.tagEntryType, OLD.placeId, OLD.tagEntryGuid,
+             OLD.tagFolderGuid, 1);
+
+      DELETE FROM moz_bookmarks WHERE id = OLD.tagEntryId;
+
+      /* Fix the positions of the sibling tag entries. */
+      UPDATE moz_bookmarks SET
+        position = position - 1
+      WHERE parent = OLD.tagFolderId AND
+            position > OLD.tagEntryPosition;
+    END`);
+
+  // Tags a URL by creating a tag folder if it doesn't exist, then inserting a
+  // tag entry for the URL into the tag folder. `NEW.placeId` can be NULL, in
+  // which case we'll just create the tag folder.
+  await db.execute(`
+    CREATE TEMP TRIGGER tagLocalPlace
+    INSTEAD OF INSERT ON localTags
+    BEGIN
+      /* Ensure the tag folder exists. */
+      INSERT OR IGNORE INTO moz_bookmarks(guid, parent, position, type, title,
+                                          dateAdded, lastModified)
+      VALUES(IFNULL((SELECT b.guid FROM moz_bookmarks b
+                     JOIN moz_bookmarks p ON p.id = b.parent
+                     WHERE b.title = NEW.tag AND
+                           p.guid = '${PlacesUtils.bookmarks.tagsGuid}'),
+                    GENERATE_GUID()),
+             (SELECT id FROM moz_bookmarks
+              WHERE guid = '${PlacesUtils.bookmarks.tagsGuid}'),
+             (SELECT COUNT(*) FROM moz_bookmarks b
+              JOIN moz_bookmarks p ON p.id = b.parent
+              WHERE p.guid = '${PlacesUtils.bookmarks.tagsGuid}'),
+             ${PlacesUtils.bookmarks.TYPE_FOLDER}, NEW.tag,
+             STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000,
+             STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000);
+
+      /* Record an item added notification if we created a tag folder.
+         "CHANGES()" returns the number of rows affected by the INSERT above:
+         1 if we created the folder, or 0 if the folder already existed. */
+      INSERT INTO itemsAdded(guid, isTagging)
+      SELECT b.guid, 1 FROM moz_bookmarks b
+      JOIN moz_bookmarks p ON p.id = b.parent
+      WHERE CHANGES() > 0 AND
+            b.title = NEW.tag AND
+            p.guid = '${PlacesUtils.bookmarks.tagsGuid}';
+
+      /* Add a tag entry for the URL under the tag folder. Omitting the place
+         ID creates a tag folder without tagging the URL. */
+      INSERT OR IGNORE INTO moz_bookmarks(guid, parent, position, type, fk,
+                                          dateAdded, lastModified)
+      SELECT GENERATE_GUID(),
+             (SELECT b.id FROM moz_bookmarks b
+              JOIN moz_bookmarks p ON p.id = b.parent
+              WHERE p.guid = '${PlacesUtils.bookmarks.tagsGuid}' AND
+                    b.title = NEW.tag),
+             (SELECT COUNT(*) FROM moz_bookmarks b
+              JOIN moz_bookmarks p ON p.id = b.parent
+              JOIN moz_bookmarks r ON r.id = p.parent
+              WHERE p.title = NEW.tag AND
+                    r.guid = '${PlacesUtils.bookmarks.tagsGuid}'),
+             ${PlacesUtils.bookmarks.TYPE_BOOKMARK}, NEW.placeId,
+             STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000,
+             STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000
+        WHERE NEW.placeId NOT NULL;
+
+      /* Record an item added notification for the tag entry. */
+      INSERT INTO itemsAdded(guid, isTagging)
+      SELECT b.guid, 1 FROM moz_bookmarks b
+      JOIN moz_bookmarks p ON p.id = b.parent
+      JOIN moz_bookmarks r ON r.id = p.parent
+      WHERE b.fk = NEW.placeId AND
+            p.title = NEW.tag AND
+            r.guid = '${PlacesUtils.bookmarks.tagsGuid}';
+    END`);
+
+  // Stores properties to pass to `onItem{Added, Changed, Moved, Removed}`
+  // bookmark observers for new, updated, moved, and deleted items.
+  await db.execute(`CREATE TEMP TABLE itemsAdded(
+    guid TEXT PRIMARY KEY,
+    isTagging BOOLEAN NOT NULL DEFAULT 0
+  ) WITHOUT ROWID`);
+
+  await db.execute(`CREATE TEMP TABLE guidsChanged(
+    itemId INTEGER NOT NULL,
+    oldGuid TEXT NOT NULL,
+    PRIMARY KEY(itemId, oldGuid)
+  ) WITHOUT ROWID`);
+
+  await db.execute(`CREATE TEMP TABLE itemsChanged(
+    itemId INTEGER PRIMARY KEY,
+    oldTitle TEXT,
+    oldPlaceId INTEGER
+  )`);
+
+  await db.execute(`CREATE TEMP TABLE itemsMoved(
+    itemId INTEGER PRIMARY KEY,
+    oldParentId INTEGER NOT NULL,
+    oldParentGuid TEXT NOT NULL,
+    oldPosition INTEGER NOT NULL
+  )`);
+
+  await db.execute(`CREATE TEMP TABLE itemsRemoved(
+    guid TEXT PRIMARY KEY,
+    itemId INTEGER NOT NULL,
+    parentId INTEGER NOT NULL,
+    position INTEGER NOT NULL,
+    type INTEGER NOT NULL,
+    placeId INTEGER,
+    parentGuid TEXT NOT NULL,
+    /* We record the original level of the removed item in the tree so that we
+       can notify children before parents. */
+    level INTEGER NOT NULL DEFAULT -1,
+    isUntagging BOOLEAN NOT NULL DEFAULT 0
+  ) WITHOUT ROWID`);
+
+  // Stores properties to pass to `onItemAnnotation{Set, Removed}` anno
+  // observers.
+  await db.execute(`CREATE TEMP TABLE annosChanged(
+    itemId INTEGER NOT NULL,
+    annoName TEXT NOT NULL,
+    wasRemoved BOOLEAN NOT NULL,
+    PRIMARY KEY(itemId, annoName, wasRemoved)
+  ) WITHOUT ROWID`);
+
+  // Stores properties to pass to `onItemChanged` observers for new and removed
+  // keywords. A NULL keyword means we're removing all keywords from a Place.
+  // Note that an item may appear multiple times in this table, so `itemId` is
+  // not a primary key.
+  await db.execute(`CREATE TEMP TABLE keywordsChanged(
+    itemId INTEGER NOT NULL,
+    placeId INTEGER NOT NULL,
+    keyword TEXT
+  )`);
+
+  // Stores locally changed items staged for upload. See `stageItemsToUpload`
+  // for an explanation of why these tables exists.
+  await db.execute(`CREATE TEMP TABLE itemsToUpload(
+    guid TEXT PRIMARY KEY,
+    syncChangeCounter INTEGER NOT NULL,
+    isDeleted BOOLEAN NOT NULL DEFAULT 0,
+    parentGuid TEXT,
+    parentTitle TEXT,
+    dateAdded INTEGER,
+    type INTEGER,
+    title TEXT,
+    isQuery BOOLEAN NOT NULL DEFAULT 0,
+    url TEXT,
+    tags TEXT,
+    description TEXT,
+    loadInSidebar BOOLEAN,
+    smartBookmarkName TEXT,
+    tagFolderName TEXT,
+    keyword TEXT,
+    feedURL TEXT,
+    siteURL TEXT,
+    position INTEGER
+  ) WITHOUT ROWID`);
+
+  await db.execute(`CREATE TEMP TABLE structureToUpload(
+    guid TEXT PRIMARY KEY,
+    parentGuid TEXT NOT NULL REFERENCES itemsToUpload(guid)
+                             ON DELETE CASCADE,
+    position INTEGER NOT NULL
+  ) WITHOUT ROWID`);
+}
+
+// Converts a Sync record ID to a Places GUID. Returns `null` if the ID is
+// invalid.
+function validateGuid(recordId) {
+  let guid = PlacesSyncUtils.bookmarks.recordIdToGuid(recordId);
+  return PlacesUtils.isValidGuid(guid) ? guid : null;
+}
+
+// Converts a Sync record's last modified time to milliseconds.
+function determineServerModified(record) {
+  return Math.max(record.modified * 1000, 0) || 0;
+}
+
+// Determines a Sync record's creation date.
+function determineDateAdded(record) {
+  let serverModified = determineServerModified(record);
+  let dateAdded = PlacesSyncUtils.bookmarks.ratchetTimestampBackwards(
+    record.dateAdded, serverModified);
+  return dateAdded ? PlacesUtils.toPRTime(new Date(dateAdded)) : 0;
+}
+
+function validateTitle(rawTitle) {
+  if (typeof rawTitle != "string" || !rawTitle) {
+    return null;
+  }
+  return rawTitle.slice(0, DB_TITLE_LENGTH_MAX);
+}
+
+function validateURL(rawURL) {
+  if (typeof rawURL != "string" || rawURL.length > DB_URL_LENGTH_MAX) {
+    return null;
+  }
+  let url = null;
+  try {
+    url = new URL(rawURL);
+  } catch (ex) {}
+  return url;
+}
+
+function validateDescription(rawDescription) {
+  if (typeof rawDescription != "string" || !rawDescription) {
+    return null;
+  }
+  return rawDescription.slice(0, DB_DESCRIPTION_LENGTH_MAX);
+}
+
+function validateKeyword(rawKeyword) {
+  if (typeof rawKeyword != "string") {
+    return null;
+  }
+  let keyword = rawKeyword.trim();
+  // Drop empty keywords.
+  return keyword ? keyword.toLowerCase() : null;
+}
+
+function validateTag(rawTag) {
+  if (typeof rawTag != "string") {
+    return null;
+  }
+  let tag = rawTag.trim();
+  if (!tag || tag.length > Ci.nsITaggingService.MAX_TAG_LENGTH) {
+    // Drop empty and oversized tags.
+    return null;
+  }
+  return tag;
+}
+
+// Recursively inflates a bookmark tree from a pseudo-tree that maps
+// parents to children.
+function inflateTree(tree, pseudoTree, parentGuid) {
+  let nodes = pseudoTree.get(parentGuid);
+  if (nodes) {
+    for (let node of nodes) {
+      tree.insert(parentGuid, node);
+      inflateTree(tree, pseudoTree, node.guid);
+    }
+  }
+}
+
+/**
+ * Content info for an item in the local or remote tree. This is used to dedupe
+ * NEW local items to remote items that don't exist locally. See `contentsMatch`
+ * for how we determine if two items are dupes.
+ */
+class BookmarkContent {
+  constructor(title, urlHref, smartBookmarkName, position) {
+    this.title = title;
+    this.url = urlHref ? new URL(urlHref) : null;
+    this.smartBookmarkName = smartBookmarkName;
+    this.position = position;
+  }
+
+  static fromRow(row) {
+    let title = row.getResultByName("title");
+    let urlHref = row.getResultByName("url");
+    let smartBookmarkName = row.getResultByName("smartBookmarkName");
+    let position = row.getResultByName("position");
+    return new BookmarkContent(title, urlHref, smartBookmarkName, position);
+  }
+
+  hasSameURL(otherContent) {
+    return !!this.url == !!otherContent.url &&
+           this.url.href == otherContent.url.href;
+  }
+}
+
+/**
+ * The merge state indicates which node we should prefer when reconciling
+ * with Places. Recall that a merged node may point to a local node, remote
+ * node, or both.
+ */
+class BookmarkMergeState {
+  constructor(type, newStructureNode = null) {
+    this.type = type;
+    this.newStructureNode = newStructureNode;
+  }
+
+  /**
+   * Takes an existing value state, and a new node for the structure state. We
+   * use the new merge state to resolve conflicts caused by moving local items
+   * out of a remotely deleted folder, or remote items out of a locally deleted
+   * folder.
+   *
+   * Applying a new merged node bumps its local change counter, so that the
+   * merged structure is reuploaded to the server.
+   *
+   * @param  {BookmarkMergeState} oldState
+   *         The existing value state.
+   * @param  {BookmarkNode} newStructureNode
+   *         A node to use for the new structure state.
+   * @return {BookmarkMergeState}
+   *         The new merge state.
+   */
+  static new(oldState, newStructureNode) {
+    return new BookmarkMergeState(oldState.type, newStructureNode);
+  }
+
+  // Returns the structure state type: `LOCAL`, `REMOTE`, or `NEW`.
+  structure() {
+    return this.newStructureNode ? BookmarkMergeState.TYPE.NEW : this.type;
+  }
+
+  // Returns the value state type: `LOCAL` or `REMOTE`.
+  value() {
+    return this.type;
+  }
+}
+
+BookmarkMergeState.TYPE = {
+  LOCAL: 1,
+  REMOTE: 2,
+  NEW: 3,
+};
+
+/**
+ * A local merge state means no changes: we keep the local value and structure
+ * state. This could mean that the item doesn't exist on the server yet, or that
+ * it has newer local changes that we should upload.
+ *
+ * It's an error for a merged node to have a local merge state without a local
+ * node. Deciding the value state for the merged node asserts this.
+ */
+BookmarkMergeState.local = new BookmarkMergeState(
+  BookmarkMergeState.TYPE.LOCAL);
+
+/**
+ * A remote merge state means we should update Places with new value and
+ * structure state from the mirror. The item might not exist locally yet, or
+ * might have newer remote changes that we should apply.
+ *
+ * As with local, a merged node can't have a remote merge state without a
+ * remote node.
+ */
+BookmarkMergeState.remote = new BookmarkMergeState(
+  BookmarkMergeState.TYPE.REMOTE);
+
+/**
+ * A node in a local or remote bookmark tree. Nodes are lightweight: they carry
+ * enough information for the merger to resolve trivial conflicts without
+ * querying the mirror or Places for the complete value state.
+ */
+class BookmarkNode {
+  constructor(guid, age, kind, needsMerge = false) {
+    this.guid = guid;
+    this.kind = kind;
+    this.age = age;
+    this.needsMerge = needsMerge;
+    this.children = [];
+  }
+
+  // Creates a virtual folder node for the Places root.
+  static root() {
+    let guid = PlacesUtils.bookmarks.rootGuid;
+    return new BookmarkNode(guid, 0, SyncedBookmarksMirror.KIND.FOLDER);
+  }
+
+  /**
+   * Creates a bookmark node from a Places row.
+   *
+   * @param  {mozIStorageRow} row
+   *         The Places row containing the node info.
+   * @param  {Number} localTimeSeconds
+   *         The current local time, in seconds, used to calculate the
+   *         item's age.
+   * @return {BookmarkNode}
+   *         A bookmark node for the local item.
+   */
+  static fromLocalRow(row, localTimeSeconds) {
+    let guid = row.getResultByName("guid");
+
+    // Note that this doesn't account for local clock skew. `localModified`
+    // is in *microseconds*.
+    let localModified = row.getResultByName("lastModified");
+    let age = Math.max(localTimeSeconds - localModified / 1000000, 0) || 0;
+
+    let kind = row.getResultByName("kind");
+
+    let syncChangeCounter = row.getResultByName("syncChangeCounter");
+    let needsMerge = syncChangeCounter > 0;
+
+    return new BookmarkNode(guid, age, kind, needsMerge);
+  }
+
+  /**
+   * Creates a bookmark node from a mirror row.
+   *
+   * @param  {mozIStorageRow} row
+   *         The mirror row containing the node info.
+   * @param  {Number} remoteTimeSeconds
+   *         The current server time, in seconds, used to calculate the
+   *         item's age.
+   * @return {BookmarkNode}
+   *         A bookmark node for the remote item.
+   */
+  static fromRemoteRow(row, remoteTimeSeconds) {
+    let guid = row.getResultByName("guid");
+
+    // `serverModified` is in *milliseconds*.
+    let serverModified = row.getResultByName("serverModified");
+    let age = Math.max(remoteTimeSeconds - serverModified / 1000, 0) || 0;
+
+    let kind = row.getResultByName("kind");
+    let needsMerge = !!row.getResultByName("needsMerge");
+
+    return new BookmarkNode(guid, age, kind, needsMerge);
+  }
+
+  isRoot() {
+    return this.guid == PlacesUtils.bookmarks.rootGuid ||
+           PlacesUtils.bookmarks.userContentRoots.includes(this.guid);
+  }
+
+  isFolder() {
+    return this.kind == SyncedBookmarksMirror.KIND.FOLDER;
+  }
+
+  newerThan(otherNode) {
+    return this.age < otherNode.age;
+  }
+
+  * descendants() {
+    for (let node of this.children) {
+      yield node;
+      if (node.isFolder()) {
+        yield* node.descendants();
+      }
+    }
+  }
+}
+
+/**
+ * A complete, rooted tree with tombstones.
+ */
+class BookmarkTree {
+  constructor(root) {
+    this.byGuid = new Map();
+    this.infosByNode = new WeakMap();
+    this.deletedGuids = new Set();
+
+    this.root = root;
+    this.byGuid.set(this.root.guid, this.root);
+  }
+
+  isDeleted(guid) {
+    return this.deletedGuids.has(guid);
+  }
+
+  nodeForGuid(guid) {
+    return this.byGuid.get(guid);
+  }
+
+  parentNodeFor(childNode) {
+    let info = this.infosByNode.get(childNode);
+    return info ? info.parentNode : null;
+  }
+
+  levelForGuid(guid) {
+    let node = this.byGuid.get(guid);
+    if (!node) {
+      return -1;
+    }
+    let info = this.infosByNode.get(node);
+    return info ? info.level : -1;
+  }
+
+  /**
+   * Inserts a node into the tree. The node must not already exist in the tree,
+   * and the node's parent must be a folder.
+   */
+  insert(parentGuid, node) {
+    if (this.byGuid.has(node.guid)) {
+      let existingNode = this.byGuid.get(node.guid);
+      MirrorLog.error("Can't replace existing node ${existingNode} with node " +
+                      "${node}", { existingNode, node });
+      throw new TypeError("Node already exists in tree");
+    }
+    let parentNode = this.byGuid.get(parentGuid);
+    if (!parentNode) {
+      MirrorLog.error("Missing parent ${parentGuid} for node ${node}",
+                      { parentGuid, node });
+      throw new TypeError("Can't insert node into nonexistent parent");
+    }
+    if (!parentNode.isFolder()) {
+      MirrorLog.error("Non-folder parent ${parentNode} for node ${node}",
+                      { parentNode, node });
+      throw new TypeError("Can't insert node into non-folder");
+    }
+
+    parentNode.children.push(node);
+    this.byGuid.set(node.guid, node);
+
+    let parentInfo = this.infosByNode.get(parentNode);
+    let level = parentInfo ? parentInfo.level + 1 : 0;
+    this.infosByNode.set(node, { parentNode, level });
+  }
+
+  noteDeleted(guid) {
+    this.deletedGuids.add(guid);
+  }
+
+  * guids() {
+    for (let [guid, node] of this.byGuid) {
+      if (node == this.root) {
+        continue;
+      }
+      yield guid;
+    }
+    for (let guid of this.deletedGuids) {
+      yield guid;
+    }
+  }
+
+  toJSON() {
+    let deleted = Array.from(this.deletedGuids);
+    return { root: this.root, deleted };
+  }
+}
+
+/**
+ * A node in a merged bookmark tree. Holds the local node, remote node,
+ * merged children, and a merge state indicating which side to prefer.
+ */
+class MergedBookmarkNode {
+  constructor(guid, localNode, remoteNode, mergeState) {
+    this.guid = guid;
+    this.localNode = localNode;
+    this.remoteNode = remoteNode;
+    this.mergeState = mergeState;
+    this.mergedChildren = [];
+  }
+
+  /**
+   * Yields the decided value and structure states of the merged node's
+   * descendants. We use these as binding parameters to populate the temporary
+   * `mergeStates` table when applying the merged tree to Places.
+   */
+  * mergeStatesParams(level = 0) {
+    for (let position = 0; position < this.mergedChildren.length; ++position) {
+      let mergedChild = this.mergedChildren[position];
+      let mergeStateParam = {
+        localGuid: mergedChild.localNode ? mergedChild.localNode.guid : null,
+        // The merged GUID is different than the local GUID if we deduped a
+        // NEW local item to a remote item.
+        mergedGuid: mergedChild.guid,
+        parentGuid: this.guid,
+        level,
+        position,
+        valueState: mergedChild.mergeState.value(),
+        structureState: mergedChild.mergeState.structure(),
+      };
+      yield mergeStateParam;
+      yield* mergedChild.mergeStatesParams(level + 1);
+    }
+  }
+
+  /**
+   * Creates a bookmark node from this merged node.
+   *
+   * @return {BookmarkNode}
+   *         A node containing the decided value and structure state.
+   */
+  toBookmarkNode() {
+    if (MergedBookmarkNode.cachedBookmarkNodes.has(this)) {
+      return MergedBookmarkNode.cachedBookmarkNodes.get(this);
+    }
+
+    let decidedValueNode = this.decidedValue();
+    let decidedStructureState = this.mergeState.structure();
+    let needsMerge = decidedStructureState == BookmarkMergeState.TYPE.NEW ||
+                     (decidedStructureState == BookmarkMergeState.TYPE.LOCAL &&
+                      decidedValueNode.needsMerge);
+
+    let newNode = new BookmarkNode(this.guid, decidedValueNode.age,
+                                   decidedValueNode.kind, needsMerge);
+    MergedBookmarkNode.cachedBookmarkNodes.set(this, newNode);
+
+    if (newNode.isFolder()) {
+      for (let mergedChildNode of this.mergedChildren) {
+        newNode.children.push(mergedChildNode.toBookmarkNode());
+      }
+    }
+
+    return newNode;
+  }
+
+  /**
+   * Decides the value state for the merged node. Note that you can't walk the
+   * decided node's children: since the value node doesn't include structure
+   * changes from the other side, you'll depart from the merged tree. You'll
+   * want to use `toBookmarkNode` instead, which returns a node with the
+   * decided value *and* structure.
+   *
+   * @return {BookmarkNode}
+   *         The local or remote node containing the decided value state.
+   */
+  decidedValue() {
+    let valueState = this.mergeState.value();
+    switch (valueState) {
+      case BookmarkMergeState.TYPE.LOCAL:
+        if (!this.localNode) {
+          MirrorLog.error("Merged node ${guid} has local value state, but " +
+                          "no local node", this);
+          throw new TypeError(
+            "Can't take local value state without local node");
+        }
+        return this.localNode;
+
+      case BookmarkMergeState.TYPE.REMOTE:
+        if (!this.remoteNode) {
+          MirrorLog.error("Merged node ${guid} has remote value state, but " +
+                          "no remote node", this);
+          throw new TypeError(
+            "Can't take remote value state without remote node");
+        }
+        return this.remoteNode;
+    }
+    MirrorLog.error("Merged node ${guid} has unknown value state ${valueState}",
+                    { guid: this.guid, valueState });
+    throw new TypeError("Can't take unknown value state");
+  }
+}
+
+// Caches bookmark nodes containing the decided value and structure.
+MergedBookmarkNode.cachedBookmarkNodes = new WeakMap();
+
+/**
+ * A two-way merger that produces a complete merged tree from a complete local
+ * tree and a complete remote tree with changes since the last sync.
+ *
+ * This is ported almost directly from iOS. On iOS, the `ThreeWayMerger` takes a
+ * complete "mirror" tree with the server state after the last sync, and two
+ * incomplete trees with local and remote changes to the mirror: "local" and
+ * "mirror", respectively. Overlaying buffer onto mirror yields the current
+ * server tree; overlaying local onto mirror yields the complete local tree.
+ *
+ * On Desktop, our `localTree` is the union of iOS's mirror and local, and our
+ * `remoteTree` is the union of iOS's mirror and buffer. Mapping the iOS
+ * concepts to Desktop:
+ *
+ * - "Mirror" is approximately all `moz_bookmarks` where `syncChangeCounter = 0`
+ *   and `items` where `needsMerge = 0`. This is approximate because Desktop
+ *   doesn't store the shared parent for changed items.
+ * - "Local" is all `moz_bookmarks` where `syncChangeCounter > 0`.
+ * - "Buffer" is all `items` where `needsMerge = 1`.
+ *
+ * Since we don't store the shared parent, we can only do two-way merges. Also,
+ * our merger doesn't distinguish between structure and value changes, since we
+ * don't record that state in Places. The change counter notes *that* a bookmark
+ * changed, but not *how*. This means we might choose the wrong side when
+ * resolving merge conflicts, while iOS will do the right thing.
+ *
+ * Fortunately, most of our users don't organize their bookmarks into deeply
+ * nested hierarchies, or make conflicting changes on multiple devices
+ * simultaneously. Changing Places to record structure and value changes would
+ * require significant changes to the storage schema. A simpler two-way tree
+ * merge strikes a good balance between correctness and complexity.
+ */
+class BookmarkMerger {
+  constructor(localTree, newLocalContents, remoteTree, newRemoteContents) {
+    this.localTree = localTree;
+    this.newLocalContents = newLocalContents;
+    this.remoteTree = remoteTree;
+    this.newRemoteContents = newRemoteContents;
+    this.mergedGuids = new Set();
+    this.deleteLocally = new Set();
+    this.deleteRemotely = new Set();
+    this.telemetryEvents = [];
+  }
+
+  merge() {
+    let localRoot = this.localTree.nodeForGuid(PlacesUtils.bookmarks.rootGuid);
+    let remoteRoot = this.remoteTree.nodeForGuid(PlacesUtils.bookmarks.rootGuid);
+    let mergedRoot = this.mergeNode(PlacesUtils.bookmarks.rootGuid, localRoot,
+                                    remoteRoot);
+    return mergedRoot;
+  }
+
+  subsumes(tree) {
+    for (let guid of tree.guids()) {
+      if (!this.mergedGuids.has(guid) && !this.deleteLocally.has(guid) &&
+          !this.deleteRemotely.has(guid)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Merges two nodes, recursively walking folders.
+   *
+   * @param  {String} guid
+   *         The GUID to use for the merged node.
+   * @param  {BookmarkNode?} localNode
+   *         The local node. May be `null` if the node only exists remotely.
+   * @param  {BookmarkNode?} remoteNode
+   *         The remote node. May be `null` if the node only exists locally.
+   * @return {MergedBookmarkNode}
+   *         The merged node, with merged folder children.
+   */
+  mergeNode(mergedGuid, localNode, remoteNode) {
+    this.mergedGuids.add(mergedGuid);
+
+    if (localNode) {
+      if (localNode.guid != mergedGuid) {
+        // We deduped a NEW local item to a remote item.
+        this.mergedGuids.add(localNode.guid);
+      }
+
+      if (remoteNode) {
+        MirrorLog.trace("Item ${mergedGuid} exists locally as ${localNode} " +
+                        "and remotely as ${remoteNode}; merging",
+                        { mergedGuid, localNode, remoteNode });
+        let mergedNode = this.twoWayMerge(mergedGuid, localNode, remoteNode);
+        return mergedNode;
+      }
+
+      MirrorLog.trace("Item ${mergedGuid} only exists locally as " +
+                      "${localNode}; taking local state", { mergedGuid,
+                                                            localNode });
+      let mergedNode = new MergedBookmarkNode(mergedGuid, localNode, null,
+                                              BookmarkMergeState.local);
+      if (localNode.isFolder()) {
+        // The local folder doesn't exist remotely, but its children might, so
+        // we still need to recursively walk and merge them. This method will
+        // change the merge state from local to new if any children were moved
+        // or deleted.
+        this.mergeChildListsIntoMergedNode(mergedNode, localNode,
+                                           /* remoteNode */ null);
+      }
+      return mergedNode;
+    }
+
+    if (remoteNode) {
+      MirrorLog.trace("Item ${mergedGuid} only exists remotely as " +
+                      "${remoteNode}; taking remote state", { mergedGuid,
+                                                              remoteNode });
+      let mergedNode = new MergedBookmarkNode(mergedGuid, null, remoteNode,
+                                              BookmarkMergeState.remote);
+      if (remoteNode.isFolder()) {
+        // As above, a remote folder's children might still exist locally, so we
+        // need to merge them and update the merge state from remote to new if
+        // any children were moved or deleted.
+        this.mergeChildListsIntoMergedNode(mergedNode, /* localNode */ null,
+                                           remoteNode);
+      }
+      return mergedNode;
+    }
+
+    // Should never happen. We need to have at least one node for a two-way
+    // merge.
+    throw new TypeError("Can't merge two nonexistent nodes");
+  }
+
+  /**
+   * Merges two nodes that exist locally and remotely.
+   *
+   * @param  {String} mergedGuid
+   *         The GUID to use for the merged node.
+   * @param  {BookmarkNode} localNode
+   *         The existing local node.
+   * @param  {BookmarkNode} remoteNode
+   *         The existing remote node.
+   * @return {MergedBookmarkNode}
+   *         The merged node, with merged folder children.
+   */
+  twoWayMerge(mergedGuid, localNode, remoteNode) {
+    let mergeState = this.resolveTwoWayValueConflict(mergedGuid, localNode,
+                                                     remoteNode);
+    MirrorLog.trace("Merge state for ${mergedGuid} is ${mergeState}",
+                    { mergedGuid, mergeState });
+
+    let mergedNode = new MergedBookmarkNode(mergedGuid, localNode, remoteNode,
+                                            mergeState);
+
+    if (localNode.isFolder()) {
+      if (remoteNode.isFolder()) {
+        // Merging two folders, so we need to walk their children to handle
+        // structure changes.
+        MirrorLog.trace("Merging folders ${localNode} and ${remoteNode}",
+                        { localNode, remoteNode });
+        this.mergeChildListsIntoMergedNode(mergedNode, localNode, remoteNode);
+        return mergedNode;
+      }
+
+      if (remoteNode.kind == SyncedBookmarksMirror.KIND.LIVEMARK) {
+        // We allow merging local folders and remote livemarks because Places
+        // stores livemarks as empty folders with feed and site URL annotations.
+        // The livemarks service first inserts the folder, and *then* sets
+        // annotations. Since this isn't wrapped in a transaction, we might sync
+        // before the annotations are set, and upload a folder record instead
+        // of a livemark record (bug 632287), then replace the folder with a
+        // livemark on the next sync.
+        MirrorLog.trace("Merging local folder ${localNode} and remote " +
+                        "livemark ${remoteNode}", { localNode, remoteNode });
+        this.telemetryEvents.push({
+          value: "kind",
+          extra: { local: "folder", remote: "folder" },
+        });
+        return mergedNode;
+      }
+
+      MirrorLog.error("Merging local folder ${localNode} and remote " +
+                      "non-folder ${remoteNode}", { localNode, remoteNode });
+      throw new SyncedBookmarksMirror.ConsistencyError(
+        "Can't merge folder and non-folder");
+    }
+
+    if (localNode.kind == remoteNode.kind) {
+      // Merging two non-folders, so no need to walk children.
+      MirrorLog.trace("Merging non-folders ${localNode} and ${remoteNode}",
+                      { localNode, remoteNode });
+      return mergedNode;
+    }
+
+    MirrorLog.error("Merging local ${localNode} and remote ${remoteNode} " +
+                    "with different kinds", { localNode, remoteNode });
+    throw new SyncedBookmarksMirror.ConsistencyError(
+      "Can't merge different item kinds");
+  }
+
+  /**
+   * Determines the merge state for a node that exists locally and remotely.
+   *
+   * @param  {String} mergedGuid
+   *         The GUID of the merged node. This is the same as the remote GUID,
+   *         and usually the same as the local GUID. The local GUID may be
+   *         different if we're deduping a local item to a remote item.
+   * @param  {String} localNode
+   *         The local bookmark node.
+   * @param  {BookmarkNode} remoteNode
+   *         The remote bookmark node.
+   * @return {BookmarkMergeState}
+   *         The two-way merge state.
+   */
+  resolveTwoWayValueConflict(mergedGuid, localNode, remoteNode) {
+    if (!remoteNode.needsMerge) {
+      // The node wasn't changed remotely since the last sync. Keep the local
+      // state.
+      return BookmarkMergeState.local;
+    }
+    if (!localNode.needsMerge) {
+      // The node was changed remotely, but not locally. Take the remote state.
+      return BookmarkMergeState.remote;
+    }
+    // At this point, we know the item changed locally and remotely. We could
+    // query storage to determine if the value state is the same, as iOS does.
+    // However, that's an expensive check that requires joining `moz_bookmarks`,
+    // `moz_items_annos`, and `moz_places` to the mirror. It's unlikely that
+    // the value state is identical, so we skip the value check and use the
+    // timestamp to decide which node is newer.
+    let valueState = localNode.newerThan(remoteNode) ?
+                     BookmarkMergeState.local :
+                     BookmarkMergeState.remote;
+    return valueState;
+  }
+
+  /**
+   * Merges a remote child node into a merged folder node.
+   *
+   * @param  {MergedBookmarkNode} mergedNode
+   *         The merged folder node.
+   * @param  {BookmarkNode} remoteParentNode
+   *         The remote folder node.
+   * @param  {BookmarkNode} remoteChildNode
+   *         The remote child node.
+   * @return {Boolean}
+   *         `true` if the merged structure state changed because the remote
+   *         child was locally moved or deleted; `false` otherwise.
+   */
+  mergeRemoteChildIntoMergedNode(mergedNode, remoteParentNode,
+                                 remoteChildNode) {
+    if (this.mergedGuids.has(remoteChildNode.guid)) {
+      MirrorLog.trace("Remote child ${remoteChildNode} already seen in " +
+                      "another folder and merged", { remoteChildNode });
+      return false;
+    }
+
+    MirrorLog.trace("Merging remote child ${remoteChildNode} of " +
+                    "${remoteParentNode} into ${mergedNode}",
+                    { remoteChildNode, remoteParentNode, mergedNode });
+
+    // Make sure the remote child isn't locally deleted. If it is, we need
+    // to move all descendants that aren't also remotely deleted to the
+    // merged node. This handles the case where a user deletes a folder
+    // on this device, and adds a bookmark to the same folder on another
+    // device. We want to keep the folder deleted, but we also don't want
+    // to lose the new bookmark, so we move the bookmark to the deleted
+    // folder's parent.
+    let locallyDeleted = this.checkForLocalDeletionOfRemoteNode(mergedNode,
+      remoteChildNode);
+    if (locallyDeleted) {
+      return true;
+    }
+
+    // The remote child isn't locally deleted. Does it exist in the local tree?
+    let localChildNode = this.localTree.nodeForGuid(remoteChildNode.guid);
+    if (!localChildNode) {
+      // Remote child doesn't exist locally, either. Try to find a content
+      // match in the containing folder, and dedupe the local item if we can.
+      MirrorLog.trace("Remote child ${remoteChildNode} doesn't exist " +
+                      "locally; looking for content match",
+                      { remoteChildNode });
+
+      let localChildNodeByContent = this.findLocalNodeMatchingRemoteNode(
+        mergedNode, remoteChildNode);
+
+      let mergedChildNode = this.mergeNode(remoteChildNode.guid,
+                                           localChildNodeByContent,
+                                           remoteChildNode);
+      mergedNode.mergedChildren.push(mergedChildNode);
+      return false;
+    }
+
+    // Otherwise, the remote child exists in the local tree. Did it move?
+    let localParentNode = this.localTree.parentNodeFor(localChildNode);
+    if (!localParentNode) {
+      // Should never happen. The local tree must be complete.
+      MirrorLog.error("Remote child ${remoteChildNode} exists locally as " +
+                      "${localChildNode} without local parent",
+                      { remoteChildNode, localChildNode });
+      throw new SyncedBookmarksMirror.ConsistencyError(
+        "Local child node is orphan");
+    }
+
+    MirrorLog.trace("Remote child ${remoteChildNode} exists locally in " +
+                    "${localParentNode} and remotely in ${remoteParentNode}",
+                    { remoteChildNode, localParentNode, remoteParentNode });
+
+    if (localParentNode.needsMerge) {
+      if (remoteParentNode.needsMerge) {
+        MirrorLog.trace("Local ${localParentNode} and remote " +
+                        "${remoteParentNode} parents changed; comparing " +
+                        "modified times to decide parent for remote child " +
+                        "${remoteChildNode}",
+                        { localParentNode, remoteParentNode, remoteChildNode });
+
+        let latestLocalAge = Math.min(localChildNode.age,
+                                      localParentNode.age);
+        let latestRemoteAge = Math.min(remoteChildNode.age,
+                                       remoteParentNode.age);
+
+        if (latestLocalAge < latestRemoteAge) {
+          // Local move is younger, so we ignore the remote move. We'll
+          // merge the child later, when we walk its new local parent.
+          MirrorLog.trace("Ignoring older remote move for ${remoteChildNode} " +
+                          "to ${remoteParentNode} at ${latestRemoteAge}; " +
+                          "local move to ${localParentNode} at " +
+                          "${latestLocalAge} is newer",
+                          { remoteChildNode, remoteParentNode, latestRemoteAge,
+                            localParentNode, latestLocalAge });
+          return true;
+        }
+
+        // Otherwise, the remote move is younger, so we ignore the local move
+        // and merge the child now.
+        MirrorLog.trace("Taking newer remote move for ${remoteChildNode} to " +
+                        "${remoteParentNode} at ${latestRemoteAge}; local " +
+                        "move to ${localParentNode} at ${latestLocalAge} is " +
+                        "older", { remoteChildNode, remoteParentNode,
+                                   latestRemoteAge, localParentNode,
+                                   latestLocalAge });
+
+        let mergedChildNode = this.mergeNode(remoteChildNode.guid,
+                                             localChildNode, remoteChildNode);
+        mergedNode.mergedChildren.push(mergedChildNode);
+        return false;
+      }
+
+      MirrorLog.trace("Remote parent unchanged; keeping remote child " +
+                      "${remoteChildNode} in ${localParentNode}",
+                      { remoteChildNode, localParentNode });
+      return true;
+    }
+
+    MirrorLog.trace("Local parent unchanged; keeping remote child " +
+                    "${remoteChildNode} in ${remoteParentNode}",
+                    { remoteChildNode, remoteParentNode });
+
+    let mergedChildNode = this.mergeNode(remoteChildNode.guid, localChildNode,
+                                         remoteChildNode);
+    mergedNode.mergedChildren.push(mergedChildNode);
+    return false;
+  }
+
+  /**
+   * Merges a local child node into a merged folder node.
+   *
+   * @param  {MergedBookmarkNode} mergedNode
+   *         The merged folder node.
+   * @param  {BookmarkNode} localParentNode
+   *         The local folder node.
+   * @param  {BookmarkNode} localChildNode
+   *         The local child node.
+   * @return {Boolean}
+   *         `true` if the merged structure state changed because the local
+   *         child doesn't exist remotely or was locally moved; `false`
+   *         otherwise.
+   */
+  mergeLocalChildIntoMergedNode(mergedNode, localParentNode, localChildNode) {
+    if (this.mergedGuids.has(localChildNode.guid)) {
+      // We already merged the child when we walked another folder.
+      MirrorLog.trace("Local child ${localChildNode} already seen in " +
+                      "another folder and merged", { localChildNode });
+      return false;
+    }
+
+    MirrorLog.trace("Merging local child ${localChildNode} of " +
+                    "${localParentNode} into ${mergedNode}",
+                    { localChildNode, localParentNode, mergedNode });
+
+    // Now, we know we haven't seen the local child before, and it's not in
+    // this folder on the server. Check if the child is remotely deleted.
+    // If so, we need to move any new local descendants to the merged node,
+    // just as we did for new remote descendants of locally deleted parents.
+    let remotelyDeleted = this.checkForRemoteDeletionOfLocalNode(mergedNode,
+      localChildNode);
+    if (remotelyDeleted) {
+      return true;
+    }
+
+    // At this point, we know the local child isn't deleted. See if it
+    // exists in the remote tree.
+    let remoteChildNode = this.remoteTree.nodeForGuid(localChildNode.guid);
+    if (!remoteChildNode) {
+      // The local child doesn't exist remotely, but we still need to walk
+      // its children.
+      let mergedChildNode = this.mergeNode(localChildNode.guid, localChildNode,
+                                           /* remoteChildNode */ null);
+      mergedNode.mergedChildren.push(mergedChildNode);
+      return true;
+    }
+
+    // The local child exists remotely. It must have moved; otherwise, we
+    // would have seen it when we walked the remote children.
+    let remoteParentNode = this.remoteTree.parentNodeFor(remoteChildNode);
+    if (!remoteParentNode) {
+      // Should never happen. The remote tree must be complete.
+      MirrorLog.error("Local child ${localChildNode} exists remotely as " +
+                      "${remoteChildNode} without remote parent",
+                      { localChildNode, remoteChildNode });
+      throw new SyncedBookmarksMirror.ConsistencyError(
+        "Remote child node is orphan");
+    }
+
+    MirrorLog.trace("Local child ${localChildNode} exists locally in " +
+                    "${localParentNode} and remotely in ${remoteParentNode}",
+                    { localChildNode, localParentNode, remoteParentNode });
+
+    if (localParentNode.needsMerge) {
+      if (remoteParentNode.needsMerge) {
+        MirrorLog.trace("Local ${localParentNode} and remote " +
+                        "${remoteParentNode} parents changed; comparing " +
+                        "modified times to decide parent for local child " +
+                        "${localChildNode}", { localParentNode,
+                                               remoteParentNode,
+                                               localChildNode });
+
+        let latestLocalAge = Math.min(localChildNode.age,
+                                      localParentNode.age);
+        let latestRemoteAge = Math.min(remoteChildNode.age,
+                                       remoteParentNode.age);
+
+        if (latestRemoteAge <= latestLocalAge) {
+          MirrorLog.trace("Ignoring older local move for ${localChildNode} " +
+                          "to ${localParentNode} at ${latestLocalAge}; " +
+                          "remote move to ${remoteParentNode} at " +
+                          "${latestRemoteAge} is newer",
+                          { localChildNode, localParentNode, latestLocalAge,
+                            remoteParentNode, latestRemoteAge });
+          return false;
+        }
+
+        MirrorLog.trace("Taking newer local move for ${localChildNode} to " +
+                        "${localParentNode} at ${latestLocalAge}; remote " +
+                        "move to ${remoteParentNode} at ${latestRemoteAge} " +
+                        "is older", { localChildNode, localParentNode,
+                                      latestLocalAge, remoteParentNode,
+                                      latestRemoteAge });
+
+        let mergedChildNode = this.mergeNode(localChildNode.guid,
+                                             localChildNode, remoteChildNode);
+        mergedNode.mergedChildren.push(mergedChildNode);
+        return true;
+      }
+
+      MirrorLog.trace("Remote parent unchanged; keeping local child " +
+                      "${localChildNode} in local parent ${localParentNode}",
+                      { localChildNode, localParentNode });
+
+      let mergedChildNode = this.mergeNode(localChildNode.guid, localChildNode,
+                                           remoteChildNode);
+      mergedNode.mergedChildren.push(mergedChildNode);
+      return true;
+    }
+
+    MirrorLog.trace("Local parent unchanged; keeping local child " +
+                    "${localChildNode} in remote parent ${remoteParentNode}",
+                    { localChildNode, remoteParentNode });
+    return false;
+  }
+
+  /**
+   * Recursively merges the children of a local folder node and a matching
+   * remote folder node.
+   *
+   * @param {MergedBookmarkNode} mergedNode
+   *        The merged folder state. This method mutates the merged node to
+   *        append merged children, and change the node's merge state to new
+   *        if needed.
+   * @param {BookmarkNode?} localNode
+   *        The local folder node. May be `null` if the folder only exists
+   *        remotely.
+   * @param {BookmarkNode?} remoteNode
+   *        The remote folder node. May be `null` if the folder only exists
+   *        locally.
+   */
+  mergeChildListsIntoMergedNode(mergedNode, localNode, remoteNode) {
+    let mergeStateChanged = false;
+
+    // Walk and merge remote children first.
+    MirrorLog.trace("Merging remote children of ${remoteNode} into " +
+                    "${mergedNode}", { remoteNode, mergedNode });
+    if (remoteNode) {
+      for (let remoteChildNode of remoteNode.children) {
+        let remoteChildrenChanged = this.mergeRemoteChildIntoMergedNode(
+          mergedNode, remoteNode, remoteChildNode);
+        if (remoteChildrenChanged) {
+          mergeStateChanged = true;
+        }
+      }
+    }
+
+    // Now walk and merge any local children that we haven't already merged.
+    MirrorLog.trace("Merging local children of ${localNode} into " +
+                    "${mergedNode}", { localNode, mergedNode });
+    if (localNode) {
+      for (let localChildNode of localNode.children) {
+        let remoteChildrenChanged = this.mergeLocalChildIntoMergedNode(
+          mergedNode, localNode, localChildNode);
+        if (remoteChildrenChanged) {
+          mergeStateChanged = true;
+        }
+      }
+    }
+
+    // Update the merge state if we moved children orphaned on one side by a
+    // deletion on the other side, if we kept newer locally moved children,
+    // or if the child order changed. We already updated the merge state of the
+    // orphans, but we also need to flag the containing folder so that it's
+    // reuploaded to the server along with the new children.
+    if (mergeStateChanged) {
+      let newStructureNode = mergedNode.toBookmarkNode();
+      let newMergeState = BookmarkMergeState.new(mergedNode.mergeState,
+                                                 newStructureNode);
+      MirrorLog.trace("Merge state for ${mergedNode} has new structure " +
+                      "${newMergeState}", { mergedNode, newMergeState });
+      this.telemetryEvents.push({
+        value: "structure",
+        extra: { type: "new" },
+      });
+      mergedNode.mergeState = newMergeState;
+    }
+  }
+
+  /**
+   * Walks a locally deleted remote node's children, reparenting any children
+   * that aren't also deleted remotely to the merged node. Returns `true` if
+   * `remoteNode` is deleted locally; `false` if `remoteNode` is not deleted or
+   * doesn't exist locally.
+   *
+   * This is the inverse of `checkForRemoteDeletionOfLocalNode`.
+   */
+  checkForLocalDeletionOfRemoteNode(mergedNode, remoteNode) {
+    if (!this.localTree.isDeleted(remoteNode.guid)) {
+      return false;
+    }
+
+    if (remoteNode.needsMerge) {
+      if (!remoteNode.isFolder()) {
+        // If a non-folder child is deleted locally and changed remotely, we
+        // ignore the local deletion and take the remote child.
+        MirrorLog.trace("Remote non-folder ${remoteNode} deleted locally " +
+                        "and changed remotely; taking remote change",
+                        { remoteNode });
+        this.telemetryEvents.push({
+          value: "structure",
+          extra: { type: "delete", kind: "item", prefer: "remote" },
+        });
+        return false;
+      }
+      // For folders, we always take the local deletion and relocate remotely
+      // changed grandchildren to the merged node. We could use the mirror to
+      // revive the child folder, but it's easier to relocate orphaned
+      // grandchildren than to partially revive the child folder.
+      MirrorLog.trace("Remote folder ${remoteNode} deleted locally " +
+                      "and changed remotely; taking local deletion",
+                      { remoteNode });
+      this.telemetryEvents.push({
+        value: "structure",
+        extra: { type: "delete", kind: "folder", prefer: "local" },
+      });
+    } else {
+      MirrorLog.trace("Remote node ${remoteNode} deleted locally and not " +
+                       "changed remotely; taking local deletion",
+                       { remoteNode });
+    }
+
+    this.deleteRemotely.add(remoteNode.guid);
+
+    let mergedOrphanNodes = this.processRemoteOrphansForNode(mergedNode,
+                                                             remoteNode);
+    this.relocateOrphansTo(mergedNode, mergedOrphanNodes);
+    MirrorLog.trace("Relocating remote orphans ${mergedOrphanNodes} to " +
+                    "${mergedNode}", { mergedOrphanNodes, mergedNode });
+
+    return true;
+  }
+
+  /**
+   * Walks a remotely deleted local node's children, reparenting any children
+   * that aren't also deleted locally to the merged node. Returns `true` if
+   * `localNode` is deleted remotely; `false` if `localNode` is not deleted or
+   * doesn't exist locally.
+   *
+   * This is the inverse of `checkForLocalDeletionOfRemoteNode`.
+   */
+  checkForRemoteDeletionOfLocalNode(mergedNode, localNode) {
+    if (!this.remoteTree.isDeleted(localNode.guid)) {
+      return false;
+    }
+
+    if (localNode.needsMerge) {
+      if (!localNode.isFolder()) {
+        MirrorLog.trace("Local non-folder ${localNode} deleted remotely and " +
+                        "changed locally; taking local change", { localNode });
+        this.telemetryEvents.push({
+          value: "structure",
+          extra: { type: "delete", kind: "item", prefer: "local" },
+        });
+        return false;
+      }
+      MirrorLog.trace("Local folder ${localNode} deleted remotely and " +
+                      "changed locally; taking remote deletion", { localNode });
+      this.telemetryEvents.push({
+        value: "structure",
+        extra: { type: "delete", kind: "folder", prefer: "remote" },
+      });
+    } else {
+      MirrorLog.trace("Local node ${localNode} deleted remotely and not " +
+                      "changed locally; taking remote deletion", { localNode });
+    }
+
+    MirrorLog.trace("Local node ${localNode} deleted remotely; taking remote " +
+                    "deletion", { localNode });
+
+    this.deleteLocally.add(localNode.guid);
+
+    let mergedOrphanNodes = this.processLocalOrphansForNode(mergedNode,
+                                                            localNode);
+    this.relocateOrphansTo(mergedNode, mergedOrphanNodes);
+    MirrorLog.trace("Relocating local orphans ${mergedOrphanNodes} to " +
+                    "${mergedNode}", { mergedOrphanNodes, mergedNode });
+
+    return true;
+  }
+
+  /**
+   * Recursively merges all remote children of a locally deleted folder that
+   * haven't also been deleted remotely. This can happen if the user adds a
+   * bookmark to a folder on another device, and deletes that folder locally.
+   * This is the inverse of `processLocalOrphansForNode`.
+   */
+  processRemoteOrphansForNode(mergedNode, remoteNode) {
+    let remoteOrphanNodes = [];
+
+    for (let remoteChildNode of remoteNode.children) {
+      let locallyDeleted = this.checkForLocalDeletionOfRemoteNode(mergedNode,
+        remoteChildNode);
+      if (locallyDeleted) {
+        // The remote child doesn't exist locally, or is also deleted locally,
+        // so we can safely delete its parent.
+        continue;
+      }
+      remoteOrphanNodes.push(remoteChildNode);
+    }
+
+    let mergedOrphanNodes = [];
+    for (let remoteOrphanNode of remoteOrphanNodes) {
+      let localOrphanNode = this.localTree.nodeForGuid(remoteOrphanNode.guid);
+      let mergedOrphanNode = this.mergeNode(remoteOrphanNode.guid,
+                                            localOrphanNode, remoteOrphanNode);
+      mergedOrphanNodes.push(mergedOrphanNode);
+    }
+
+    return mergedOrphanNodes;
+  }
+
+  /**
+   * Recursively merges all local children of a remotely deleted folder that
+   * haven't also been deleted locally. This is the inverse of
+   * `processRemoteOrphansForNode`.
+   */
+  processLocalOrphansForNode(mergedNode, localNode) {
+    if (!localNode.isFolder()) {
+      // The local node isn't a folder, so it won't have orphans.
+      return [];
+    }
+
+    let localOrphanNodes = [];
+    for (let localChildNode of localNode.children) {
+      let remotelyDeleted = this.checkForRemoteDeletionOfLocalNode(mergedNode,
+        localChildNode);
+      if (remotelyDeleted) {
+        // The local child doesn't exist or is also deleted on the server, so we
+        // can safely delete its parent without orphaning any local children.
+        continue;
+      }
+      localOrphanNodes.push(localChildNode);
+    }
+
+    let mergedOrphanNodes = [];
+    for (let localOrphanNode of localOrphanNodes) {
+      let remoteOrphanNode = this.remoteTree.nodeForGuid(localOrphanNode.guid);
+      let mergedNode = this.mergeNode(localOrphanNode.guid,
+                                      localOrphanNode, remoteOrphanNode);
+      mergedOrphanNodes.push(mergedNode);
+    }
+
+    return mergedOrphanNodes;
+  }
+
+  /**
+   * Moves a list of merged orphan nodes to the closest surviving ancestor.
+   * Changes the merge state of the moved orphans to new, so that we reupload
+   * them along with their new parent on the next sync.
+   *
+   * @param {MergedBookmarkNode} mergedNode
+   * @param {MergedBookmarkNode[]} mergedOrphanNodes
+   */
+  relocateOrphansTo(mergedNode, mergedOrphanNodes) {
+    for (let mergedOrphanNode of mergedOrphanNodes) {
+      let newStructureNode = mergedOrphanNode.toBookmarkNode();
+      let newMergeState = BookmarkMergeState.new(mergedOrphanNode.mergeState,
+                                                 newStructureNode);
+      mergedOrphanNode.mergeState = newMergeState;
+      mergedNode.mergedChildren.push(mergedOrphanNode);
+    }
+  }
+
+  /**
+   * Finds a local node with a different GUID that matches the content of a
+   * remote node. This is used to dedupe local items that haven't been uploaded
+   * to remote items that don't exist locally.
+   *
+   * @param  {MergedBookmarkNode} mergedNode
+   *         The merged folder node.
+   * @param  {BookmarkNode} remoteChildNode
+   *         The remote child node.
+   * @return {BookmarkNode?}
+   *         A matching local child node, or `null` if there are no matching
+   *         local items.
+   */
+  findLocalNodeMatchingRemoteNode(mergedNode, remoteChildNode) {
+    let localParentNode = mergedNode.localNode;
+    if (!localParentNode) {
+      MirrorLog.trace("Merged node ${mergedNode} doesn't exist locally; no " +
+                      "potential dupes for ${remoteChildNode}",
+                      { mergedNode, remoteChildNode });
+      return null;
+    }
+    let remoteChildContent = this.newRemoteContents.get(remoteChildNode.guid);
+    if (!remoteChildContent) {
+      // The node doesn't exist locally, but it's also flagged as merged in the
+      // mirror.
+      return null;
+    }
+    let newLocalNode = null;
+    for (let localChildNode of localParentNode.children) {
+      if (this.mergedGuids.has(localChildNode.guid)) {
+        MirrorLog.trace("Not deduping ${localChildNode}; already seen in " +
+                        "another folder", { localChildNode });
+        continue;
+      }
+      if (!this.newLocalContents.has(localChildNode.guid)) {
+        MirrorLog.trace("Not deduping ${localChildNode}; already uploaded",
+                        { localChildNode });
+        continue;
+      }
+      let remoteCandidate = this.remoteTree.nodeForGuid(localChildNode.guid);
+      if (remoteCandidate) {
+        MirrorLog.trace("Not deduping ${localChildNode}; already exists " +
+                        "remotely", { localChildNode });
+        continue;
+      }
+      if (this.remoteTree.isDeleted(localChildNode.guid)) {
+        MirrorLog.trace("Not deduping ${localChildNode}; deleted on server",
+                        { localChildNode });
+        continue;
+      }
+      let localChildContent = this.newLocalContents.get(localChildNode.guid);
+      if (!contentsMatch(localChildNode, localChildContent, remoteChildNode,
+                         remoteChildContent)) {
+        MirrorLog.trace("${localChildNode} is not a dupe of ${remoteChildNode}",
+                        { localChildNode, remoteChildNode });
+        continue;
+      }
+      this.telemetryEvents.push({ value: "dupe" });
+      newLocalNode = localChildNode;
+      break;
+    }
+    return newLocalNode;
+  }
+}
+
+/**
+ * Determines if two new local and remote nodes are of the same kind, and have
+ * similar contents.
+ *
+ * - Bookmarks must have the same title and URL.
+ * - Smart bookmarks must have the same smart bookmark name. Other queries
+ *   must have the same title and query URL.
+ * - Folders and livemarks must have the same title.
+ * - Separators must have the same position within their parents.
+ *
+ * @param  {BookmarkNode} localNode
+ * @param  {BookmarkContent} localContent
+ * @param  {BookmarkNode} remoteNode
+ * @param  {BookmarkContent} remoteContent
+ * @return {Boolean}
+ */
+function contentsMatch(localNode, localContent, remoteNode, remoteContent) {
+  if (localNode.kind != remoteNode.kind) {
+    return false;
+  }
+  switch (localNode.kind) {
+    case SyncedBookmarksMirror.KIND.BOOKMARK:
+      return localContent.title == remoteContent.title &&
+             localContent.hasSameURL(remoteContent);
+
+    case SyncedBookmarksMirror.KIND.QUERY:
+      if (localContent.smartBookmarkName || remoteContent.smartBookmarkName) {
+        return localContent.smartBookmarkName ==
+               remoteContent.smartBookmarkName;
+      }
+      return localContent.title == remoteContent.title &&
+             localContent.hasSameURL(remoteContent);
+
+    case SyncedBookmarksMirror.KIND.FOLDER:
+    case SyncedBookmarksMirror.KIND.LIVEMARK:
+      return localContent.title == remoteContent.title;
+
+    case SyncedBookmarksMirror.KIND.SEPARATOR:
+      return localContent.position == remoteContent.position;
+  }
+  return false;
+}
+
+/**
+ * Records bookmark, annotation, and keyword observer notifications for all
+ * changes made during the merge, then fires the notifications after the merge
+ * is done.
+ *
+ * Recording bookmark changes and deletions is somewhat expensive, because we
+ * need to fetch all observer infos before writing. Making this more efficient
+ * is tracked in bug 1340498.
+ *
+ * Annotation observers don't require the extra context, so they're cheap to
+ * record and fire.
+ */
+class BookmarkObserverRecorder {
+  constructor(db) {
+    this.db = db;
+    this.bookmarkObserverNotifications = [];
+    this.annoObserverNotifications = [];
+    this.shouldInvalidateLivemarks = false;
+  }
+
+  /**
+   * Fires all recorded observer notifications, invalidates the livemark cache
+   * if necessary, and recalculates frecencies for changed URLs. This is called
+   * outside the merge transaction.
+   */
+  async notifyAll() {
+    this.notifyBookmarkObservers();
+    this.notifyAnnoObservers();
+    if (this.shouldInvalidateLivemarks) {
+      await PlacesUtils.livemarks.invalidateCachedLivemarks();
+    }
+    await this.updateFrecencies();
+  }
+
+  async updateFrecencies() {
+    await this.db.execute(`
+      UPDATE moz_places SET
+        frecency = CALCULATE_FRECENCY(id)
+      WHERE frecency = -1`);
+  }
+
+  noteItemAdded(info) {
+    let uri = info.urlHref ? Services.io.newURI(info.urlHref) : null;
+    this.bookmarkObserverNotifications.push({
+      name: "onItemAdded",
+      isTagging: info.isTagging,
+      args: [info.id, info.parentId, info.position, info.type, uri, info.title,
+        info.dateAdded, info.guid, info.parentGuid,
+        PlacesUtils.bookmarks.SOURCES.SYNC],
+    });
+  }
+
+  noteGuidChanged(info) {
+    PlacesUtils.invalidateCachedGuidFor(info.id);
+    this.bookmarkObserverNotifications.push({
+      name: "onItemChanged",
+      isTagging: false,
+      args: [info.id, "guid", /* isAnnotationProperty */ false, info.newGuid,
+        info.lastModified, info.type, info.parentId, info.newGuid,
+        info.parentGuid, info.oldGuid, PlacesUtils.bookmarks.SOURCES.SYNC],
+    });
+  }
+
+  noteItemMoved(info) {
+    this.bookmarkObserverNotifications.push({
+      name: "onItemMoved",
+      isTagging: false,
+      args: [info.id, info.oldParentId, info.oldPosition, info.newParentId,
+        info.newPosition, info.type, info.guid, info.oldParentGuid,
+        info.newParentGuid, PlacesUtils.bookmarks.SOURCES.SYNC],
+    });
+  }
+
+  noteItemChanged(info) {
+    if (info.oldTitle != info.newTitle) {
+      this.bookmarkObserverNotifications.push({
+        name: "onItemChanged",
+        isTagging: false,
+        args: [info.id, "title", /* isAnnotationProperty */ false,
+          info.newTitle, info.lastModified, info.type, info.parentId,
+          info.guid, info.parentGuid, info.oldTitle,
+          PlacesUtils.bookmarks.SOURCES.SYNC],
+      });
+    }
+    if (info.oldURLHref != info.newURLHref) {
+      this.bookmarkObserverNotifications.push({
+        name: "onItemChanged",
+        isTagging: false,
+        args: [info.id, "uri", /* isAnnotationProperty */ false,
+          info.newURLHref, info.lastModified, info.type, info.parentId,
+          info.guid, info.parentGuid, info.oldURLHref,
+          PlacesUtils.bookmarks.SOURCES.SYNC],
+      });
+    }
+  }
+
+  noteItemRemoved(info) {
+    let uri = info.urlHref ? Services.io.newURI(info.urlHref) : null;
+    this.bookmarkObserverNotifications.push({
+      name: "onItemRemoved",
+      isTagging: info.isUntagging,
+      args: [info.id, info.parentId, info.position, info.type, uri, info.guid,
+        info.parentGuid, PlacesUtils.bookmarks.SOURCES.SYNC],
+    });
+  }
+
+  noteKeywordChanged(info) {
+    this.bookmarkObserverNotifications.push({
+      name: "onItemChanged",
+      isTagging: false,
+      args: [info.id, "keyword", /* isAnnotationProperty */ false, info.keyword,
+        info.lastModified, info.type, info.parentId, info.guid, info.parentGuid,
+      /* oldValue */ info.urlHref, PlacesUtils.bookmarks.SOURCES.SYNC],
+    });
+  }
+
+  noteAnnoSet(id, name) {
+    if (isLivemarkAnno(name)) {
+      this.shouldInvalidateLivemarks = true;
+    }
+    this.annoObserverNotifications.push({
+      name: "onItemAnnotationSet",
+      args: [id, name, PlacesUtils.bookmarks.SOURCES.SYNC],
+    });
+  }
+
+  noteAnnoRemoved(id, name) {
+    if (isLivemarkAnno(name)) {
+      this.shouldInvalidateLivemarks = true;
+    }
+    this.annoObserverNotifications.push({
+      name: "onItemAnnotationRemoved",
+      args: [id, name, PlacesUtils.bookmarks.SOURCES.SYNC],
+    });
+  }
+
+  notifyBookmarkObservers() {
+    MirrorLog.debug("Notifying bookmark observers");
+    let observers = PlacesUtils.bookmarks.getObservers();
+    for (let observer of observers) {
+      this.notifyObserver(observer, "onBeginUpdateBatch");
+      for (let info of this.bookmarkObserverNotifications) {
+        if (info.isTagging && observer.skipTags) {
+          continue;
+        }
+        this.notifyObserver(observer, info.name, info.args);
+      }
+      this.notifyObserver(observer, "onEndUpdateBatch");
+    }
+  }
+
+  notifyAnnoObservers() {
+    MirrorLog.debug("Notifying anno observers");
+    let observers = PlacesUtils.annotations.getObservers();
+    for (let observer of observers) {
+      for (let { name, args } of this.annoObserverNotifications) {
+        this.notifyObserver(observer, name, args);
+      }
+    }
+  }
+
+  notifyObserver(observer, notification, args = []) {
+    try {
+      observer[notification](...args);
+    } catch (ex) {
+      MirrorLog.warn("Error notifying observer", ex);
+    }
+  }
+}
+
+function isLivemarkAnno(name) {
+  return name == PlacesUtils.LMANNO_FEEDURI ||
+         name == PlacesUtils.LMANNO_SITEURI;
+}
+
+/**
+ * Holds Sync metadata and the cleartext for a locally changed record. The
+ * bookmarks engine inflates a Sync record from the cleartext, and updates the
+ * `synced` property for successfully uploaded items.
+ *
+ * At the end of the sync, the engine writes the uploaded cleartext back to the
+ * mirror, and passes the updated change record as part of the changeset to
+ * `PlacesSyncUtils.bookmarks.pushChanges`.
+ */
+class BookmarkChangeRecord {
+  constructor(syncChangeCounter, cleartext) {
+    this.tombstone = cleartext.deleted === true;
+    this.counter = syncChangeCounter;
+    this.cleartext = cleartext;
+    this.synced = false;
+  }
+}
+
+// In conclusion, this is why bookmark syncing is hard.
diff --git a/toolkit/components/places/moz.build b/toolkit/components/places/moz.build
index aeb9b141ff42..629ab1ec5e0d 100644
--- a/toolkit/components/places/moz.build
+++ b/toolkit/components/places/moz.build
@@ -74,6 +74,7 @@ if CONFIG['MOZ_PLACES']:
         'PlacesSyncUtils.jsm',
         'PlacesTransactions.jsm',
         'PlacesUtils.jsm',
+        'SyncedBookmarksMirror.jsm',
     ]
 
     EXTRA_COMPONENTS += [
diff --git a/toolkit/components/places/tests/PlacesTestUtils.jsm b/toolkit/components/places/tests/PlacesTestUtils.jsm
index c145c985ab88..d748b4ebb988 100644
--- a/toolkit/components/places/tests/PlacesTestUtils.jsm
+++ b/toolkit/components/places/tests/PlacesTestUtils.jsm
@@ -356,4 +356,42 @@ this.PlacesTestUtils = Object.freeze({
       PlacesUtils[type].addObserver(proxifiedObserver);
     });
   },
+
+  /**
+   * A debugging helper that dumps the contents of an SQLite table.
+   *
+   * @param {Sqlite.OpenedConnection} db
+   *        The mirror database connection.
+   * @param {String} table
+   *        The table name.
+   */
+  async dumpTable(db, table) {
+    let rows = await db.execute(`SELECT * FROM ${table}`);
+    dump(`Table ${table} contains ${rows.length} rows\n`);
+
+    let results = [];
+    for (let row of rows) {
+      let numColumns = row.numEntries;
+      let rowValues = [];
+      for (let i = 0; i < numColumns; ++i) {
+        switch (row.getTypeOfIndex(i)) {
+          case Ci.mozIStorageValueArray.VALUE_TYPE_NULL:
+            rowValues.push("NULL");
+            break;
+          case Ci.mozIStorageValueArray.VALUE_TYPE_INTEGER:
+            rowValues.push(row.getInt64(i));
+            break;
+          case Ci.mozIStorageValueArray.VALUE_TYPE_FLOAT:
+            rowValues.push(row.getDouble(i));
+            break;
+          case Ci.mozIStorageValueArray.VALUE_TYPE_TEXT:
+            rowValues.push(JSON.stringify(row.getString(i)));
+            break;
+        }
+      }
+      results.push(rowValues.join("\t"));
+    }
+    results.push("\n");
+    dump(results.join("\n"));
+  },
 });
diff --git a/toolkit/components/places/tests/moz.build b/toolkit/components/places/tests/moz.build
index 935a80c1b72e..dde31d186eec 100644
--- a/toolkit/components/places/tests/moz.build
+++ b/toolkit/components/places/tests/moz.build
@@ -18,6 +18,7 @@ XPCSHELL_TESTS_MANIFESTS += [
     'legacy/xpcshell.ini',
     'migration/xpcshell.ini',
     'queries/xpcshell.ini',
+    'sync/xpcshell.ini',
     'unifiedcomplete/xpcshell.ini',
     'unit/xpcshell.ini',
 ]
diff --git a/toolkit/components/places/tests/sync/head_sync.js b/toolkit/components/places/tests/sync/head_sync.js
new file mode 100644
index 000000000000..1c2f91305cff
--- /dev/null
+++ b/toolkit/components/places/tests/sync/head_sync.js
@@ -0,0 +1,190 @@
+const { utils: Cu, interfaces: Ci, classes: Cc, results: Cr } = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/ObjectUtils.jsm");
+Cu.import("resource://gre/modules/osfile.jsm");
+Cu.import("resource://gre/modules/PlacesUtils.jsm");
+Cu.import("resource://gre/modules/PlacesSyncUtils.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/SyncedBookmarksMirror.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+Cu.import("resource://testing-common/PlacesTestUtils.jsm");
+Cu.import("resource://testing-common/httpd.js");
+
+function run_test() {
+  let bufLog = Log.repository.getLogger("Sync.Engine.Bookmarks.Mirror");
+  bufLog.level = Log.Level.Error;
+
+  let sqliteLog = Log.repository.getLogger("Sqlite");
+  sqliteLog.level = Log.Level.Error;
+
+  let formatter = new Log.BasicFormatter();
+  let appender = new Log.DumpAppender(formatter);
+  appender.level = Log.Level.All;
+
+  for (let log of [bufLog, sqliteLog]) {
+    log.addAppender(appender);
+  }
+
+  do_get_profile();
+  run_next_test();
+}
+
+function inspectChangeRecords(changeRecords) {
+  let results = { updated: [], deleted: [] };
+  for (let [id, record] of Object.entries(changeRecords)) {
+    (record.tombstone ? results.deleted : results.updated).push(id);
+  }
+  results.updated.sort();
+  results.deleted.sort();
+  return results;
+}
+
+async function assertLocalTree(rootGuid, expected, message) {
+  function bookmarkNodeToInfo(node) {
+    let { guid, index, title, typeCode: type } = node;
+    let itemInfo = { guid, index, title, type };
+    if (node.annos) {
+      let syncableAnnos = node.annos.filter(anno => [
+        PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO,
+        PlacesSyncUtils.bookmarks.SIDEBAR_ANNO,
+        PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO,
+        PlacesUtils.LMANNO_FEEDURI,
+        PlacesUtils.LMANNO_SITEURI,
+      ].includes(anno.name));
+      if (syncableAnnos.length) {
+        itemInfo.annos = syncableAnnos;
+      }
+    }
+    if (node.uri) {
+      itemInfo.url = node.uri;
+    }
+    if (node.keyword) {
+      itemInfo.keyword = node.keyword;
+    }
+    if (node.children) {
+      itemInfo.children = node.children.map(bookmarkNodeToInfo);
+    }
+    return itemInfo;
+  }
+  let root = await PlacesUtils.promiseBookmarksTree(rootGuid);
+  let actual = bookmarkNodeToInfo(root);
+  if (!ObjectUtils.deepEqual(actual, expected)) {
+    info(`Expected structure for ${rootGuid}: ${JSON.stringify(expected)}`);
+    info(`Actual structure for ${rootGuid}: ${JSON.stringify(actual)}`);
+    throw new Assert.constructor.AssertionError({ actual, expected, message });
+  }
+}
+
+function makeLivemarkServer() {
+  let server = new HttpServer();
+  server.registerPrefixHandler("/feed/", do_get_file("./livemark.xml"));
+  server.start(-1);
+  return {
+    server,
+    get site() {
+      let { identity } = server;
+      let host = identity.primaryHost.includes(":") ?
+        `[${identity.primaryHost}]` : identity.primaryHost;
+      return `${identity.primaryScheme}://${host}:${identity.primaryPort}`;
+    },
+    stopServer() {
+      return new Promise(resolve => server.stop(resolve));
+    },
+  };
+}
+
+function shuffle(array) {
+  let results = [];
+  for (let i = 0; i < array.length; ++i) {
+    let randomIndex = Math.floor(Math.random() * (i + 1));
+    results[i] = results[randomIndex];
+    results[randomIndex] = array[i];
+  }
+  return results;
+}
+
+async function fetchAllKeywords(info) {
+  let entries = [];
+  await PlacesUtils.keywords.fetch(info, entry => entries.push(entry));
+  return entries;
+}
+
+async function openMirror(name) {
+  let path = OS.Path.join(OS.Constants.Path.profileDir, `${name}_buf.sqlite`);
+  let buf = await SyncedBookmarksMirror.open({
+    path,
+    recordTelemetryEvent() {},
+  });
+  return buf;
+}
+
+function BookmarkObserver({ ignoreDates = true, skipTags = false } = {}) {
+  this.notifications = [];
+  this.ignoreDates = ignoreDates;
+  this.skipTags = skipTags;
+}
+
+BookmarkObserver.prototype = {
+  onBeginUpdateBatch() {},
+  onEndUpdateBatch() {},
+  onItemAdded(itemId, parentId, index, type, uri, title, dateAdded, guid,
+              parentGuid, source) {
+    let urlHref = uri ? uri.spec : null;
+    let params = { itemId, parentId, index, type, urlHref, title, guid,
+                   parentGuid, source };
+    if (!this.ignoreDates) {
+      params.dateAdded = dateAdded;
+    }
+    this.notifications.push({ name: "onItemAdded", params });
+  },
+  onItemRemoved(itemId, parentId, index, type, uri, guid, parentGuid, source) {
+    let urlHref = uri ? uri.spec : null;
+    this.notifications.push({
+      name: "onItemRemoved",
+      params: { itemId, parentId, index, type, urlHref, guid, parentGuid,
+                 source },
+    });
+  },
+  onItemChanged(itemId, property, isAnnoProperty, newValue, lastModified, type,
+                parentId, guid, parentGuid, oldValue, source) {
+    let params = { itemId, property, isAnnoProperty, newValue, type, parentId,
+                   guid, parentGuid, oldValue, source };
+    if (!this.ignoreDates) {
+      params.lastModified = lastModified;
+    }
+    this.notifications.push({ name: "onItemChanged", params });
+  },
+  onItemVisited() {},
+  onItemMoved(itemId, oldParentId, oldIndex, newParentId, newIndex, type, guid,
+              oldParentGuid, newParentGuid, source) {
+    this.notifications.push({
+      name: "onItemMoved",
+      params: { itemId, oldParentId, oldIndex, newParentId, newIndex, type,
+                guid, oldParentGuid, newParentGuid, source },
+    });
+  },
+
+  QueryInterface: XPCOMUtils.generateQI([
+    Ci.nsINavBookmarkObserver,
+  ]),
+
+  check(expectedNotifications) {
+    PlacesUtils.bookmarks.removeObserver(this);
+    if (!ObjectUtils.deepEqual(this.notifications, expectedNotifications)) {
+      info(`Expected notifications: ${JSON.stringify(expectedNotifications)}`);
+      info(`Actual notifications: ${JSON.stringify(this.notifications)}`);
+      throw new Assert.constructor.AssertionError({
+        actual: this.notifications,
+        expected: expectedNotifications,
+      });
+    }
+  },
+};
+
+function expectBookmarkChangeNotifications(options) {
+  let observer = new BookmarkObserver(options);
+  PlacesUtils.bookmarks.addObserver(observer);
+  return observer;
+}
diff --git a/toolkit/components/places/tests/sync/livemark.xml b/toolkit/components/places/tests/sync/livemark.xml
new file mode 100644
index 000000000000..db2ea9023ce1
--- /dev/null
+++ b/toolkit/components/places/tests/sync/livemark.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>Livemark Feed</title>
+  <link href="https://example.com/"/>
+  <updated>2016-08-09T19:51:45.147Z</updated>
+  <author>
+    <name>John Doe</name>
+  </author>
+  <id>urn:uuid:e7947414-6ee0-4009-ae75-8b0ad3c6894b</id>
+  <entry>
+    <title>Some awesome article</title>
+    <link href="https://example.com/some-article"/>
+    <id>urn:uuid:d72ce019-0a56-4a0b-ac03-f66117d78141</id>
+    <updated>2016-08-09T19:57:22.178Z</updated>
+    <summary>My great article summary.</summary>
+  </entry>
+</feed>
diff --git a/toolkit/components/places/tests/sync/test_bookmark_corruption.js b/toolkit/components/places/tests/sync/test_bookmark_corruption.js
new file mode 100644
index 000000000000..70611f812377
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_corruption.js
@@ -0,0 +1,1000 @@
+add_task(async function test_missing_children() {
+  let buf = await openMirror("missing_childen");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes: A > ([B] C [D E])");
+  {
+    await buf.store(shuffle([{
+      id: "menu",
+      type: "folder",
+      title: "Bookmarks Menu",
+      children: ["bookmarkBBBB", "bookmarkCCCC", "bookmarkDDDD",
+                 "bookmarkEEEE"],
+    }, {
+      id: "bookmarkCCCC",
+      type: "bookmark",
+      bmkUri: "http://example.com/c",
+      title: "C",
+    }]));
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload menu with missing children (B D E)");
+    await assertLocalTree(PlacesUtils.bookmarks.menuGuid, {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "C",
+        url: "http://example.com/c",
+      }],
+    }, "Menu children should be (C)");
+    let { missingChildren } = await buf.fetchRemoteOrphans();
+    deepEqual(missingChildren.sort(), ["bookmarkBBBB", "bookmarkDDDD",
+      "bookmarkEEEE"], "Should report (B D E) as missing");
+  }
+
+  info("Add (B E) to remote");
+  {
+    await buf.store(shuffle([{
+      id: "bookmarkBBBB",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b",
+    }, {
+      id: "bookmarkEEEE",
+      type: "bookmark",
+      title: "E",
+      bmkUri: "http://example.com/e",
+    }]));
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload menu with missing child D");
+    await assertLocalTree(PlacesUtils.bookmarks.menuGuid, {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "bookmarkBBBB",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "B",
+        url: "http://example.com/b",
+      }, {
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 1,
+        title: "C",
+        url: "http://example.com/c",
+      }, {
+        guid: "bookmarkEEEE",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 2,
+        title: "E",
+        url: "http://example.com/e",
+      }],
+    }, "Menu children should be (B C E)");
+    let { missingChildren } = await buf.fetchRemoteOrphans();
+    deepEqual(missingChildren, ["bookmarkDDDD"],
+      "Should report (D) as missing");
+  }
+
+  info("Add D to remote");
+  {
+    await buf.store([{
+      id: "bookmarkDDDD",
+      type: "bookmark",
+      title: "D",
+      bmkUri: "http://example.com/d",
+    }]);
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload complete menu");
+    await assertLocalTree(PlacesUtils.bookmarks.menuGuid, {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "bookmarkBBBB",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "B",
+        url: "http://example.com/b",
+      }, {
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 1,
+        title: "C",
+        url: "http://example.com/c",
+      }, {
+        guid: "bookmarkDDDD",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 2,
+        title: "D",
+        url: "http://example.com/d",
+      }, {
+        guid: "bookmarkEEEE",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 3,
+        title: "E",
+        url: "http://example.com/e",
+      }],
+    }, "Menu children should be (B C D E)");
+    let { missingChildren } = await buf.fetchRemoteOrphans();
+    deepEqual(missingChildren, [], "Should not report any missing children");
+  }
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_new_orphan_without_local_parent() {
+  let buf = await openMirror("new_orphan_without_local_parent");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // A doesn't exist locally, so we move the bookmarks into "unfiled" without
+  // reuploading. When the partial uploader returns and uploads A, we'll
+  // move the bookmarks to the correct folder.
+  info("Make remote changes: [A] > (B C D)");
+  await buf.store(shuffle([{
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B (remote)",
+    bmkUri: "http://example.com/b-remote",
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C (remote)",
+    bmkUri: "http://example.com/c-remote",
+  }, {
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D (remote)",
+    bmkUri: "http://example.com/d-remote",
+  }]));
+
+  info("Apply remote with (B C D)");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload orphans (B C D)");
+  }
+
+  await assertLocalTree(PlacesUtils.bookmarks.unfiledGuid, {
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 3,
+    title: "Other Bookmarks",
+    children: [{
+      guid: "bookmarkBBBB",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 0,
+      title: "B (remote)",
+      url: "http://example.com/b-remote",
+    }, {
+      guid: "bookmarkCCCC",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 1,
+      title: "C (remote)",
+      url: "http://example.com/c-remote",
+    }, {
+      guid: "bookmarkDDDD",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 2,
+      title: "D (remote)",
+      url: "http://example.com/d-remote",
+    }],
+  }, "Should move (B C D) to unfiled");
+
+  // A is an orphan because we don't have E locally, but we should move
+  // (B C D) into A.
+  info("Add [E] > A to remote");
+  await buf.store([{
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkDDDD", "bookmarkCCCC", "bookmarkBBBB"],
+  }]);
+
+  info("Apply remote with A");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload orphan A");
+  }
+
+  await assertLocalTree(PlacesUtils.bookmarks.unfiledGuid, {
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 3,
+    title: "Other Bookmarks",
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "A",
+      children: [{
+        guid: "bookmarkDDDD",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "D (remote)",
+        url: "http://example.com/d-remote",
+      }, {
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 1,
+        title: "C (remote)",
+        url: "http://example.com/c-remote",
+      }, {
+        guid: "bookmarkBBBB",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 2,
+        title: "B (remote)",
+        url: "http://example.com/b-remote",
+      }],
+    }],
+  }, "Should move (D C B) into A");
+
+  info("Add E to remote");
+  await buf.store([{
+    id: "folderEEEEEE",
+    type: "folder",
+    title: "E",
+    children: ["folderAAAAAA"],
+  }]);
+
+  info("Apply remote with E");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload orphan E");
+  }
+
+  // E is still in unfiled because we don't have a record for the menu.
+  await assertLocalTree(PlacesUtils.bookmarks.unfiledGuid, {
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 3,
+    title: "Other Bookmarks",
+    children: [{
+      guid: "folderEEEEEE",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "E",
+      children: [{
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "A",
+        children: [{
+          guid: "bookmarkDDDD",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "D (remote)",
+          url: "http://example.com/d-remote",
+        }, {
+          guid: "bookmarkCCCC",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "C (remote)",
+          url: "http://example.com/c-remote",
+        }, {
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "B (remote)",
+          url: "http://example.com/b-remote",
+        }],
+      }],
+    }],
+  }, "Should move A into E");
+
+  info("Add Menu > E to remote");
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderEEEEEE"],
+  }]);
+
+  info("Apply remote with menu");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload after forming complete tree");
+  }
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "folderEEEEEE",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "E",
+        children: [{
+          guid: "folderAAAAAA",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "A",
+          children: [{
+            guid: "bookmarkDDDD",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "D (remote)",
+            url: "http://example.com/d-remote",
+          }, {
+            guid: "bookmarkCCCC",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 1,
+            title: "C (remote)",
+            url: "http://example.com/c-remote",
+          }, {
+            guid: "bookmarkBBBB",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 2,
+            title: "B (remote)",
+            url: "http://example.com/b-remote",
+          }],
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should form complete tree after applying E");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_move_into_orphaned() {
+  let buf = await openMirror("move_into_orphaned");
+
+  info("Set up mirror: Menu > (A B (C > (D (E > F))))");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "bookmarkAAAA",
+      url: "http://example.com/a",
+      title: "A",
+    }, {
+      guid: "bookmarkBBBB",
+      url: "http://example.com/b",
+      title: "B",
+    }, {
+      guid: "folderCCCCCC",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "C",
+      children: [{
+        guid: "bookmarkDDDD",
+        title: "D",
+        url: "http://example.com/d",
+      }, {
+        guid: "folderEEEEEE",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "E",
+        children: [{
+          guid: "bookmarkFFFF",
+          title: "F",
+          url: "http://example.com/f",
+        }],
+      }],
+    }],
+  });
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkAAAA", "bookmarkBBBB", "folderCCCCCC"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+  }, {
+    id: "folderCCCCCC",
+    type: "folder",
+    title: "C",
+    children: ["bookmarkDDDD", "folderEEEEEE"],
+  }, {
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D",
+    bmkUri: "http://example.com/d",
+  }, {
+    id: "folderEEEEEE",
+    type: "folder",
+    title: "E",
+    children: ["bookmarkFFFF"],
+  }, {
+    id: "bookmarkFFFF",
+    type: "bookmark",
+    title: "F",
+    bmkUri: "http://example.com/f",
+  }], { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: delete D, add E > I");
+  await PlacesUtils.bookmarks.remove("bookmarkDDDD");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkIIII",
+    parentGuid: "folderEEEEEE",
+    title: "I (local)",
+    url: "http://example.com/i",
+  });
+
+  // G doesn't exist on the server.
+  info("Make remote changes: ([G] > A (C > (D H E))), (C > H)");
+  await buf.store(shuffle([{
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+  }, {
+    id: "folderCCCCCC",
+    type: "folder",
+    title: "C",
+    children: ["bookmarkDDDD", "bookmarkHHHH", "folderEEEEEE"],
+  }, {
+    id: "bookmarkHHHH",
+    type: "bookmark",
+    title: "H (remote)",
+    bmkUri: "http://example.com/h-remote",
+  }]));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkIIII", "folderCCCCCC", "folderEEEEEE"],
+    deleted: ["bookmarkDDDD"],
+  }, "Should upload records for (I C E); tombstone for D");
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        // A remains in its original place, since we don't use the `parentid`,
+        // and we don't have a record for G.
+        guid: "bookmarkAAAA",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "A",
+        url: "http://example.com/a",
+      }, {
+        guid: "bookmarkBBBB",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 1,
+        title: "B",
+        url: "http://example.com/b",
+      }, {
+        // C exists on the server, so we take its children and order. D was
+        // deleted locally, and doesn't exist remotely. C is also a child of
+        // G, but we don't have a record for it on the server.
+        guid: "folderCCCCCC",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 2,
+        title: "C",
+        children: [{
+          guid: "bookmarkHHHH",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "H (remote)",
+          url: "http://example.com/h-remote",
+        }, {
+          guid: "folderEEEEEE",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "E",
+          children: [{
+            guid: "bookmarkFFFF",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "F",
+            url: "http://example.com/f",
+          }, {
+            guid: "bookmarkIIII",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 1,
+            title: "I (local)",
+            url: "http://example.com/i",
+          }],
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should treat local tree as canonical if server is missing new parent");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_new_orphan_with_local_parent() {
+  let buf = await openMirror("new_orphan_with_local_parent");
+
+  info("Set up mirror: A > (B D)");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      }, {
+        guid: "bookmarkEEEE",
+        url: "http://example.com/e",
+        title: "E",
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkBBBB", "bookmarkEEEE"],
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E",
+    bmkUri: "http://example.com/e",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // Simulate a partial write by another device that uploaded only B and C. A
+  // exists locally, so we can move B and C into the correct folder, but not
+  // the correct positions.
+  info("Set up remote with orphans: [A] > (C D)");
+  await buf.store([{
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D (remote)",
+    bmkUri: "http://example.com/d-remote",
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C (remote)",
+    bmkUri: "http://example.com/c-remote",
+  }]);
+
+  info("Apply remote with (C D)");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload orphans (C D)");
+  }
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "A",
+        children: [{
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "B",
+          url: "http://example.com/b",
+        }, {
+          guid: "bookmarkEEEE",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "E",
+          url: "http://example.com/e",
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+      children: [{
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "C (remote)",
+        url: "http://example.com/c-remote",
+      }, {
+        guid: "bookmarkDDDD",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 1,
+        title: "D (remote)",
+        url: "http://example.com/d-remote",
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should move (C D) to unfiled");
+
+  // The partial uploader returns and uploads A.
+  info("Add A to remote");
+  await buf.store([{
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkCCCC", "bookmarkDDDD", "bookmarkEEEE", "bookmarkBBBB"],
+  }]);
+
+  info("Apply remote with A");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(idsToUpload, {
+      updated: [],
+      deleted: [],
+    }, "Should not reupload orphan A");
+  }
+
+  await assertLocalTree("folderAAAAAA", {
+    guid: "folderAAAAAA",
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "A",
+    children: [{
+      guid: "bookmarkCCCC",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 0,
+      title: "C (remote)",
+      url: "http://example.com/c-remote",
+    }, {
+      guid: "bookmarkDDDD",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 1,
+      title: "D (remote)",
+      url: "http://example.com/d-remote",
+    }, {
+      guid: "bookmarkEEEE",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 2,
+      title: "E",
+      url: "http://example.com/e",
+    }, {
+      guid: "bookmarkBBBB",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 3,
+      title: "B",
+      url: "http://example.com/b",
+    }],
+  }, "Should update child positions once A exists in mirror");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_tombstone_as_child() {
+  // TODO (Bug 1433180): Add a folder that mentions a tombstone in its
+  // `children`.
+});
+
+add_task(async function test_left_pane_root() {
+  // TODO (Bug 1433182): Add a left pane root to the mirror.
+});
+
+add_task(async function test_partial_cycle() {
+  let buf = await openMirror("partial_cycle");
+
+  info("Set up mirror: Menu > A > B > C");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+        children: [{
+          guid: "bookmarkCCCC",
+          url: "http://example.com/c",
+          title: "C",
+        }],
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["folderBBBBBB"],
+  }, {
+    id: "folderBBBBBB",
+    type: "folder",
+    title: "B",
+    children: ["bookmarkCCCC"],
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // Try to create a cycle: move A into B, and B into the menu, but don't upload
+  // a record for the menu. B is still a child of A locally. Since we ignore the
+  // `parentid`, we'll move (B A) into unfiled.
+  info("Make remote changes: A > C");
+  await buf.store([{
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A (remote)",
+    children: ["bookmarkCCCC"],
+  }, {
+    id: "folderBBBBBB",
+    type: "folder",
+    title: "B (remote)",
+    children: ["folderAAAAAA"],
+  }]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, { updated: [], deleted: [] },
+    "Should not mark any local items for upload");
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+      children: [{
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "B (remote)",
+        children: [{
+          guid: "folderAAAAAA",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "A (remote)",
+          children: [{
+            guid: "bookmarkCCCC",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "C",
+            url: "http://example.com/c",
+          }],
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should move A and B to unfiled");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_complete_cycle() {
+  let buf = await openMirror("complete_cycle");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // This test is order-dependent. We shouldn't recurse infinitely, but,
+  // depending on the order of the records, we might ignore the circular
+  // subtree because there's nothing linking it back to the rest of the
+  // tree.
+  info("Make remote changes: Menu > A > B > C > A");
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["folderBBBBBB"],
+  }, {
+    id: "folderBBBBBB",
+    type: "folder",
+    title: "B",
+    children: ["folderCCCCCC"],
+  }, {
+    id: "folderCCCCCC",
+    type: "folder",
+    title: "C",
+    children: ["folderDDDDDD"],
+  }, {
+    id: "folderDDDDDD",
+    type: "folder",
+    title: "D",
+    children: ["folderAAAAAA"],
+  }]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual((await buf.fetchUnmergedGuids()).sort(), ["folderAAAAAA",
+    "folderBBBBBB", "folderCCCCCC", "folderDDDDDD"],
+    "Should leave items in circular subtree unmerged");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, { updated: [], deleted: [] },
+    "Should not mark any local items for upload");
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should not be confused into creating a cycle");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_deduping.js b/toolkit/components/places/tests/sync/test_bookmark_deduping.js
new file mode 100644
index 000000000000..53ad39ceb476
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_deduping.js
@@ -0,0 +1,444 @@
+add_task(async function test_duping() {
+  let buf = await openMirror("duping");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      // Shouldn't dupe to `folderA11111` because its sync status is "NORMAL".
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        // Shouldn't dupe to `bookmarkG111`.
+        guid: "bookmarkGGGG",
+        url: "http://example.com/g",
+        title: "G",
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkGGGG"],
+  }, {
+    id: "bookmarkGGGG",
+    type: "bookmark",
+    title: "G",
+    url: "http://example.com/g",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Insert local dupes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      // Should dupe to `folderB11111`.
+      guid: "folderBBBBBB",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "B",
+      children: [{
+        // Should dupe to `bookmarkC222`.
+        guid: "bookmarkC111",
+        url: "http://example.com/c",
+        title: "C",
+      }, {
+        // Should dupe to `separatorF11` because the positions are the same.
+        guid: "separatorFFF",
+        type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+      }],
+    }, {
+      // Shouldn't dupe to `separatorE11`, because the positions are different.
+      guid: "separatorEEE",
+      type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+    }, {
+      // Shouldn't dupe to `bookmarkC222` because the parents are different.
+      guid: "bookmarkCCCC",
+      url: "http://example.com/c",
+      title: "C",
+    }, {
+      // Should dupe to `queryD111111`.
+      guid: "queryDDDDDDD",
+      url: "place:sort=8&maxResults=10",
+      title: "Most Visited",
+      annos: [{
+        name: PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO,
+        value: "MostVisited",
+      }],
+    }],
+  });
+  // Not a candidate for `bookmarkH111` because we didn't dupe `folderAAAAAA`.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: "folderAAAAAA",
+    guid: "bookmarkHHHH",
+    url: "http://example.com/h",
+    title: "H",
+  });
+
+  info("Make remote changes");
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA", "folderB11111", "folderA11111",
+               "separatorE11", "queryD111111"],
+  }, {
+    id: "folderB11111",
+    type: "folder",
+    title: "B",
+    children: ["bookmarkC222", "separatorF11"],
+  }, {
+    id: "bookmarkC222",
+    type: "bookmark",
+    bmkUri: "http://example.com/c",
+    title: "C",
+  }, {
+    id: "separatorF11",
+    type: "separator",
+  }, {
+    id: "folderA11111",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkG111"],
+  }, {
+    id: "bookmarkG111",
+    type: "bookmark",
+    bmkUri: "http://example.com/g",
+    title: "G",
+  }, {
+    id: "separatorE11",
+    type: "separator",
+  }, {
+    id: "queryD111111",
+    type: "query",
+    bmkUri: "place:maxResults=10&sort=8",
+    title: "Most Visited",
+    queryId: "MostVisited",
+  }]));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkCCCC", "bookmarkHHHH", "folderAAAAAA", "menu",
+              "separatorEEE"],
+    deleted: [],
+  }, "Should not upload deduped local records");
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "A",
+        children: [{
+          guid: "bookmarkGGGG",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "G",
+          url: "http://example.com/g",
+        }, {
+          guid: "bookmarkHHHH",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "H",
+          url: "http://example.com/h",
+        }],
+      }, {
+        guid: "folderB11111",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 1,
+        title: "B",
+        children: [{
+          guid: "bookmarkC222",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "C",
+          url: "http://example.com/c",
+        }, {
+          guid: "separatorF11",
+          type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+          index: 1,
+          title: "",
+        }],
+      }, {
+        guid: "folderA11111",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 2,
+        title: "A",
+        children: [{
+          guid: "bookmarkG111",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "G",
+          url: "http://example.com/g",
+        }],
+      }, {
+        guid: "separatorE11",
+        type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+        index: 3,
+        title: "",
+      }, {
+        guid: "queryD111111",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 4,
+        title: "Most Visited",
+        url: "place:maxResults=10&sort=8",
+        annos: [{
+          name: PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO,
+          flags: 0,
+          expires: PlacesUtils.annotations.EXPIRE_NEVER,
+          value: "MostVisited",
+        }],
+      }, {
+        guid: "separatorEEE",
+        type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+        index: 5,
+        title: "",
+      }, {
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 6,
+        title: "C",
+        url: "http://example.com/c",
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should dedupe matching NEW bookmarks");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_applying_two_empty_folders_doesnt_smush() {
+  let buf = await openMirror("applying_two_empty_folders_doesnt_smush");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await buf.store(shuffle([{
+    id: "mobile",
+    type: "folder",
+    title: "mobile",
+    children: ["emptyempty01", "emptyempty02"],
+  }, {
+    id: "emptyempty01",
+    type: "folder",
+    title: "Empty",
+  }, {
+    id: "emptyempty02",
+    type: "folder",
+    title: "Empty",
+  }]));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: [],
+    deleted: [],
+  }, "Should not upload records for remote-only value changes");
+
+  await assertLocalTree(PlacesUtils.bookmarks.mobileGuid, {
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 4,
+    title: "mobile",
+    children: [{
+      guid: "emptyempty01",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Empty",
+    }, {
+      guid: "emptyempty02",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Empty",
+    }],
+  }, "Should not smush 1 and 2");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_applying_two_empty_folders_matches_only_one() {
+  let buf = await openMirror("applying_two_empty_folders_doesnt_smush");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    children: [{
+      guid: "emptyempty02",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "Empty",
+    }, {
+      guid: "emptyemptyL0",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "Empty",
+    }],
+  });
+
+  info("Make remote changes");
+  await buf.store(shuffle([{
+    id: "mobile",
+    type: "folder",
+    title: "mobile",
+    children: ["emptyempty01", "emptyempty02", "emptyempty03"],
+  }, {
+    id: "emptyempty01",
+    type: "folder",
+    title: "Empty",
+  }, {
+    id: "emptyempty02",
+    type: "folder",
+    title: "Empty",
+  }, {
+    id: "emptyempty03",
+    type: "folder",
+    title: "Empty",
+  }]));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: [],
+    deleted: [],
+  }, "Should not upload records after applying empty folders");
+
+  await assertLocalTree(PlacesUtils.bookmarks.mobileGuid, {
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 4,
+    title: "mobile",
+    children: [{
+      guid: "emptyempty01",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Empty",
+    }, {
+      guid: "emptyempty02",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Empty",
+    }, {
+      guid: "emptyempty03",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 2,
+      title: "Empty",
+    }],
+  }, "Should apply 1 and dedupe L0 to 3");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+// Bug 747699.
+add_task(async function test_duping_mobile_bookmarks() {
+  let buf = await openMirror("duping_mobile_bookmarks");
+
+  info("Set up empty mirror with localized mobile root title");
+  let mobileInfo = await PlacesUtils.bookmarks.fetch(
+    PlacesUtils.bookmarks.mobileGuid);
+  await PlacesUtils.bookmarks.update({
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    title: "Favoritos do celular",
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkAAA1",
+    parentGuid: PlacesUtils.bookmarks.mobileGuid,
+    title: "A",
+    url: "http://example.com/a",
+  });
+
+  info("Make remote changes");
+  await buf.store(shuffle([{
+    id: "mobile",
+    type: "folder",
+    title: "Mobile Bookmarks",
+    children: ["bookmarkAAAA"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+  }]));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: [],
+    deleted: [],
+  }, "Should not upload records after applying deduped mobile bookmark");
+
+  await assertLocalTree(PlacesUtils.bookmarks.mobileGuid, {
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 4,
+    title: "Mobile Bookmarks",
+    children: [{
+      guid: "bookmarkAAAA",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 0,
+      title: "A",
+      url: "http://example.com/a",
+    }],
+  }, "Should dedupe A1 to A with different parent title");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  // Restore the original mobile root title.
+  await PlacesUtils.bookmarks.update({
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    title: mobileInfo.title,
+  });
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_deletion.js b/toolkit/components/places/tests/sync/test_bookmark_deletion.js
new file mode 100644
index 000000000000..1759b3a1cfc7
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_deletion.js
@@ -0,0 +1,587 @@
+add_task(async function test_complex_orphaning() {
+  let buf = await openMirror("complex_orphaning");
+
+  // On iOS, the mirror exists as a separate table. On Desktop, we have a
+  // shadow mirror of synced local bookmarks without new changes.
+  info("Set up mirror: ((Toolbar > A > B) (Menu > G > C > D))");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "toolbar",
+    type: "folder",
+    title: "Bookmarks Toolbar",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["folderBBBBBB"],
+  }, {
+    id: "folderBBBBBB",
+    type: "folder",
+    title: "B",
+  }]), { needsMerge: false });
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "folderGGGGGG",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "G",
+      children: [{
+        guid: "folderCCCCCC",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "C",
+        children: [{
+          guid: "folderDDDDDD",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          title: "D",
+        }],
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderGGGGGG"],
+  }, {
+    id: "folderGGGGGG",
+    type: "folder",
+    title: "G",
+    children: ["folderCCCCCC"],
+  }, {
+    id: "folderCCCCCC",
+    type: "folder",
+    title: "C",
+    children: ["folderDDDDDD"],
+  }, {
+    id: "folderDDDDDD",
+    type: "folder",
+    title: "D",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: delete D, add B > E");
+  await PlacesUtils.bookmarks.remove("folderDDDDDD");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkEEEE",
+    parentGuid: "folderBBBBBB",
+    title: "E",
+    url: "http://example.com/e",
+  });
+
+  info("Make remote changes: delete B, add D > F");
+  await buf.store(shuffle([{
+    id: "folderBBBBBB",
+    deleted: true,
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+  }, {
+    id: "folderDDDDDD",
+    type: "folder",
+    children: ["bookmarkFFFF"],
+  }, {
+    id: "bookmarkFFFF",
+    type: "bookmark",
+    title: "F",
+    bmkUri: "http://example.com/f",
+  }]));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkEEEE", "bookmarkFFFF", "folderAAAAAA", "folderCCCCCC"],
+    deleted: ["folderDDDDDD"],
+  }, "Should upload new records for (A > E), (C > F); tombstone for D");
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "folderGGGGGG",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "G",
+        children: [{
+          guid: "folderCCCCCC",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "C",
+          children: [{
+            // D was deleted, so F moved to C, the closest surviving parent.
+            guid: "bookmarkFFFF",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "F",
+            url: "http://example.com/f",
+          }],
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+      children: [{
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "A",
+        children: [{
+          // B was deleted, so E moved to A.
+          guid: "bookmarkEEEE",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "E",
+          url: "http://example.com/e",
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should move orphans to closest surviving parent");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_locally_modified_remotely_deleted() {
+  let buf = await openMirror("locally_modified_remotely_deleted");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "bookmarkAAAA",
+      title: "A",
+      url: "http://example.com/a",
+    }, {
+      guid: "folderBBBBBB",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "B",
+      children: [{
+        guid: "bookmarkCCCC",
+        title: "C",
+        url: "http://example.com/c",
+      }, {
+        guid: "folderDDDDDD",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "D",
+        children: [{
+          guid: "bookmarkEEEE",
+          title: "E",
+          url: "http://example.com/e",
+        }],
+      }],
+    }],
+  });
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkAAAA", "folderBBBBBB"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+  }, {
+    id: "folderBBBBBB",
+    type: "folder",
+    title: "B",
+    children: ["bookmarkCCCC", "folderDDDDDD"],
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }, {
+    id: "folderDDDDDD",
+    type: "folder",
+    title: "D",
+    children: ["bookmarkEEEE"],
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E",
+    bmkUri: "http://example.com/e",
+  }], { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: change A; B > ((D > F) G)");
+  await PlacesUtils.bookmarks.update({
+    guid: "bookmarkAAAA",
+    title: "A (local)",
+    url: "http://example.com/a-local",
+  });
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkFFFF",
+    parentGuid: "folderDDDDDD",
+    title: "F (local)",
+    url: "http://example.com/f-local",
+  });
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkGGGG",
+    parentGuid: "folderBBBBBB",
+    title: "G (local)",
+    url: "http://example.com/g-local",
+  });
+
+  info("Make remote changes: delete A, B");
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: [],
+  }, {
+    id: "bookmarkAAAA",
+    deleted: true,
+  }, {
+    id: "folderBBBBBB",
+    deleted: true,
+  }, {
+    id: "bookmarkCCCC",
+    deleted: true,
+  }, {
+    id: "folderDDDDDD",
+    deleted: true,
+  }, {
+    id: "bookmarkEEEE",
+    deleted: true,
+  }]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkAAAA", "bookmarkFFFF", "bookmarkGGGG", "menu"],
+    deleted: [],
+  }, "Should upload A, relocated local orphans, and menu");
+
+  await assertLocalTree(PlacesUtils.bookmarks.menuGuid, {
+    guid: PlacesUtils.bookmarks.menuGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "Bookmarks Menu",
+    children: [{
+      guid: "bookmarkAAAA",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 0,
+      title: "A (local)",
+      url: "http://example.com/a-local",
+    }, {
+      guid: "bookmarkFFFF",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 1,
+      title: "F (local)",
+      url: "http://example.com/f-local",
+    }, {
+      guid: "bookmarkGGGG",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 2,
+      title: "G (local)",
+      url: "http://example.com/g-local",
+    }],
+  }, "Should restore A and relocate (F G) to menu");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_locally_deleted_remotely_modified() {
+  let buf = await openMirror("locally_deleted_remotely_modified");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "bookmarkAAAA",
+      title: "A",
+      url: "http://example.com/a",
+    }, {
+      guid: "folderBBBBBB",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "B",
+      children: [{
+        guid: "bookmarkCCCC",
+        title: "C",
+        url: "http://example.com/c",
+      }, {
+        guid: "folderDDDDDD",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "D",
+        children: [{
+          guid: "bookmarkEEEE",
+          title: "E",
+          url: "http://example.com/e",
+        }],
+      }],
+    }],
+  });
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkAAAA", "folderBBBBBB"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+  }, {
+    id: "folderBBBBBB",
+    type: "folder",
+    title: "B",
+    children: ["bookmarkCCCC", "folderDDDDDD"],
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }, {
+    id: "folderDDDDDD",
+    type: "folder",
+    title: "D",
+    children: ["bookmarkEEEE"],
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E",
+    bmkUri: "http://example.com/e",
+  }], { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: delete A, B");
+  await PlacesUtils.bookmarks.remove("bookmarkAAAA");
+  await PlacesUtils.bookmarks.remove("folderBBBBBB");
+
+  info("Make remote changes: change A; B > ((D > F) G)");
+  await buf.store([{
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A (remote)",
+    bmkUri: "http://example.com/a-remote",
+  }, {
+    id: "folderBBBBBB",
+    type: "folder",
+    title: "B (remote)",
+    children: ["bookmarkCCCC", "folderDDDDDD", "bookmarkGGGG"],
+  }, {
+    id: "folderDDDDDD",
+    type: "folder",
+    title: "D",
+    children: ["bookmarkEEEE", "bookmarkFFFF"],
+  }, {
+    id: "bookmarkFFFF",
+    type: "bookmark",
+    title: "F (remote)",
+    bmkUri: "http://example.com/f-remote",
+  }, {
+    id: "bookmarkGGGG",
+    type: "bookmark",
+    title: "G (remote)",
+    bmkUri: "http://example.com/g-remote",
+  }]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkFFFF", "bookmarkGGGG", "menu"],
+    deleted: ["bookmarkCCCC", "bookmarkEEEE", "folderBBBBBB", "folderDDDDDD"],
+  }, "Should upload relocated remote orphans and menu");
+
+  await assertLocalTree(PlacesUtils.bookmarks.menuGuid, {
+    guid: PlacesUtils.bookmarks.menuGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "Bookmarks Menu",
+    children: [{
+      guid: "bookmarkAAAA",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 0,
+      title: "A (remote)",
+      url: "http://example.com/a-remote",
+    }, {
+      guid: "bookmarkFFFF",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 1,
+      title: "F (remote)",
+      url: "http://example.com/f-remote",
+    }, {
+      guid: "bookmarkGGGG",
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      index: 2,
+      title: "G (remote)",
+      url: "http://example.com/g-remote",
+    }],
+  }, "Should restore A and relocate (F G) to menu");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_move_to_new_then_delete() {
+  let buf = await openMirror("move_to_new_then_delete");
+
+  info("Set up mirror: A > B > (C D)");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+        children: [{
+          guid: "bookmarkCCCC",
+          url: "http://example.com/c",
+          title: "C",
+        }, {
+          guid: "bookmarkDDDD",
+          url: "http://example.com/d",
+          title: "D",
+        }],
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["folderBBBBBB"],
+  }, {
+    id: "folderBBBBBB",
+    type: "folder",
+    title: "B",
+    children: ["bookmarkCCCC", "bookmarkDDDD"],
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }, {
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D",
+    bmkUri: "http://example.com/d",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: E > A, delete E");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    guid: "folderEEEEEE",
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    title: "E",
+  });
+  await PlacesUtils.bookmarks.update({
+    guid: "folderAAAAAA",
+    parentGuid: "folderEEEEEE",
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+  });
+  // E isn't synced, so we shouldn't upload a tombstone.
+  await PlacesUtils.bookmarks.remove("folderEEEEEE");
+
+  info("Make remote changes");
+  await buf.store([{
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C (remote)",
+    bmkUri: "http://example.com/c-remote",
+  }]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkCCCC", "menu", "toolbar"],
+    deleted: ["bookmarkDDDD", "folderAAAAAA", "folderBBBBBB"],
+  }, "Should upload records for Menu > C, Toolbar");
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "C (remote)",
+        url: "http://example.com/c-remote",
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should move C to closest surviving parent");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_kinds.js b/toolkit/components/places/tests/sync/test_bookmark_kinds.js
new file mode 100644
index 000000000000..9ee2840090b4
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_kinds.js
@@ -0,0 +1,287 @@
+add_task(async function test_livemarks() {
+  let { site, stopServer } = makeLivemarkServer();
+
+  try {
+    let buf = await openMirror("livemarks");
+
+    info("Set up mirror");
+    await PlacesUtils.bookmarks.insertTree({
+      guid: PlacesUtils.bookmarks.menuGuid,
+      children: [{
+        guid: "livemarkAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        annos: [{
+          name: PlacesUtils.LMANNO_FEEDURI,
+          value: site + "/feed/a",
+        }],
+      }],
+    });
+    await buf.store(shuffle([{
+      id: "menu",
+      type: "folder",
+      title: "Bookmarks Menu",
+      children: ["livemarkAAAA"],
+    }, {
+      id: "livemarkAAAA",
+      type: "livemark",
+      title: "A",
+      feedUri: site + "/feed/a",
+    }]), { needsMerge: false });
+    await PlacesTestUtils.markBookmarksAsSynced();
+
+    info("Make local changes");
+    await PlacesUtils.livemarks.addLivemark({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      guid: "livemarkBBBB",
+      title: "B",
+      feedURI: Services.io.newURI(site + "/feed/b-local"),
+      siteURI: Services.io.newURI(site + "/site/b-local"),
+    });
+    let livemarkD = await PlacesUtils.livemarks.addLivemark({
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      guid: "livemarkDDDD",
+      title: "D",
+      feedURI: Services.io.newURI(site + "/feed/d"),
+      siteURI: Services.io.newURI(site + "/site/d"),
+    });
+
+    info("Make remote changes");
+    await buf.store(shuffle([{
+      id: "livemarkAAAA",
+      type: "livemark",
+      title: "A (remote)",
+      feedUri: site + "/feed/a-remote",
+    }, {
+      id: "toolbar",
+      type: "folder",
+      title: "Bookmarks Toolbar",
+      children: ["livemarkCCCC", "livemarkB111"],
+    }, {
+      id: "unfiled",
+      type: "folder",
+      title: "Other Bookmarks",
+      children: ["livemarkEEEE"],
+    }, {
+      id: "livemarkCCCC",
+      type: "livemark",
+      title: "C (remote)",
+      feedUri: site + "/feed/c-remote",
+    }, {
+      id: "livemarkB111",
+      type: "livemark",
+      title: "B",
+      feedUri: site + "/feed/b-remote",
+    }, {
+      id: "livemarkEEEE",
+      type: "livemark",
+      title: "E",
+      feedUri: site + "/feed/e",
+      siteUri: site + "/site/e",
+    }]));
+
+    info("Apply remote");
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+    let menuInfo = await PlacesUtils.bookmarks.fetch(
+      PlacesUtils.bookmarks.menuGuid);
+    deepEqual(changesToUpload, {
+      livemarkDDDD: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "livemarkDDDD",
+          type: "livemark",
+          parentid: "menu",
+          hasDupe: false,
+          parentName: "Bookmarks Menu",
+          dateAdded: PlacesUtils.toDate(livemarkD.dateAdded).getTime(),
+          title: "D",
+          feedUri: site + "/feed/d",
+          siteUri: site + "/site/d",
+        },
+      },
+      menu: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: false,
+          parentName: "",
+          dateAdded: menuInfo.dateAdded.getTime(),
+          title: menuInfo.title,
+          children: ["livemarkAAAA", "livemarkDDDD"],
+        },
+      },
+    }, "Should upload new local livemark A");
+
+    await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [{
+        guid: PlacesUtils.bookmarks.menuGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "Bookmarks Menu",
+        children: [{
+          guid: "livemarkAAAA",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "A (remote)",
+          annos: [{
+            name: PlacesUtils.LMANNO_FEEDURI,
+            flags: 0,
+            expires: PlacesUtils.annotations.EXPIRE_NEVER,
+            value: site + "/feed/a-remote",
+          }],
+        }, {
+          guid: "livemarkDDDD",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "D",
+          annos: [{
+            name: PlacesUtils.LMANNO_FEEDURI,
+            flags: 0,
+            expires: PlacesUtils.annotations.EXPIRE_NEVER,
+            value: site + "/feed/d",
+          }, {
+            name: PlacesUtils.LMANNO_SITEURI,
+            flags: 0,
+            expires: PlacesUtils.annotations.EXPIRE_NEVER,
+            value: site + "/site/d",
+          }],
+        }],
+      }, {
+        guid: PlacesUtils.bookmarks.toolbarGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 1,
+        title: "Bookmarks Toolbar",
+        children: [{
+          guid: "livemarkCCCC",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "C (remote)",
+          annos: [{
+            name: PlacesUtils.LMANNO_FEEDURI,
+            flags: 0,
+            expires: PlacesUtils.annotations.EXPIRE_NEVER,
+            value: site + "/feed/c-remote",
+          }],
+        }, {
+          guid: "livemarkB111",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "B",
+          annos: [{
+            name: PlacesUtils.LMANNO_FEEDURI,
+            flags: 0,
+            expires: PlacesUtils.annotations.EXPIRE_NEVER,
+            value: site + "/feed/b-remote",
+          }],
+        }],
+      }, {
+        guid: PlacesUtils.bookmarks.unfiledGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 3,
+        title: "Other Bookmarks",
+        children: [{
+          guid: "livemarkEEEE",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "E",
+          annos: [{
+            name: PlacesUtils.LMANNO_FEEDURI,
+            flags: 0,
+            expires: PlacesUtils.annotations.EXPIRE_NEVER,
+            value: site + "/feed/e",
+          }, {
+            name: PlacesUtils.LMANNO_SITEURI,
+            flags: 0,
+            expires: PlacesUtils.annotations.EXPIRE_NEVER,
+            value: site + "/site/e",
+          }],
+        }],
+      }, {
+        guid: PlacesUtils.bookmarks.mobileGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 4,
+        title: "mobile",
+      }],
+    }, "Should apply and dedupe livemarks");
+
+    let cLivemark = await PlacesUtils.livemarks.getLivemark({
+      guid: "livemarkCCCC",
+    });
+    equal(cLivemark.title, "C (remote)", "Should set livemark C title");
+    ok(cLivemark.feedURI.equals(Services.io.newURI(site + "/feed/c-remote")),
+      "Should set livemark C feed URL");
+
+    let bLivemark = await PlacesUtils.livemarks.getLivemark({
+      guid: "livemarkB111",
+    });
+    ok(bLivemark.feedURI.equals(Services.io.newURI(site + "/feed/b-remote")),
+      "Should set deduped livemark B feed URL");
+    strictEqual(bLivemark.siteURI, null,
+      "Should remove deduped livemark B site URL");
+
+    await buf.finalize();
+  } finally {
+    await stopServer();
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+// Bug 632287.
+add_task(async function test_mismatched_types() {
+  let buf = await openMirror("mismatched_types");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "l1nZZXfB8nC7",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "Innerst i Sneglehode",
+    }],
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await buf.store([{
+    "id": "l1nZZXfB8nC7",
+    "type": "livemark",
+    "siteUri": "http://sneglehode.wordpress.com/",
+    "feedUri": "http://sneglehode.wordpress.com/feed/",
+    "parentName": "Bookmarks Toolbar",
+    "title": "Innerst i Sneglehode",
+    "description": null,
+    "children":
+      ["HCRq40Rnxhrd", "YeyWCV1RVsYw", "GCceVZMhvMbP", "sYi2hevdArlF",
+       "vjbZlPlSyGY8", "UtjUhVyrpeG6", "rVq8WMG2wfZI", "Lx0tcy43ZKhZ",
+       "oT74WwV8_j4P", "IztsItWVSo3-"],
+    "parentid": "toolbar"
+  }]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: [],
+    deleted: [],
+  }, "Should not reupload merged livemark");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_structure_changes.js b/toolkit/components/places/tests/sync/test_bookmark_structure_changes.js
new file mode 100644
index 000000000000..2fd117b317af
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_structure_changes.js
@@ -0,0 +1,742 @@
+add_task(async function test_value_structure_conflict() {
+  let buf = await openMirror("value_structure_conflict");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      }, {
+        guid: "bookmarkCCCC",
+        url: "http://example.com/c",
+        title: "C",
+      }],
+    }, {
+      guid: "folderDDDDDD",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "D",
+      children: [{
+        guid: "bookmarkEEEE",
+        url: "http://example.com/e",
+        title: "E",
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA", "folderDDDDDD"],
+    modified: Date.now() / 1000 - 60,
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkBBBB", "bookmarkCCCC"],
+    modified: Date.now() / 1000 - 60,
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+    modified: Date.now() / 1000 - 60,
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+    modified: Date.now() / 1000 - 60,
+  }, {
+    id: "folderDDDDDD",
+    type: "folder",
+    title: "D",
+    children: ["bookmarkEEEE"],
+    modified: Date.now() / 1000 - 60,
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E",
+    bmkUri: "http://example.com/e",
+    modified: Date.now() / 1000 - 60,
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local value change");
+  await PlacesUtils.bookmarks.update({
+    guid: "folderAAAAAA",
+    title: "A (local)",
+  });
+
+  info("Make local structure change");
+  await PlacesUtils.bookmarks.update({
+    guid: "bookmarkBBBB",
+    parentGuid: "folderDDDDDD",
+    index: 0,
+  });
+
+  info("Make remote value change");
+  await buf.store([{
+    id: "folderDDDDDD",
+    type: "folder",
+    title: "D (remote)",
+    children: ["bookmarkEEEE"],
+    modified: Date.now() / 1000 + 60,
+  }]);
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply({
+    remoteTimeSeconds: Date.now() / 1000,
+  });
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkBBBB", "folderAAAAAA", "folderDDDDDD"],
+    deleted: [],
+  }, "Should upload records for merged and new local items");
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds(["folderAAAAAA",
+    "bookmarkEEEE", "bookmarkBBBB", "folderDDDDDD"]);
+  observer.check([{
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bookmarkEEEE"),
+              oldParentId: localItemIds.get("folderDDDDDD"),
+              oldIndex: 1, newParentId: localItemIds.get("folderDDDDDD"),
+              newIndex: 0, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bookmarkEEEE",
+              oldParentGuid: "folderDDDDDD",
+              newParentGuid: "folderDDDDDD",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bookmarkBBBB"),
+              oldParentId: localItemIds.get("folderDDDDDD"),
+              oldIndex: 0, newParentId: localItemIds.get("folderDDDDDD"),
+              newIndex: 1, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bookmarkBBBB",
+              oldParentGuid: "folderDDDDDD",
+              newParentGuid: "folderDDDDDD",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("folderDDDDDD"), property: "title",
+              isAnnoProperty: false, newValue: "D (remote)",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "folderDDDDDD",
+              parentGuid: PlacesUtils.bookmarks.menuGuid, oldValue: "D",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }]);
+
+  await assertLocalTree(PlacesUtils.bookmarks.menuGuid, {
+    guid: PlacesUtils.bookmarks.menuGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "Bookmarks Menu",
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "A (local)",
+      children: [{
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "C",
+        url: "http://example.com/c",
+      }],
+    }, {
+      guid: "folderDDDDDD",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "D (remote)",
+      children: [{
+        guid: "bookmarkEEEE",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "E",
+        url: "http://example.com/e",
+      }, {
+        guid: "bookmarkBBBB",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 1,
+        title: "B",
+        url: "http://example.com/b",
+      }],
+    }],
+  }, "Should reconcile structure and value changes");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_move() {
+  let buf = await openMirror("move");
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "devFolder___",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "Dev",
+      children: [{
+        guid: "mdnBmk______",
+        title: "MDN",
+        url: "https://developer.mozilla.org",
+      }, {
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        guid: "mozFolder___",
+        title: "Mozilla",
+        children: [{
+          guid: "fxBmk_______",
+          title: "Get Firefox!",
+          url: "http://getfirefox.com/",
+        }, {
+          guid: "nightlyBmk__",
+          title: "Nightly",
+          url: "https://nightly.mozilla.org",
+        }],
+      }, {
+        guid: "wmBmk_______",
+        title: "Webmaker",
+        url: "https://webmaker.org",
+      }],
+    }, {
+      guid: "bzBmk_______",
+      title: "Bugzilla",
+      url: "https://bugzilla.mozilla.org",
+    }]
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  await buf.store(shuffle([{
+    id: "unfiled",
+    type: "folder",
+    title: "Other Bookmarks",
+    children: ["mozFolder___"],
+  }, {
+    id: "toolbar",
+    type: "folder",
+    title: "Bookmarks Toolbar",
+    children: ["devFolder___"],
+  }, {
+    id: "devFolder___",
+    // Moving to toolbar.
+    type: "folder",
+    title: "Dev",
+    children: ["bzBmk_______", "wmBmk_______"],
+  }, {
+    // Moving to "Mozilla".
+    id: "mdnBmk______",
+    type: "bookmark",
+    title: "MDN",
+    bmkUri: "https://developer.mozilla.org",
+  }, {
+    // Rearranging children and moving to unfiled.
+    id: "mozFolder___",
+    type: "folder",
+    title: "Mozilla",
+    children: ["nightlyBmk__", "mdnBmk______", "fxBmk_______"],
+  }, {
+    id: "fxBmk_______",
+    type: "bookmark",
+    title: "Get Firefox!",
+    bmkUri: "http://getfirefox.com/",
+  }, {
+    id: "nightlyBmk__",
+    type: "bookmark",
+    title: "Nightly",
+    bmkUri: "https://nightly.mozilla.org",
+  }, {
+    id: "wmBmk_______",
+    type: "bookmark",
+    title: "Webmaker",
+    bmkUri: "https://webmaker.org",
+  }, {
+    id: "bzBmk_______",
+    type: "bookmark",
+    title: "Bugzilla",
+    bmkUri: "https://bugzilla.mozilla.org",
+  }]));
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: [],
+    deleted: [],
+  }, "Should not upload records for remotely moved items");
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds(["devFolder___",
+    "mozFolder___", "bzBmk_______", "wmBmk_______", "nightlyBmk__",
+    "mdnBmk______", "fxBmk_______"]);
+  observer.check([{
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("devFolder___"),
+              oldParentId: PlacesUtils.bookmarksMenuFolderId,
+              oldIndex: 0, newParentId: PlacesUtils.toolbarFolderId,
+              newIndex: 0, type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              guid: "devFolder___",
+              oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+              newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("mozFolder___"),
+              oldParentId: localItemIds.get("devFolder___"),
+              oldIndex: 1, newParentId: PlacesUtils.unfiledBookmarksFolderId,
+              newIndex: 0, type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              guid: "mozFolder___",
+              oldParentGuid: "devFolder___",
+              newParentGuid: PlacesUtils.bookmarks.unfiledGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bzBmk_______"),
+              oldParentId: PlacesUtils.bookmarksMenuFolderId,
+              oldIndex: 1, newParentId: localItemIds.get("devFolder___"),
+              newIndex: 0, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bzBmk_______",
+              oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+              newParentGuid: "devFolder___",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("wmBmk_______"),
+              oldParentId: localItemIds.get("devFolder___"),
+              oldIndex: 2, newParentId: localItemIds.get("devFolder___"),
+              newIndex: 1, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "wmBmk_______",
+              oldParentGuid: "devFolder___",
+              newParentGuid: "devFolder___",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("nightlyBmk__"),
+              oldParentId: localItemIds.get("mozFolder___"),
+              oldIndex: 1, newParentId: localItemIds.get("mozFolder___"),
+              newIndex: 0, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "nightlyBmk__",
+              oldParentGuid: "mozFolder___",
+              newParentGuid: "mozFolder___",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("mdnBmk______"),
+              oldParentId: localItemIds.get("devFolder___"),
+              oldIndex: 0, newParentId: localItemIds.get("mozFolder___"),
+              newIndex: 1, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "mdnBmk______",
+              oldParentGuid: "devFolder___",
+              newParentGuid: "mozFolder___",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("fxBmk_______"),
+              oldParentId: localItemIds.get("mozFolder___"),
+              oldIndex: 0, newParentId: localItemIds.get("mozFolder___"),
+              newIndex: 2, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "fxBmk_______",
+              oldParentGuid: "mozFolder___",
+              newParentGuid: "mozFolder___",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }]);
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+      children: [{
+        guid: "devFolder___",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "Dev",
+        children: [{
+          guid: "bzBmk_______",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "Bugzilla",
+          url: "https://bugzilla.mozilla.org/",
+        }, {
+          guid: "wmBmk_______",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "Webmaker",
+          url: "https://webmaker.org/",
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+      children: [{
+        guid: "mozFolder___",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "Mozilla",
+        children: [{
+          guid: "nightlyBmk__",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "Nightly",
+          url: "https://nightly.mozilla.org/",
+        }, {
+          guid: "mdnBmk______",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "MDN",
+          url: "https://developer.mozilla.org/",
+        }, {
+          guid: "fxBmk_______",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "Get Firefox!",
+          url: "http://getfirefox.com/",
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should move and reorder bookmarks to match remote");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_move_into_parent_sibling() {
+  // This test moves a bookmark that exists locally into a new folder that only
+  // exists remotely, and is a later sibling of the local parent. This ensures
+  // we set up the local structure before applying structure changes.
+  let buf = await openMirror("move_into_parent_sibling");
+
+  info("Set up mirror: Menu > A > B");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkBBBB"],
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes: Menu > (A (B > C))");
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA", "folderCCCCCC"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+  }, {
+    id: "folderCCCCCC",
+    type: "folder",
+    title: "C",
+    children: ["bookmarkBBBB"],
+  }]);
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: [],
+    deleted: [],
+  }, "Should not upload records for remote-only structure changes");
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds(["folderCCCCCC",
+    "bookmarkBBBB", "folderAAAAAA"]);
+  observer.check([{
+    name: "onItemAdded",
+    params: { itemId: localItemIds.get("folderCCCCCC"),
+              parentId: PlacesUtils.bookmarksMenuFolderId, index: 1,
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              urlHref: null, title: "C",
+              guid: "folderCCCCCC",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bookmarkBBBB"),
+              oldParentId: localItemIds.get("folderAAAAAA"),
+              oldIndex: 0, newParentId: localItemIds.get("folderCCCCCC"),
+              newIndex: 0, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bookmarkBBBB",
+              oldParentGuid: "folderAAAAAA",
+              newParentGuid: "folderCCCCCC",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }]);
+
+  await assertLocalTree(PlacesUtils.bookmarks.menuGuid, {
+    guid: PlacesUtils.bookmarks.menuGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "Bookmarks Menu",
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "A",
+    }, {
+      guid: "folderCCCCCC",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "C",
+      children: [{
+        guid: "bookmarkBBBB",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "B",
+        url: "http://example.com/b",
+      }],
+    }],
+  }, "Should set up local structure correctly");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_complex_move_with_additions() {
+  let buf = await openMirror("complex_move_with_additions");
+
+  info("Set up mirror: Menu > A > (B C)");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      }, {
+        guid: "bookmarkCCCC",
+        url: "http://example.com/c",
+        title: "C",
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkBBBB", "bookmarkCCCC"],
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local change: Menu > A > (B C D)");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkDDDD",
+    parentGuid: "folderAAAAAA",
+    title: "D (local)",
+    url: "http://example.com/d-local",
+  });
+
+  info("Make remote change: ((Menu > C) (Toolbar > A > (B E)))");
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkCCCC"],
+  }, {
+    id: "toolbar",
+    type: "folder",
+    title: "Bookmarks Toolbar",
+    children: ["folderAAAAAA"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkBBBB", "bookmarkEEEE"],
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E",
+    bmkUri: "http://example.com/e",
+  }]));
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkDDDD", "folderAAAAAA"],
+    deleted: [],
+  }, "Should upload new records for (A D)");
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds(["bookmarkEEEE",
+    "folderAAAAAA", "bookmarkCCCC"]);
+  observer.check([{
+    name: "onItemAdded",
+    params: { itemId: localItemIds.get("bookmarkEEEE"),
+              parentId: localItemIds.get("folderAAAAAA"), index: 1,
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              urlHref: "http://example.com/e", title: "E",
+              guid: "bookmarkEEEE",
+              parentGuid: "folderAAAAAA",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bookmarkCCCC"),
+              oldParentId: localItemIds.get("folderAAAAAA"),
+              oldIndex: 1, newParentId: PlacesUtils.bookmarksMenuFolderId,
+              newIndex: 0, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bookmarkCCCC",
+              oldParentGuid: "folderAAAAAA",
+              newParentGuid: PlacesUtils.bookmarks.menuGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("folderAAAAAA"),
+              oldParentId: PlacesUtils.bookmarksMenuFolderId,
+              oldIndex: 0, newParentId: PlacesUtils.toolbarFolderId,
+              newIndex: 0, type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              guid: "folderAAAAAA",
+              oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+              newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }]);
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "bookmarkCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "C",
+        url: "http://example.com/c",
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+      children: [{
+        // We can guarantee child order (B E D), since we always walk remote
+        // children first, and the remote folder A record is newer than the
+        // local folder. If the local folder were newer, the order would be
+        // (D B E).
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "A",
+        children: [{
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "B",
+          url: "http://example.com/b",
+        }, {
+          guid: "bookmarkEEEE",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "E",
+          url: "http://example.com/e",
+        }, {
+          guid: "bookmarkDDDD",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "D (local)",
+          url: "http://example.com/d-local",
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should take remote order and preserve local children");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_value_changes.js b/toolkit/components/places/tests/sync/test_bookmark_value_changes.js
new file mode 100644
index 000000000000..fd0008a7ec6b
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_value_changes.js
@@ -0,0 +1,1202 @@
+add_task(async function test_value_combo() {
+  let buf = await openMirror("value_combo");
+
+  info("Set up mirror with existing bookmark to update");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "mozBmk______",
+      url: "https://mozilla.org",
+      title: "Mozilla",
+      tags: ["moz", "dot", "org"],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["mozBmk______"],
+  }, {
+    id: "mozBmk______",
+    type: "bookmark",
+    title: "Mozilla",
+    bmkUri: "https://mozilla.org",
+    tags: ["moz", "dot", "org"],
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Insert new local bookmark to upload");
+  let [bzBmk] = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [{
+      guid: "bzBmk_______",
+      url: "https://bugzilla.mozilla.org",
+      title: "Bugzilla",
+      tags: ["new", "tag"],
+    }],
+  });
+
+  info("Insert remote bookmarks and folder to apply");
+  await buf.store(shuffle([{
+    id: "mozBmk______",
+    type: "bookmark",
+    title: "Mozilla home page",
+    bmkUri: "https://mozilla.org",
+    tags: ["browsers"],
+  }, {
+    id: "toolbar",
+    type: "folder",
+    title: "Bookmarks Toolbar",
+    children: ["fxBmk_______", "tFolder_____"],
+  }, {
+    id: "fxBmk_______",
+    type: "bookmark",
+    title: "Get Firefox",
+    bmkUri: "http://getfirefox.com",
+    tags: ["taggy", "browsers"],
+  }, {
+    id: "tFolder_____",
+    type: "folder",
+    title: "Mail",
+    children: ["tbBmk_______"],
+  }, {
+    id: "tbBmk_______",
+    type: "bookmark",
+    title: "Get Thunderbird",
+    bmkUri: "http://getthunderbird.com",
+    keyword: "tb",
+  }]));
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications({ skipTags: true });
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  deepEqual(changesToUpload, {
+    bzBmk_______: {
+      tombstone: false,
+      counter: 3,
+      synced: false,
+      cleartext: {
+        id: "bzBmk_______",
+        type: "bookmark",
+        parentid: "toolbar",
+        hasDupe: false,
+        parentName: "Bookmarks Toolbar",
+        dateAdded: bzBmk.dateAdded.getTime(),
+        bmkUri: "https://bugzilla.mozilla.org/",
+        title: "Bugzilla",
+        tags: ["new", "tag"],
+      },
+    },
+    toolbar: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "toolbar",
+        type: "folder",
+        parentid: "places",
+        hasDupe: false,
+        parentName: "",
+        dateAdded: 0,
+        title: "Bookmarks Toolbar",
+        children: ["fxBmk_______", "tFolder_____", "bzBmk_______"],
+      },
+    },
+  }, "Should upload new local bookmarks and parents");
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds(["fxBmk_______",
+    "tFolder_____", "tbBmk_______", "bzBmk_______", "mozBmk______"]);
+  observer.check([{
+    name: "onItemAdded",
+    params: { itemId: localItemIds.get("fxBmk_______"),
+              parentId: PlacesUtils.toolbarFolderId, index: 0,
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              urlHref: "http://getfirefox.com/", title: "Get Firefox",
+              guid: "fxBmk_______",
+              parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemAdded",
+    params: { itemId: localItemIds.get("tFolder_____"),
+              parentId: PlacesUtils.toolbarFolderId,
+              index: 1, type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              urlHref: null, title: "Mail", guid: "tFolder_____",
+              parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemAdded",
+    params: { itemId: localItemIds.get("tbBmk_______"),
+              parentId: localItemIds.get("tFolder_____"), index: 0,
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              urlHref: "http://getthunderbird.com/", title: "Get Thunderbird",
+              guid: "tbBmk_______", parentGuid: "tFolder_____",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bzBmk_______"),
+              oldParentId: PlacesUtils.toolbarFolderId,
+              oldIndex: 0, newParentId: PlacesUtils.toolbarFolderId,
+              newIndex: 2, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bzBmk_______",
+              oldParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+              newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("mozBmk______"), property: "title",
+              isAnnoProperty: false, newValue: "Mozilla home page",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "mozBmk______",
+              parentGuid: PlacesUtils.bookmarks.menuGuid, oldValue: "Mozilla",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("tbBmk_______"), property: "keyword",
+              isAnnoProperty: false, newValue: "tb",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: localItemIds.get("tFolder_____"), guid: "tbBmk_______",
+              parentGuid: "tFolder_____",
+              oldValue: "http://getthunderbird.com/",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }]);
+
+  let fxBmk = await PlacesUtils.bookmarks.fetch("fxBmk_______");
+  ok(fxBmk, "New Firefox bookmark should exist");
+  equal(fxBmk.parentGuid, PlacesUtils.bookmarks.toolbarGuid,
+    "Should add Firefox bookmark to toolbar");
+  let fxTags = PlacesUtils.tagging.getTagsForURI(
+    Services.io.newURI("http://getfirefox.com"));
+  deepEqual(fxTags.sort(), ["browsers", "taggy"],
+    "Should tag new Firefox bookmark");
+
+  let folder = await PlacesUtils.bookmarks.fetch("tFolder_____");
+  ok(folder, "New folder should exist");
+  equal(folder.parentGuid, PlacesUtils.bookmarks.toolbarGuid,
+    "Should add new folder to toolbar");
+
+  let tbBmk = await PlacesUtils.bookmarks.fetch("tbBmk_______");
+  ok(tbBmk, "Should insert Thunderbird child bookmark");
+  equal(tbBmk.parentGuid, folder.guid,
+    "Should add Thunderbird bookmark to new folder");
+  let keywordInfo = await PlacesUtils.keywords.fetch("tb");
+  equal(keywordInfo.url.href, "http://getthunderbird.com/",
+    "Should set keyword for Thunderbird bookmark");
+
+  let updatedBmk = await PlacesUtils.bookmarks.fetch("mozBmk______");
+  equal(updatedBmk.title, "Mozilla home page",
+    "Should rename Mozilla bookmark");
+  equal(updatedBmk.parentGuid, PlacesUtils.bookmarks.menuGuid,
+    "Should not move Mozilla bookmark");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_value_only_changes() {
+  let buf = await openMirror("value_only_changes");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "A",
+      children: [{
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      }, {
+        guid: "bookmarkCCCC",
+        url: "http://example.com/c",
+        title: "C",
+      }, {
+        guid: "folderJJJJJJ",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "J",
+        children: [{
+          guid: "bookmarkKKKK",
+          url: "http://example.com/k",
+          title: "K",
+        }],
+      }, {
+        guid: "bookmarkDDDD",
+        url: "http://example.com/d",
+        title: "D",
+      }, {
+        guid: "bookmarkEEEE",
+        url: "http://example.com/e",
+        title: "E",
+      }],
+    }, {
+      guid: "folderFFFFFF",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "F",
+      children: [{
+        guid: "bookmarkGGGG",
+        url: "http://example.com/g",
+        title: "G",
+      }, {
+        guid: "folderHHHHHH",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "H",
+        children: [{
+          guid: "bookmarkIIII",
+          url: "http://example.com/i",
+          title: "I",
+        }],
+      }],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["folderAAAAAA", "folderFFFFFF"],
+  }, {
+    id: "folderAAAAAA",
+    type: "folder",
+    title: "A",
+    children: ["bookmarkBBBB", "bookmarkCCCC", "folderJJJJJJ", "bookmarkDDDD",
+               "bookmarkEEEE"],
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }, {
+    id: "folderJJJJJJ",
+    type: "folder",
+    title: "J",
+    children: ["bookmarkKKKK"],
+  }, {
+    id: "bookmarkKKKK",
+    type: "bookmark",
+    title: "K",
+    bmkUri: "http://example.com/k",
+  }, {
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D",
+    bmkUri: "http://example.com/d",
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E",
+    bmkUri: "http://example.com/e",
+  }, {
+    id: "folderFFFFFF",
+    type: "folder",
+    title: "F",
+    children: ["bookmarkGGGG", "folderHHHHHH"],
+  }, {
+    id: "bookmarkGGGG",
+    type: "bookmark",
+    title: "G",
+    bmkUri: "http://example.com/g",
+  }, {
+    id: "folderHHHHHH",
+    type: "folder",
+    title: "H",
+    children: ["bookmarkIIII"],
+  }, {
+    id: "bookmarkIIII",
+    type: "bookmark",
+    title: "I",
+    bmkUri: "http://example.com/i",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await buf.store(shuffle([{
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C (remote)",
+    bmkUri: "http://example.com/c-remote",
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E (remote)",
+    bmkUri: "http://example.com/e-remote",
+  }, {
+    id: "bookmarkIIII",
+    type: "bookmark",
+    title: "I (remote)",
+    bmkUri: "http://example.com/i-remote",
+  }, {
+    id: "folderFFFFFF",
+    type: "folder",
+    title: "F (remote)",
+    children: ["bookmarkGGGG", "folderHHHHHH"],
+  }]));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: [],
+    deleted: [],
+  }, "Should not upload records for remote-only value changes");
+
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, {
+    guid: PlacesUtils.bookmarks.rootGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: "",
+    children: [{
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "Bookmarks Menu",
+      children: [{
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: "A",
+        children: [{
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "B",
+          url: "http://example.com/b",
+        }, {
+          guid: "bookmarkCCCC",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "C (remote)",
+          url: "http://example.com/c-remote",
+        }, {
+          guid: "folderJJJJJJ",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 2,
+          title: "J",
+          children: [{
+            guid: "bookmarkKKKK",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "K",
+            url: "http://example.com/k",
+          }],
+        }, {
+          guid: "bookmarkDDDD",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 3,
+          title: "D",
+          url: "http://example.com/d",
+        }, {
+          guid: "bookmarkEEEE",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 4,
+          title: "E (remote)",
+          url: "http://example.com/e-remote",
+        }],
+      }, {
+        guid: "folderFFFFFF",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 1,
+        title: "F (remote)",
+        children: [{
+          guid: "bookmarkGGGG",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "G",
+          url: "http://example.com/g",
+        }, {
+          guid: "folderHHHHHH",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "H",
+          children: [{
+            guid: "bookmarkIIII",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "I (remote)",
+            url: "http://example.com/i-remote",
+          }],
+        }],
+      }],
+    }, {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: "Bookmarks Toolbar",
+    }, {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: "Other Bookmarks",
+    }, {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+    }],
+  }, "Should not change structure for value-only changes");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_keywords() {
+  let buf = await openMirror("keywords");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "bookmarkAAAA",
+      title: "A",
+      url: "http://example.com/a",
+      keyword: "one",
+    }, {
+      guid: "bookmarkBBBB",
+      title: "B",
+      url: "http://example.com/b",
+      keyword: "two",
+    }, {
+      guid: "bookmarkCCCC",
+      title: "C",
+      url: "http://example.com/c",
+    }, {
+      guid: "bookmarkDDDD",
+      title: "D",
+      url: "http://example.com/d",
+      keyword: "three",
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkCCCC", "bookmarkDDDD"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+    keyword: "one",
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+    keyword: "two",
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }, {
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D",
+    bmkUri: "http://example.com/d",
+    keyword: "three",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Change keywords remotely");
+  await buf.store(shuffle([{
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+    keyword: "two",
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+  }]));
+
+  info("Change keywords locally");
+  await PlacesUtils.keywords.insert({
+    keyword: "four",
+    url: "http://example.com/c",
+  });
+  await PlacesUtils.keywords.remove("three");
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkAAAA", "bookmarkCCCC", "bookmarkDDDD"],
+    deleted: [],
+  }, "Should reupload all local records with changed keywords");
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds(["bookmarkAAAA",
+    "bookmarkBBBB"]);
+  observer.check([{
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkAAAA"), property: "keyword",
+              isAnnoProperty: false, newValue: "",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkAAAA",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              oldValue: "http://example.com/a",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkBBBB"), property: "keyword",
+              isAnnoProperty: false, newValue: "",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkBBBB",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              oldValue: "http://example.com/b",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkAAAA"), property: "keyword",
+              isAnnoProperty: false, newValue: "two",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkAAAA",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              oldValue: "http://example.com/a",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }]);
+
+  let entryForOne = await PlacesUtils.keywords.fetch("one");
+  ok(!entryForOne, "Should remove existing keyword from A");
+
+  let entriesForTwo = await fetchAllKeywords("two");
+  deepEqual(entriesForTwo.map(entry => ({
+    keyword: entry.keyword,
+    url: entry.url.href,
+  })), [{
+    keyword: "two",
+    url: "http://example.com/a",
+  }], "Should move keyword for B to A");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_keywords_complex() {
+  let buf = await openMirror("keywords_complex");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "bookmarkBBBB",
+      title: "B",
+      url: "http://example.com/b",
+      keyword: "four",
+    }, {
+      guid: "bookmarkCCCC",
+      title: "C",
+      url: "http://example.com/c",
+      keyword: "five",
+    }, {
+      guid: "bookmarkDDDD",
+      title: "D",
+      url: "http://example.com/d",
+    }, {
+      guid: "bookmarkEEEE",
+      title: "E",
+      url: "http://example.com/e",
+      keyword: "three",
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkBBBB", "bookmarkCCCC", "bookmarkDDDD", "bookmarkEEEE"],
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+    keyword: "four",
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+    keyword: "five",
+  }, {
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D",
+    bmkUri: "http://example.com/d",
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E",
+    bmkUri: "http://example.com/e",
+    keyword: "three",
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkAAAA", "bookmarkAAA1", "bookmarkBBBB", "bookmarkCCCC",
+               "bookmarkDDDD", "bookmarkEEEE"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+    keyword: "one",
+  }, {
+    id: "bookmarkAAA1",
+    type: "bookmark",
+    title: "A (copy)",
+    bmkUri: "http://example.com/a",
+    keyword: "two",
+  }, {
+    id: "bookmarkBBB1",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C (remote)",
+    bmkUri: "http://example.com/c-remote",
+    keyword: "six",
+  }]));
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  let expectedIdsToUpload = {
+    updated: ["bookmarkBBBB", "bookmarkCCCC"],
+    deleted: [],
+  };
+
+  // We'll take the keyword of either "bookmarkAAAA" or "bookmarkAAA1",
+  // depending on which we see first, and reupload the other.
+  let entriesForOne = await fetchAllKeywords("one");
+  let entriesForTwo = await fetchAllKeywords("two");
+  if (entriesForOne.length) {
+    expectedIdsToUpload.updated.push("bookmarkAAA1");
+    ok(!entriesForTwo.length, "Should drop conflicting keyword from A1");
+    deepEqual(entriesForOne.map(keyword => keyword.url.href),
+      ["http://example.com/a"], "Should use A keyword for A and A1");
+  } else {
+    expectedIdsToUpload.updated.push("bookmarkAAAA");
+    ok(!entriesForOne.length, "Should drop conflicting keyword from A");
+    deepEqual(entriesForTwo.map(keyword => keyword.url.href),
+      ["http://example.com/a"], "Should use A1 keyword for A and A1");
+  }
+  expectedIdsToUpload.updated.sort();
+  deepEqual(idsToUpload, expectedIdsToUpload,
+    "Should reupload all local records with corrected keywords");
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds(["bookmarkAAAA",
+    "bookmarkAAA1", "bookmarkBBB1", "bookmarkBBBB", "bookmarkCCCC",
+    "bookmarkDDDD", "bookmarkEEEE"]);
+  let expectedNotifications = [{
+    name: "onItemAdded",
+    params: { itemId: localItemIds.get("bookmarkAAAA"),
+              parentId: PlacesUtils.bookmarksMenuFolderId, index: 0,
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              urlHref: "http://example.com/a", title: "A",
+              guid: "bookmarkAAAA",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemAdded",
+    params: { itemId: localItemIds.get("bookmarkAAA1"),
+              parentId: PlacesUtils.bookmarksMenuFolderId, index: 1,
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              urlHref: "http://example.com/a", title: "A (copy)",
+              guid: "bookmarkAAA1",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemAdded",
+    params: { itemId: localItemIds.get("bookmarkBBB1"),
+              parentId: PlacesUtils.unfiledBookmarksFolderId, index: 0,
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              urlHref: "http://example.com/b", title: "B",
+              guid: "bookmarkBBB1",
+              parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    // These `onItemMoved` notifications aren't necessary: we only moved
+    // (B C D E) to accomodate (A A1 B1), and Places doesn't usually fire move
+    // notifications for repositioned siblings. However, detecting and filtering
+    // these out complicates `noteObserverChanges`, so, for simplicity, we
+    // record and fire the extra notifications.
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bookmarkBBBB"),
+              oldParentId: PlacesUtils.bookmarksMenuFolderId,
+              oldIndex: 0, newParentId: PlacesUtils.bookmarksMenuFolderId,
+              newIndex: 2, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bookmarkBBBB",
+              oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+              newParentGuid: PlacesUtils.bookmarks.menuGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bookmarkCCCC"),
+              oldParentId: PlacesUtils.bookmarksMenuFolderId,
+              oldIndex: 1, newParentId: PlacesUtils.bookmarksMenuFolderId,
+              newIndex: 3, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bookmarkCCCC",
+              oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+              newParentGuid: PlacesUtils.bookmarks.menuGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bookmarkDDDD"),
+              oldParentId: PlacesUtils.bookmarksMenuFolderId,
+              oldIndex: 2, newParentId: PlacesUtils.bookmarksMenuFolderId,
+              newIndex: 4, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bookmarkDDDD",
+              oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+              newParentGuid: PlacesUtils.bookmarks.menuGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemMoved",
+    params: { itemId: localItemIds.get("bookmarkEEEE"),
+              oldParentId: PlacesUtils.bookmarksMenuFolderId,
+              oldIndex: 3, newParentId: PlacesUtils.bookmarksMenuFolderId,
+              newIndex: 5, type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              guid: "bookmarkEEEE",
+              oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+              newParentGuid: PlacesUtils.bookmarks.menuGuid,
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkCCCC"), property: "title",
+              isAnnoProperty: false, newValue: "C (remote)",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkCCCC",
+              parentGuid: PlacesUtils.bookmarks.menuGuid, oldValue: "C",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkCCCC"), property: "uri",
+              isAnnoProperty: false, newValue: "http://example.com/c-remote",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkCCCC",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              oldValue: "http://example.com/c",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }];
+  if (entriesForOne.length) {
+    expectedNotifications.push({
+      name: "onItemChanged",
+      params: { itemId: localItemIds.get("bookmarkAAA1"), property: "keyword",
+                isAnnoProperty: false, newValue: "two",
+                type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkAAA1",
+                parentGuid: PlacesUtils.bookmarks.menuGuid,
+                oldValue: "http://example.com/a",
+                source: PlacesUtils.bookmarks.SOURCES.SYNC },
+    }, {
+      name: "onItemChanged",
+      params: { itemId: localItemIds.get("bookmarkAAA1"), property: "keyword",
+                isAnnoProperty: false, newValue: "",
+                type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkAAA1",
+                parentGuid: PlacesUtils.bookmarks.menuGuid,
+                oldValue: "http://example.com/a",
+                source: PlacesUtils.bookmarks.SOURCES.SYNC },
+    }, {
+      name: "onItemChanged",
+      params: { itemId: localItemIds.get("bookmarkAAAA"), property: "keyword",
+                isAnnoProperty: false, newValue: "one",
+                type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkAAAA",
+                parentGuid: PlacesUtils.bookmarks.menuGuid,
+                oldValue: "http://example.com/a",
+                source: PlacesUtils.bookmarks.SOURCES.SYNC },
+    });
+  } else {
+    // TODO(kitcambridge): This never happens, even in chaos mode.
+  }
+  expectedNotifications.push({
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkBBBB"), property: "keyword",
+              isAnnoProperty: false, newValue: "",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkBBBB",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              oldValue: "http://example.com/b",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkCCCC"), property: "keyword",
+              isAnnoProperty: false, newValue: "",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkCCCC",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              oldValue: "http://example.com/c",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkCCCC"), property: "keyword",
+              isAnnoProperty: false, newValue: "six",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkCCCC",
+              parentGuid: PlacesUtils.bookmarks.menuGuid,
+              oldValue: "http://example.com/c-remote",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  });
+  observer.check(expectedNotifications);
+
+  let entriesForFour = await fetchAllKeywords("four");
+  ok(!entriesForFour.length, "Should remove all keywords for B");
+
+  let entriesForOldC = await fetchAllKeywords({
+    url: "http://example.com/c",
+  });
+  ok(!entriesForOldC.length, "Should remove all keywords from old C URL");
+  let entriesForNewC = await fetchAllKeywords({
+    url: "http://example.com/c-remote",
+  });
+  deepEqual(entriesForNewC.map(entry => entry.keyword), ["six"],
+    "Should add new keyword to new C URL");
+
+  let entriesForD = await fetchAllKeywords("http://example.com/d");
+  ok(!entriesForD.length, "Should not add keywords to D");
+
+  let entriesForThree = await fetchAllKeywords("three");
+  deepEqual(entriesForThree.map(keyword => keyword.url.href),
+    ["http://example.com/e"], "Should not change keywords for E");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_tags() {
+  let buf = await openMirror("tags");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "bookmarkAAAA",
+      title: "A",
+      url: "http://example.com/a",
+      tags: ["one", "two", "three", "four"],
+    }, {
+      guid: "bookmarkBBBB",
+      title: "B",
+      url: "http://example.com/b",
+      tags: ["five", "six"],
+    }, {
+      guid: "bookmarkCCCC",
+      title: "C",
+      url: "http://example.com/c",
+    }, {
+      guid: "bookmarkDDDD",
+      title: "D",
+      url: "http://example.com/d",
+      tags: ["seven", "eight", "nine"],
+    }],
+  });
+  await buf.store(shuffle([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkCCCC", "bookmarkDDDD"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+    tags: ["one", "two", "three", "four"],
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+    tags: ["five", "six"],
+  }, {
+    id: "bookmarkCCCC",
+    type: "bookmark",
+    title: "C",
+    bmkUri: "http://example.com/c",
+  }, {
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D",
+    bmkUri: "http://example.com/d",
+    tags: ["seven", "eight", "nine"],
+  }]), { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Change tags remotely");
+  await buf.store(shuffle([{
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+    tags: ["one", "two", "ten"],
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    bmkUri: "http://example.com/b",
+    tags: [],
+  }]));
+
+  info("Change tags locally");
+  PlacesUtils.tagging.tagURI(Services.io.newURI(
+    "http://example.com/c"), ["eleven", "twelve"]);
+  PlacesUtils.tagging.untagURI(Services.io.newURI(
+    "http://example.com/d"), null);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkCCCC", "bookmarkDDDD"],
+    deleted: [],
+  }, "Should upload local records with new tags");
+
+  deepEqual(changesToUpload.bookmarkCCCC.cleartext.tags.sort(),
+    ["eleven", "twelve"], "Should upload record with new tags for C");
+  ok(!changesToUpload.bookmarkDDDD.cleartext.tags,
+    "Should upload record for D with tags removed");
+
+  let tagsForA = PlacesUtils.tagging.getTagsForURI(
+    Services.io.newURI("http://example.com/a"));
+  deepEqual(tagsForA.sort(), ["one", "ten", "two"], "Should change tags for A");
+
+  let tagsForB = PlacesUtils.tagging.getTagsForURI(
+    Services.io.newURI("http://example.com/b"));
+  deepEqual(tagsForB, [], "Should remove all tags from B");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_rewrite_tag_queries() {
+  let buf = await openMirror("rewrite_tag_queries");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "bookmarkAAAA",
+      title: "A",
+      url: "http://example.com/a",
+    }, {
+      guid: "bookmarkDDDD",
+      title: "D",
+      url: "http://example.com/d",
+      tags: ["kitty"],
+    }],
+  });
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkAAAA", "bookmarkDDDD"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    bmkUri: "http://example.com/a",
+  }, {
+    id: "bookmarkDDDD",
+    type: "bookmark",
+    title: "D",
+    bmkUri: "http://example.com/d",
+    tags: ["kitty"],
+  }], { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Add tag queries for new and existing tags");
+  await buf.store([{
+    id: "toolbar",
+    type: "folder",
+    title: "Bookmarks Toolbar",
+    children: ["queryBBBBBBB", "queryCCCCCCC", "bookmarkEEEE"],
+  }, {
+    id: "queryBBBBBBB",
+    type: "query",
+    title: "Tagged stuff",
+    bmkUri: "place:type=7&folder=999",
+    folderName: "taggy",
+  }, {
+    id: "queryCCCCCCC",
+    type: "query",
+    title: "Cats",
+    bmkUri: "place:type=7&folder=888",
+    folderName: "kitty",
+  }, {
+    id: "bookmarkEEEE",
+    type: "bookmark",
+    title: "E",
+    bmkUri: "http://example.com/e",
+    tags: ["taggy"],
+  }]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  deepEqual(changesToUpload, {}, "Should not reupload any local records");
+
+  let urisWithTaggy = PlacesUtils.tagging.getURIsForTag("taggy");
+  deepEqual(urisWithTaggy.map(uri => uri.spec).sort(), ["http://example.com/e"],
+    "Should insert bookmark with new tag");
+
+  let urisWithKitty = PlacesUtils.tagging.getURIsForTag("kitty");
+  deepEqual(urisWithKitty.map(uri => uri.spec).sort(), ["http://example.com/d"],
+    "Should retain existing tag");
+
+  let { root: toolbarContainer } = PlacesUtils.getFolderContents(
+    PlacesUtils.toolbarFolderId, false, true);
+  equal(toolbarContainer.childCount, 3,
+    "Should add queries and bookmark to toolbar");
+
+  let containerForB = PlacesUtils.asContainer(toolbarContainer.getChild(0));
+  containerForB.containerOpen = true;
+  for (let i = 0; i < containerForB.childCount; ++i) {
+    let child = containerForB.getChild(i);
+    equal(child.uri, "http://example.com/e",
+      `Rewritten tag query B should have tagged child node at ${i}`);
+  }
+  containerForB.containerOpen = false;
+
+  let containerForC = PlacesUtils.asContainer(toolbarContainer.getChild(1));
+  containerForC.containerOpen = true;
+  for (let i = 0; i < containerForC.childCount; ++i) {
+    let child = containerForC.getChild(i);
+    equal(child.uri, "http://example.com/d",
+      `Rewritten tag query C should have tagged child node at ${i}`);
+  }
+  containerForC.containerOpen = false;
+
+  toolbarContainer.containerOpen = false;
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_date_added() {
+  let buf = await openMirror("date_added");
+
+  let aDateAdded = new Date(Date.now() - 1 * 24 * 60 * 60 * 1000);
+  let bDateAdded = new Date();
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [{
+      guid: "bookmarkAAAA",
+      dateAdded: aDateAdded,
+      title: "A",
+      url: "http://example.com/a",
+    }, {
+      guid: "bookmarkBBBB",
+      dateAdded: bDateAdded,
+      title: "B",
+      url: "http://example.com/b",
+    }],
+  });
+  await buf.store([{
+    id: "menu",
+    type: "folder",
+    title: "Bookmarks Menu",
+    children: ["bookmarkAAAA", "bookmarkBBBB"],
+  }, {
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A",
+    dateAdded: aDateAdded.getTime(),
+    bmkUri: "http://example.com/a",
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B",
+    dateAdded: bDateAdded.getTime(),
+    bmkUri: "http://example.com/b",
+  }], { needsMerge: false });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  let bNewDateAdded = new Date(bDateAdded.getTime() - 1 * 60 * 60 * 1000);
+  await buf.store([{
+    id: "bookmarkAAAA",
+    type: "bookmark",
+    title: "A (remote)",
+    dateAdded: Date.now(),
+    bmkUri: "http://example.com/a",
+  }, {
+    id: "bookmarkBBBB",
+    type: "bookmark",
+    title: "B (remote)",
+    dateAdded: bNewDateAdded.getTime(),
+    bmkUri: "http://example.com/b",
+  }]);
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload, {
+    updated: ["bookmarkAAAA"],
+    deleted: []
+  }, "Should flag A for weak reupload");
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds(["bookmarkAAAA",
+    "bookmarkBBBB"]);
+  observer.check([{
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkAAAA"), property: "title",
+              isAnnoProperty: false, newValue: "A (remote)",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkAAAA",
+              parentGuid: PlacesUtils.bookmarks.menuGuid, oldValue: "A",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }, {
+    name: "onItemChanged",
+    params: { itemId: localItemIds.get("bookmarkBBBB"), property: "title",
+              isAnnoProperty: false, newValue: "B (remote)",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              parentId: PlacesUtils.bookmarksMenuFolderId, guid: "bookmarkBBBB",
+              parentGuid: PlacesUtils.bookmarks.menuGuid, oldValue: "B",
+              source: PlacesUtils.bookmarks.SOURCES.SYNC },
+  }]);
+
+  let changeCounter = changesToUpload.bookmarkAAAA.counter;
+  strictEqual(changeCounter, 0, "Should not bump change counter for A");
+
+  let aInfo = await PlacesUtils.bookmarks.fetch("bookmarkAAAA");
+  equal(aInfo.title, "A (remote)", "Should change local title for A");
+  deepEqual(aInfo.dateAdded, aDateAdded,
+    "Should not change date added for A to newer remote date");
+
+  let bInfo = await PlacesUtils.bookmarks.fetch("bookmarkBBBB");
+  equal(bInfo.title, "B (remote)", "Should change local title for B");
+  deepEqual(bInfo.dateAdded, bNewDateAdded,
+    "Should take older date added for B");
+});
diff --git a/toolkit/components/places/tests/sync/xpcshell.ini b/toolkit/components/places/tests/sync/xpcshell.ini
new file mode 100644
index 000000000000..5beda4703356
--- /dev/null
+++ b/toolkit/components/places/tests/sync/xpcshell.ini
@@ -0,0 +1,11 @@
+[DEFAULT]
+head = head_sync.js
+support-files =
+  livemark.xml
+
+[test_bookmark_corruption.js]
+[test_bookmark_deduping.js]
+[test_bookmark_deletion.js]
+[test_bookmark_kinds.js]
+[test_bookmark_structure_changes.js]
+[test_bookmark_value_changes.js]
diff --git a/tools/lint/eslint/modules.json b/tools/lint/eslint/modules.json
index f3454136d5f3..f1ebea027831 100644
--- a/tools/lint/eslint/modules.json
+++ b/tools/lint/eslint/modules.json
@@ -193,6 +193,7 @@
   "subprocess_unix.jsm": ["SubprocessImpl"],
   "subprocess_win.jsm": ["SubprocessImpl"],
   "sync.jsm": ["Authentication"],
+  "SyncedBookmarksMirror.jsm": ["SyncedBookmarksMirror"],
   "tabs.js": ["TabEngine", "TabSetRecord"],
   "tabs.jsm": ["BrowserTabs"],
   "tcpsocket_test.jsm": ["createSocket", "createServer", "enablePrefsAndPermissions", "socketCompartmentInstanceOfArrayBuffer"],

From 2449a744b8644a7ab1b348a3e51610831c40d0be Mon Sep 17 00:00:00 2001
From: Kit Cambridge <kit@yakshaving.ninja>
Date: Tue, 19 Dec 2017 17:10:54 -0800
Subject: [PATCH 43/46] Bug 1305563 - Add a `BufferedBookmarksEngine` that can
 be toggled with a pref. r=markh,tcsc

This patch adds a new bookmarks engine, pref'd off, that writes to the
buffer instead of Places.

MozReview-Commit-ID: 7idBa03kOzm

--HG--
extra : rebase_source : 671ebeb3de824dc43c0ef14b2dfe065a33e2911a
---
 services/sync/modules/engines.js              |  60 +-
 services/sync/modules/engines/bookmarks.js    | 637 ++++++++++++------
 services/sync/modules/service.js              |  12 +-
 services/sync/services-sync.js                |   1 +
 .../sync/tests/unit/test_bookmark_engine.js   | 134 ++--
 .../sync/tests/unit/test_bookmark_repair.js   |  68 +-
 .../unit/test_bookmark_repair_responder.js    |   4 +-
 .../sync/tests/unit/test_bookmark_tracker.js  | 261 +------
 services/sync/tests/unit/test_telemetry.js    |   1 +
 tools/lint/eslint/modules.json                |   2 +-
 10 files changed, 665 insertions(+), 515 deletions(-)

diff --git a/services/sync/modules/engines.js b/services/sync/modules/engines.js
index 390f04955ab1..3202034119b2 100644
--- a/services/sync/modules/engines.js
+++ b/services/sync/modules/engines.js
@@ -895,6 +895,12 @@ SyncEngine.prototype = {
   /*
    * lastSync is a timestamp in server time.
    */
+  async getLastSync() {
+    return this.lastSync;
+  },
+  async setLastSync(lastSync) {
+    this.lastSync = lastSync;
+  },
   get lastSync() {
     return this._lastSync;
   },
@@ -1029,13 +1035,7 @@ SyncEngine.prototype = {
     // the end of a sync, or after an error, we add all objects remaining in
     // this._modified to the tracker.
     this.lastSyncLocal = Date.now();
-    let initialChanges;
-    if (this.lastSync) {
-      initialChanges = await this.pullNewChanges();
-    } else {
-      this._log.debug("First sync, uploading all items");
-      initialChanges = await this.pullAllChanges();
-    }
+    let initialChanges = await this.pullChanges();
     this._modified.replace(initialChanges);
     // Clear the tracker now. If the sync fails we'll add the ones we failed
     // to upload back.
@@ -1049,6 +1049,15 @@ SyncEngine.prototype = {
     this._delete = {};
   },
 
+  async pullChanges() {
+    let lastSync = await this.getLastSync();
+    if (lastSync) {
+      return this.pullNewChanges();
+    }
+    this._log.debug("First sync, uploading all items");
+    return this.pullAllChanges();
+  },
+
   /**
    * A tiny abstraction to make it easier to test incoming record
    * application.
@@ -1080,8 +1089,9 @@ SyncEngine.prototype = {
     this._log.trace("Downloading & applying server changes");
 
     let newitems = this.itemSource();
+    let lastSync = await this.getLastSync();
 
-    newitems.newer = this.lastSync;
+    newitems.newer = lastSync;
     newitems.full  = true;
 
     let downloadLimit = Infinity;
@@ -1117,7 +1127,7 @@ SyncEngine.prototype = {
     let downloadedIDs = new Set();
 
     // Stage 1: Fetch new records from the server, up to the download limit.
-    if (this.lastModified == null || this.lastModified > this.lastSync) {
+    if (this.lastModified == null || this.lastModified > lastSync) {
       let { response, records } = await newitems.getBatched(this.downloadBatchSize);
       if (!response.success) {
         response.failureCode = ENGINE_DOWNLOAD_FAIL;
@@ -1164,7 +1174,7 @@ SyncEngine.prototype = {
     if (downloadedIDs.size == downloadLimit) {
       let guidColl = this.itemSource();
 
-      guidColl.newer = this.lastSync;
+      guidColl.newer = lastSync;
       guidColl.older = oldestModified;
       guidColl.sort  = "oldest";
 
@@ -1182,8 +1192,9 @@ SyncEngine.prototype = {
 
     // Fast-foward the lastSync timestamp since we have backlogged the
     // remaining items.
-    if (this.lastSync < this.lastModified) {
-      this.lastSync = this.lastModified;
+    if (lastSync < this.lastModified) {
+      lastSync = this.lastModified;
+      await this.setLastSync(lastSync);
     }
 
     // Stage 3: Backfill records from the backlog, and those that failed to
@@ -1245,8 +1256,9 @@ SyncEngine.prototype = {
       this.toFetch = Utils.setDeleteAll(this.toFetch, ids);
       this.previousFailed = Utils.setAddAll(this.previousFailed, failedInBackfill);
 
-      if (this.lastSync < this.lastModified) {
-        this.lastSync = this.lastModified;
+      if (lastSync < this.lastModified) {
+        lastSync = this.lastModified;
+        await this.setLastSync(lastSync);
       }
     }
 
@@ -1624,6 +1636,7 @@ SyncEngine.prototype = {
 
       let failed = [];
       let successful = [];
+      let lastSync = await this.getLastSync();
       let handleResponse = async (resp, batchOngoing = false) => {
         // Note: We don't want to update this.lastSync, or this._modified until
         // the batch is complete, however we want to remember success/failure
@@ -1642,11 +1655,8 @@ SyncEngine.prototype = {
           // Nothing to do yet
           return;
         }
-        // Advance lastSync since we've finished the batch.
-        let modified = resp.headers["x-weave-timestamp"];
-        if (modified > this.lastSync) {
-          this.lastSync = modified;
-        }
+        let serverModifiedTime = parseFloat(resp.headers["x-weave-timestamp"]);
+
         if (failed.length && this._log.level <= Log.Level.Debug) {
           this._log.debug("Records that will be uploaded again because "
                           + "the server couldn't store them: "
@@ -1659,14 +1669,20 @@ SyncEngine.prototype = {
           this._modified.delete(id);
         }
 
-        await this._onRecordsWritten(successful, failed);
+        await this._onRecordsWritten(successful, failed, serverModifiedTime);
+
+        // Advance lastSync since we've finished the batch.
+        if (serverModifiedTime > lastSync) {
+          lastSync = serverModifiedTime;
+          await this.setLastSync(lastSync);
+        }
 
         // clear for next batch
         failed.length = 0;
         successful.length = 0;
       };
 
-      let postQueue = up.newPostQueue(this._log, this.lastSync, handleResponse);
+      let postQueue = up.newPostQueue(this._log, lastSync, handleResponse);
 
       for (let id of modifiedIDs) {
         let out;
@@ -1717,7 +1733,7 @@ SyncEngine.prototype = {
     }
   },
 
-  async _onRecordsWritten(succeeded, failed) {
+  async _onRecordsWritten(succeeded, failed, serverModifiedTime) {
     // Implement this method to take specific actions against successfully
     // uploaded records and failed records.
   },
diff --git a/services/sync/modules/engines/bookmarks.js b/services/sync/modules/engines/bookmarks.js
index c023859c941a..75413ea51243 100644
--- a/services/sync/modules/engines/bookmarks.js
+++ b/services/sync/modules/engines/bookmarks.js
@@ -4,7 +4,8 @@
 
 this.EXPORTED_SYMBOLS = ["BookmarksEngine", "PlacesItem", "Bookmark",
                          "BookmarkFolder", "BookmarkQuery",
-                         "Livemark", "BookmarkSeparator"];
+                         "Livemark", "BookmarkSeparator",
+                         "BufferedBookmarksEngine"];
 
 var Cc = Components.classes;
 var Ci = Components.interfaces;
@@ -18,23 +19,26 @@ Cu.import("resource://services-sync/engines.js");
 Cu.import("resource://services-sync/record.js");
 Cu.import("resource://services-sync/util.js");
 
-XPCOMUtils.defineLazyModuleGetter(this, "BookmarkValidator",
-                                  "resource://services-sync/bookmark_validator.js");
+XPCOMUtils.defineLazyModuleGetters(this, {
+  SyncedBookmarksMirror: "resource://gre/modules/SyncedBookmarksMirror.jsm",
+  BookmarkValidator: "resource://services-sync/bookmark_validator.js",
+  OS: "resource://gre/modules/osfile.jsm",
+  PlacesBackups: "resource://gre/modules/PlacesBackups.jsm",
+  PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.jsm",
+  PlacesUtils: "resource://gre/modules/PlacesUtils.jsm",
+  Resource: "resource://services-sync/resource.js",
+});
+
 XPCOMUtils.defineLazyGetter(this, "PlacesBundle", () => {
   return Services.strings.createBundle("chrome://places/locale/places.properties");
 });
-XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
-                                  "resource://gre/modules/PlacesUtils.jsm");
-XPCOMUtils.defineLazyModuleGetter(this, "PlacesSyncUtils",
-                                  "resource://gre/modules/PlacesSyncUtils.jsm");
-XPCOMUtils.defineLazyModuleGetter(this, "PlacesBackups",
-                                  "resource://gre/modules/PlacesBackups.jsm");
 
-const ANNOS_TO_TRACK = [PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO,
-                        PlacesSyncUtils.bookmarks.SIDEBAR_ANNO,
-                        PlacesUtils.LMANNO_FEEDURI, PlacesUtils.LMANNO_SITEURI];
+XPCOMUtils.defineLazyGetter(this, "ANNOS_TO_TRACK", () => [
+  PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO,
+  PlacesSyncUtils.bookmarks.SIDEBAR_ANNO, PlacesUtils.LMANNO_FEEDURI,
+  PlacesUtils.LMANNO_SITEURI,
+]);
 
-const SERVICE_NOT_SUPPORTED = "Service not supported on this platform";
 const FOLDER_SORTINDEX = 1000000;
 const {
   SOURCE_SYNC,
@@ -43,10 +47,6 @@ const {
   SOURCE_SYNC_REPARENT_REMOVED_FOLDER_CHILDREN,
 } = Ci.nsINavBookmarksService;
 
-const ORGANIZERQUERY_ANNO = "PlacesOrganizer/OrganizerQuery";
-const ALLBOOKMARKS_ANNO = "AllBookmarks";
-const MOBILE_ANNO = "MobileBookmarks";
-
 // Roots that should be deleted from the server, instead of applied locally.
 // This matches `AndroidBrowserBookmarksRepositorySession::forbiddenGUID`,
 // but allows tags because we don't want to reparent tag folders or tag items
@@ -273,13 +273,53 @@ BookmarkSeparator.prototype = {
 
 Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos");
 
-this.BookmarksEngine = function BookmarksEngine(service) {
+/**
+ * The rest of this file implements two different bookmarks engines and stores.
+ * The `services.sync.engine.bookmarks.buffer` pref controls which one we use.
+ * `BaseBookmarksEngine` and `BaseBookmarksStore` define a handful of methods
+ * shared between the two implementations.
+ *
+ * `BookmarksEngine` and `BookmarksStore` pull locally changed IDs before
+ * syncing, examine every incoming record, use the default record-level
+ * reconciliation to resolve merge conflicts, and update records in Places
+ * using public APIs. This is similar to how the other sync engines work.
+ *
+ * Unfortunately, this general approach doesn't serve bookmark sync well.
+ * Bookmarks form a tree locally, but they're stored as opaque, encrypted, and
+ * unordered records on the server. The records are interdependent, with a
+ * set of constraints: each parent must know the IDs and order of its children,
+ * and a child can't appear in multiple parents.
+ *
+ * This has two important implications.
+ *
+ * First, some changes require us to upload multiple records. For example,
+ * moving a bookmark into a different folder uploads the bookmark, old folder,
+ * and new folder.
+ *
+ * Second, conflict resolution, like adding a bookmark to a folder on one
+ * device, and moving a different bookmark out of the same folder on a different
+ * device, must account for the tree structure. Otherwise, we risk uploading an
+ * incomplete tree, and confuse other devices that try to sync.
+ *
+ * Historically, the lack of durable change tracking and atomic uploads meant
+ * that we'd miss these changes entirely, or leave the server in an inconsistent
+ * state after a partial sync. Another device would then sync, download and
+ * apply the partial state directly to Places, and upload its changes. This
+ * could easily result in Sync scrambling bookmarks on both devices, and user
+ * intervention to manually undo the damage would make things worse.
+ *
+ * `BufferedBookmarksEngine` and `BufferedBookmarksStore` mitigate this by
+ * mirroring incoming bookmarks in a separate database, constructing trees from
+ * the local and remote bookmarks, and merging the two trees into a single
+ * consistent tree that accounts for every bookmark. For more information about
+ * merging, please see the explanation above `SyncedBookmarksMirror`.
+ */
+function BaseBookmarksEngine(service) {
   SyncEngine.call(this, "Bookmarks", service);
-};
-BookmarksEngine.prototype = {
+}
+BaseBookmarksEngine.prototype = {
   __proto__: SyncEngine.prototype,
   _recordObj: PlacesItem,
-  _storeObj: BookmarksStore,
   _trackerObj: BookmarksTracker,
   version: 2,
   _defaultSort: "index",
@@ -287,6 +327,70 @@ BookmarksEngine.prototype = {
   syncPriority: 4,
   allowSkippedRecord: false,
 
+  async _syncFinish() {
+    await SyncEngine.prototype._syncFinish.call(this);
+    await PlacesSyncUtils.bookmarks.ensureMobileQuery();
+  },
+
+  async _createRecord(id) {
+    if (this._modified.isTombstone(id)) {
+      // If we already know a changed item is a tombstone, just create the
+      // record without dipping into Places.
+      return this._createTombstone(id);
+    }
+    let record = await SyncEngine.prototype._createRecord.call(this, id);
+    if (record.deleted) {
+      // Make sure deleted items are marked as tombstones. We do this here
+      // in addition to the `isTombstone` call above because it's possible
+      // a changed bookmark might be deleted during a sync (bug 1313967).
+      this._modified.setTombstone(record.id);
+    }
+    return record;
+  },
+
+  async pullAllChanges() {
+    return this.pullNewChanges();
+  },
+
+  async trackRemainingChanges() {
+    let changes = this._modified.changes;
+    await PlacesSyncUtils.bookmarks.pushChanges(changes);
+  },
+
+  _deleteId(id) {
+    this._noteDeletedId(id);
+  },
+
+  async _resetClient() {
+    await super._resetClient();
+    await PlacesSyncUtils.bookmarks.reset();
+  },
+
+  // Cleans up the Places root, reading list items (ignored in bug 762118,
+  // removed in bug 1155684), and pinned sites.
+  _shouldDeleteRemotely(incomingItem) {
+    return FORBIDDEN_INCOMING_IDS.includes(incomingItem.id) ||
+           FORBIDDEN_INCOMING_PARENT_IDS.includes(incomingItem.parentid);
+  },
+
+  getValidator() {
+    return new BookmarkValidator();
+  }
+};
+
+/**
+ * The original bookmarks engine. Uses an in-memory GUID map for deduping, and
+ * the default implementation for reconciling changes. Handles child ordering
+ * and deletions at the end of a sync.
+ */
+this.BookmarksEngine = function BookmarksEngine(service) {
+  BaseBookmarksEngine.apply(this, arguments);
+};
+
+BookmarksEngine.prototype = {
+  __proto__: BaseBookmarksEngine.prototype,
+  _storeObj: BookmarksStore,
+
   emptyChangeset() {
     return new BookmarksChangeset();
   },
@@ -522,34 +626,21 @@ BookmarksEngine.prototype = {
     this._store._childrenToOrder = {};
   },
 
-  async _syncFinish() {
-    await SyncEngine.prototype._syncFinish.call(this);
-    await PlacesSyncUtils.bookmarks.ensureMobileQuery();
-  },
-
   async _syncCleanup() {
     await SyncEngine.prototype._syncCleanup.call(this);
     delete this._guidMap;
   },
 
   async _createRecord(id) {
-    if (this._modified.isTombstone(id)) {
-      // If we already know a changed item is a tombstone, just create the
-      // record without dipping into Places.
-      return this._createTombstone(id);
+    let record = await super._createRecord(id);
+    if (record.deleted) {
+      return record;
     }
-    // Create the record as usual, but mark it as having dupes if necessary.
-    let record = await SyncEngine.prototype._createRecord.call(this, id);
+    // Mark the record as having dupes if necessary.
     let entry = await this._mapDupe(record);
     if (entry != null && entry.hasDupe) {
       record.hasDupe = true;
     }
-    if (record.deleted) {
-      // Make sure deleted items are marked as tombstones. We do this here
-      // in addition to the `isTombstone` call above because it's possible
-      // a changed bookmark might be deleted during a sync (bug 1313967).
-      this._modified.setTombstone(record.id);
-    }
     return record;
   },
 
@@ -569,28 +660,10 @@ BookmarksEngine.prototype = {
     return mapped ? mapped.toString() : mapped;
   },
 
-  async pullAllChanges() {
-    return this.pullNewChanges();
-  },
-
   async pullNewChanges() {
     return this._tracker.promiseChangedIDs();
   },
 
-  async trackRemainingChanges() {
-    let changes = this._modified.changes;
-    await PlacesSyncUtils.bookmarks.pushChanges(changes);
-  },
-
-  _deleteId(id) {
-    this._noteDeletedId(id);
-  },
-
-  async _resetClient() {
-    await SyncEngine.prototype._resetClient.call(this);
-    await PlacesSyncUtils.bookmarks.reset();
-  },
-
   // Called when _findDupe returns a dupe item and the engine has decided to
   // switch the existing item to the new incoming item.
   async _switchItemToDupe(localDupeGUID, incomingItem) {
@@ -600,13 +673,6 @@ BookmarksEngine.prototype = {
     this._modified.insert(newChanges);
   },
 
-  // Cleans up the Places root, reading list items (ignored in bug 762118,
-  // removed in bug 1155684), and pinned sites.
-  _shouldDeleteRemotely(incomingItem) {
-    return FORBIDDEN_INCOMING_IDS.includes(incomingItem.id) ||
-           FORBIDDEN_INCOMING_PARENT_IDS.includes(incomingItem.parentid);
-  },
-
   beforeRecordDiscard(localRecord, remoteRecord, remoteIsNewer) {
     if (localRecord.type != "folder" || remoteRecord.type != "folder") {
       return;
@@ -629,18 +695,228 @@ BookmarksEngine.prototype = {
     this._log.debug("Recording children of " + localRecord.id, order);
     this._store._childrenToOrder[localRecord.id] = order;
   },
+};
 
-  getValidator() {
-    return new BookmarkValidator();
+/**
+ * The buffered bookmarks engine uses a different store that stages downloaded
+ * bookmarks in a separate database, instead of writing directly to Places. The
+ * buffer handles reconciliation, so we stub out `_reconcile`, and wait to pull
+ * changes until we're ready to upload.
+ */
+this.BufferedBookmarksEngine = function BufferedBookmarksEngine() {
+  BaseBookmarksEngine.apply(this, arguments);
+};
+
+BufferedBookmarksEngine.prototype = {
+  __proto__: BaseBookmarksEngine.prototype,
+  _storeObj: BufferedBookmarksStore,
+
+  async getLastSync() {
+    let mirror = await this._store.ensureOpenMirror();
+    return mirror.getCollectionHighWaterMark();
+  },
+
+  async setLastSync(lastSync) {
+    let mirror = await this._store.ensureOpenMirror();
+    await mirror.setCollectionLastModified(lastSync);
+    // Update the pref so that reverting to the original bookmarks engine
+    // doesn't download records we've already applied.
+    super.lastSync = lastSync;
+  },
+
+  get lastSync() {
+    throw new TypeError("Use getLastSync");
+  },
+
+  set lastSync(value) {
+    throw new TypeError("Use setLastSync");
+  },
+
+  emptyChangeset() {
+    return new BufferedBookmarksChangeset();
+  },
+
+  async _processIncoming(newitems) {
+    try {
+      await super._processIncoming(newitems);
+    } finally {
+      let buf = await this._store.ensureOpenMirror();
+      let recordsToUpload = await buf.apply({
+        remoteTimeSeconds: Resource.serverTime,
+      });
+      this._modified.replace(recordsToUpload);
+    }
+  },
+
+  async _reconcile(item) {
+    return true;
+  },
+
+  async _createRecord(id) {
+    if (this._needWeakUpload.has(id)) {
+      return this._store.createRecord(id, this.name);
+    }
+    let change = this._modified.changes[id];
+    if (!change) {
+      this._log.error("Creating record for item ${id} not in strong " +
+                      "changeset", { id });
+      throw new TypeError("Can't create record for unchanged item");
+    }
+    let record = this._recordFromCleartext(id, change.cleartext);
+    record.sortindex = await this._store._calculateIndex(record);
+    return record;
+  },
+
+  _recordFromCleartext(id, cleartext) {
+    let recordObj = getTypeObject(cleartext.type);
+    if (!recordObj) {
+      this._log.warn("Creating record for item ${id} with unknown type ${type}",
+                     { id, type: cleartext.type });
+      recordObj = PlacesItem;
+    }
+    let record = new recordObj(this.name, id);
+    record.cleartext = cleartext;
+    return record;
+  },
+
+  async pullChanges() {
+    return {};
+  },
+
+  /**
+   * Writes successfully uploaded records back to the mirror, so that the
+   * mirror matches the server. We update the mirror before updating Places,
+   * which has implications for interrupted syncs.
+   *
+   * 1. Sync interrupted during upload; server doesn't support atomic uploads.
+   *    We'll download and reapply everything that we uploaded before the
+   *    interruption. All locally changed items retain their change counters.
+   * 2. Sync interrupted during upload; atomic uploads enabled. The server
+   *    discards the batch. All changed local items retain their change
+   *    counters, so the next sync resumes cleanly.
+   * 3. Sync interrupted during upload; outgoing records can't fit in a single
+   *    batch. We'll download and reapply all records through the most recent
+   *    committed batch. This is a variation of (1).
+   * 4. Sync interrupted after we update the mirror, but before cleanup. The
+   *    mirror matches the server, but locally changed items retain their change
+   *    counters. Reuploading them on the next sync should be idempotent, though
+   *    unnecessary. If another client makes a conflicting remote change before
+   *    we sync again, we may incorrectly prefer the local state.
+   * 5. Sync completes successfully. We'll update the mirror, and reset the
+   *    change counters for all items.
+   */
+  async _onRecordsWritten(succeeded, failed, serverModifiedTime) {
+    let records = [];
+    for (let id of succeeded) {
+      let change = this._modified.changes[id];
+      if (!change) {
+        // TODO (Bug 1433178): Write weakly uploaded records back to the mirror.
+        this._log.info("Uploaded record not in strong changeset", id);
+        continue;
+      }
+      if (!change.synced) {
+        this._log.info("Record in strong changeset not uploaded", id);
+        continue;
+      }
+      let cleartext = change.cleartext;
+      if (!cleartext) {
+        this._log.error("Missing Sync record cleartext for ${id} in ${change}",
+                        { id, change });
+        throw new TypeError("Missing cleartext for uploaded Sync record");
+      }
+      let record = this._recordFromCleartext(id, cleartext);
+      record.modified = serverModifiedTime;
+      records.push(record);
+    }
+    let buf = await this._store.ensureOpenMirror();
+    await buf.store(records, { needsMerge: false });
+  },
+
+  async _resetClient() {
+    await super._resetClient();
+    let buf = await this._store.ensureOpenMirror();
+    await buf.reset();
+  },
+
+  async finalize() {
+    await super.finalize();
+    await this._store.finalize();
+  },
+};
+
+/**
+ * The only code shared between `BookmarksStore` and `BufferedBookmarksStore`
+ * is for creating Sync records from Places items. Everything else is
+ * different.
+ */
+function BaseBookmarksStore(name, engine) {
+  Store.call(this, name, engine);
+}
+
+BaseBookmarksStore.prototype = {
+  __proto__: Store.prototype,
+
+  // Create a record starting from the weave id (places guid)
+  async createRecord(id, collection) {
+    let item = await PlacesSyncUtils.bookmarks.fetch(id);
+    if (!item) { // deleted item
+      let record = new PlacesItem(collection, id);
+      record.deleted = true;
+      return record;
+    }
+
+    let recordObj = getTypeObject(item.kind);
+    if (!recordObj) {
+      this._log.warn("Unknown item type, cannot serialize: " + item.kind);
+      recordObj = PlacesItem;
+    }
+    let record = new recordObj(collection, id);
+    record.fromSyncBookmark(item);
+
+    record.sortindex = await this._calculateIndex(record);
+
+    return record;
+  },
+
+  async _calculateIndex(record) {
+    // Ensure folders have a very high sort index so they're not synced last.
+    if (record.type == "folder")
+      return FOLDER_SORTINDEX;
+
+    // For anything directly under the toolbar, give it a boost of more than an
+    // unvisited bookmark
+    let index = 0;
+    if (record.parentid == "toolbar")
+      index += 150;
+
+    // Add in the bookmark's frecency if we have something.
+    if (record.bmkUri != null) {
+      let frecency = await PlacesSyncUtils.history.fetchURLFrecency(record.bmkUri);
+      if (frecency != -1)
+        index += frecency;
+    }
+
+    return index;
+  },
+
+  async wipe() {
+    // Save a backup before clearing out all bookmarks.
+    await PlacesBackups.create(null, true);
+    await PlacesSyncUtils.bookmarks.wipe();
   }
 };
 
-function BookmarksStore(name, engine) {
-  Store.call(this, name, engine);
+/**
+ * The original store updates Places during the sync, using public methods.
+ * `BookmarksStore` implements all abstract `Store` methods, and behaves like
+ * the other stores.
+ */
+function BookmarksStore() {
+  BaseBookmarksStore.apply(this, arguments);
   this._itemsToDelete = new Set();
 }
 BookmarksStore.prototype = {
-  __proto__: Store.prototype,
+  __proto__: BaseBookmarksStore.prototype,
 
   async itemExists(id) {
     return (await this.idForGUID(id)) > 0;
@@ -777,29 +1053,6 @@ BookmarksStore.prototype = {
     this._itemsToDelete.clear();
   },
 
-  // Create a record starting from the weave id (places guid)
-  async createRecord(id, collection) {
-    let item = await PlacesSyncUtils.bookmarks.fetch(id);
-    if (!item) { // deleted item
-      let record = new PlacesItem(collection, id);
-      record.deleted = true;
-      return record;
-    }
-
-    let recordObj = getTypeObject(item.kind);
-    if (!recordObj) {
-      this._log.warn("Unknown item type, cannot serialize: " + item.kind);
-      recordObj = PlacesItem;
-    }
-    let record = new recordObj(collection, id);
-    record.fromSyncBookmark(item);
-
-    record.sortindex = await this._calculateIndex(record);
-
-    return record;
-  },
-
-
   async GUIDForId(id) {
     let guid = await PlacesUtils.promiseItemGuid(id);
     return PlacesSyncUtils.bookmarks.guidToRecordId(guid);
@@ -816,35 +1069,74 @@ BookmarksStore.prototype = {
     }
   },
 
-  async _calculateIndex(record) {
-    // Ensure folders have a very high sort index so they're not synced last.
-    if (record.type == "folder")
-      return FOLDER_SORTINDEX;
-
-    // For anything directly under the toolbar, give it a boost of more than an
-    // unvisited bookmark
-    let index = 0;
-    if (record.parentid == "toolbar")
-      index += 150;
-
-    // Add in the bookmark's frecency if we have something.
-    if (record.bmkUri != null) {
-      let frecency = await PlacesSyncUtils.history.fetchURLFrecency(record.bmkUri);
-      if (frecency != -1)
-        index += frecency;
-    }
-
-    return index;
-  },
-
   async wipe() {
     this.clearPendingDeletions();
-    // Save a backup before clearing out all bookmarks.
-    await PlacesBackups.create(null, true);
-    await PlacesSyncUtils.bookmarks.wipe();
+    await super.wipe();
   }
 };
 
+/**
+ * The buffered store delegates to the mirror for staging and applying
+ * records. Unlike `BookmarksStore`, `BufferedBookmarksStore` only
+ * implements `applyIncoming`, and `createRecord` via `BaseBookmarksStore`.
+ * These are the only two methods that `BufferedBookmarksEngine` calls during
+ * download and upload.
+ *
+ * The other `Store` methods intentionally remain abstract, so you can't use
+ * this store to create or update bookmarks in Places. All changes must go
+ * through the mirror, which takes care of merging and producing a valid tree.
+ */
+function BufferedBookmarksStore() {
+  BaseBookmarksStore.apply(this, arguments);
+}
+
+BufferedBookmarksStore.prototype = {
+  __proto__: BaseBookmarksStore.prototype,
+  _openMirrorPromise: null,
+
+  ensureOpenMirror() {
+    if (!this._openMirrorPromise) {
+      this._openMirrorPromise = this._openMirror().catch(err => {
+        // We may have failed to open the mirror temporarily; for example, if
+        // the database is locked. Clear the promise so that subsequent
+        // `ensureOpenMirror` calls can try to open the mirror again.
+        this._openMirrorPromise = null;
+        throw err;
+      });
+    }
+    return this._openMirrorPromise;
+  },
+
+  async _openMirror() {
+    let mirrorPath = OS.Path.join(OS.Constants.Path.profileDir, "weave",
+                                  "bookmarks.sqlite");
+    await OS.File.makeDir(OS.Path.dirname(mirrorPath), {
+      from: OS.Constants.Path.profileDir,
+    });
+
+    return SyncedBookmarksMirror.open({
+      path: mirrorPath,
+      recordTelemetryEvent: (object, method, value, extra) => {
+        this.engine.service.recordTelemetryEvent(object, method, value,
+                                                 extra);
+      },
+    });
+  },
+
+  async applyIncoming(record) {
+    let buf = await this.ensureOpenMirror();
+    await buf.store([record]);
+  },
+
+  async finalize() {
+    if (!this._openMirrorPromise) {
+      return;
+    }
+    let buf = await this._openMirrorPromise;
+    await buf.finalize();
+  },
+};
+
 // The bookmarks tracker is a special flower. Instead of listening for changes
 // via observer notifications, it queries Places for the set of items that have
 // changed since the last sync. Because it's a "pull-based" tracker, it ignores
@@ -853,7 +1145,6 @@ BookmarksStore.prototype = {
 function BookmarksTracker(name, engine) {
   this._batchDepth = 0;
   this._batchSawScoreIncrement = false;
-  this._migratedOldEntries = false;
   Tracker.call(this, name, engine);
 }
 BookmarksTracker.prototype = {
@@ -912,60 +1203,11 @@ BookmarksTracker.prototype = {
     throw new Error("Don't set initial changed bookmark IDs");
   },
 
-  // Migrates tracker entries from the old JSON-based tracker to Places. This
-  // is called the first time we start tracking changes.
-  async _migrateOldEntries() {
-    let existingIDs = await Utils.jsonLoad("changes/" + this.file, this);
-    if (existingIDs === null) {
-      // If the tracker file doesn't exist, we don't need to migrate, even if
-      // the engine is enabled. It's possible we're upgrading before the first
-      // sync. In the worst case, getting this wrong has the same effect as a
-      // restore: we'll reupload everything to the server.
-      this._log.debug("migrateOldEntries: Missing bookmarks tracker file; " +
-                      "skipping migration");
-      return null;
-    }
-
-    if (!this._needsMigration()) {
-      // We have a tracker file, but bookmark syncing is disabled, or this is
-      // the first sync. It's likely the tracker file is stale. Remove it and
-      // skip migration.
-      this._log.debug("migrateOldEntries: Bookmarks engine disabled or " +
-                      "first sync; skipping migration");
-      return Utils.jsonRemove("changes/" + this.file, this);
-    }
-
-    // At this point, we know the engine is enabled, we have a tracker file
-    // (though it may be empty), and we've synced before.
-    this._log.debug("migrateOldEntries: Migrating old tracker entries");
-    let entries = [];
-    for (let id in existingIDs) {
-      let change = existingIDs[id];
-      // Allow raw timestamps for backward-compatibility with changed IDs
-      // persisted before bug 1274496.
-      let timestamp = typeof change == "number" ? change : change.modified;
-      entries.push({
-        recordId: id,
-        modified: timestamp * 1000,
-      });
-    }
-    await PlacesSyncUtils.bookmarks.migrateOldTrackerEntries(entries);
-    return Utils.jsonRemove("changes/" + this.file, this);
-  },
-
-  _needsMigration() {
-    return this.engine && this.engineIsEnabled() && this.engine.lastSync > 0;
-  },
-
   observe: function observe(subject, topic, data) {
     Tracker.prototype.observe.call(this, subject, topic, data);
 
     switch (topic) {
       case "weave:engine:start-tracking":
-        if (!this._migratedOldEntries) {
-          this._migratedOldEntries = true;
-          Async.promiseSpinningly(this._migrateOldEntries());
-        }
         break;
       case "bookmarks-restore-begin":
         this._log.debug("Ignoring changes from importing bookmarks.");
@@ -1078,16 +1320,46 @@ BookmarksTracker.prototype = {
   onItemVisited() {}
 };
 
-class BookmarksChangeset extends Changeset {
-
-  getStatus(id) {
-    let change = this.changes[id];
-    if (!change) {
-      return PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN;
-    }
-    return change.status;
+/**
+ * A changeset that stores extra metadata in a change record for each ID. The
+ * engine updates this metadata when uploading Sync records, and writes it back
+ * to Places in `BaseBookmarksEngine#trackRemainingChanges`.
+ *
+ * The `synced` property on a change record means its corresponding item has
+ * been uploaded, and we should pretend it doesn't exist in the changeset.
+ */
+class BufferedBookmarksChangeset extends Changeset {
+  // Only `_reconcile` calls `getModifiedTimestamp` and `has`, and the buffered
+  // engine does its own reconciliation.
+  getModifiedTimestamp(id) {
+    throw new Error("Don't use timestamps to resolve bookmark conflicts");
   }
 
+  has(id) {
+    throw new Error("Don't use the changeset to resolve bookmark conflicts");
+  }
+
+  delete(id) {
+    let change = this.changes[id];
+    if (change) {
+      // Mark the change as synced without removing it from the set. We do this
+      // so that we can update Places in `trackRemainingChanges`.
+      change.synced = true;
+    }
+  }
+
+  ids() {
+    let results = new Set();
+    for (let id in this.changes) {
+      if (!this.changes[id].synced) {
+        results.add(id);
+      }
+    }
+    return [...results];
+  }
+}
+
+class BookmarksChangeset extends BufferedBookmarksChangeset {
   getModifiedTimestamp(id) {
     let change = this.changes[id];
     if (change) {
@@ -1113,25 +1385,6 @@ class BookmarksChangeset extends Changeset {
     }
   }
 
-  delete(id) {
-    let change = this.changes[id];
-    if (change) {
-      // Mark the change as synced without removing it from the set. We do this
-      // so that we can update Places in `trackRemainingChanges`.
-      change.synced = true;
-    }
-  }
-
-  ids() {
-    let results = new Set();
-    for (let id in this.changes) {
-      if (!this.changes[id].synced) {
-        results.add(id);
-      }
-    }
-    return [...results];
-  }
-
   isTombstone(id) {
     let change = this.changes[id];
     if (change) {
diff --git a/services/sync/modules/service.js b/services/sync/modules/service.js
index 456aa7b5321f..1fa69cf0928a 100644
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -39,7 +39,6 @@ Cu.import("resource://services-sync/util.js");
 function getEngineModules() {
   let result = {
     Addons: {module: "addons.js", symbol: "AddonsEngine"},
-    Bookmarks: {module: "bookmarks.js", symbol: "BookmarksEngine"},
     Form: {module: "forms.js", symbol: "FormEngine"},
     History: {module: "history.js", symbol: "HistoryEngine"},
     Password: {module: "passwords.js", symbol: "PasswordEngine"},
@@ -59,6 +58,17 @@ function getEngineModules() {
       symbol: "CreditCardsEngine",
     };
   }
+  if (Svc.Prefs.get("engine.bookmarks.buffer", false)) {
+    result.Bookmarks = {
+      module: "bookmarks.js",
+      symbol: "BufferedBookmarksEngine",
+    };
+  } else {
+    result.Bookmarks = {
+      module: "bookmarks.js",
+      symbol: "BookmarksEngine",
+    };
+  }
   return result;
 }
 
diff --git a/services/sync/services-sync.js b/services/sync/services-sync.js
index 7389412cf205..a13ba881eb02 100644
--- a/services/sync/services-sync.js
+++ b/services/sync/services-sync.js
@@ -25,6 +25,7 @@ pref("services.sync.errorhandler.networkFailureReportTimeout", 1209600); // 2 we
 pref("services.sync.engine.addons", true);
 pref("services.sync.engine.addresses", false);
 pref("services.sync.engine.bookmarks", true);
+pref("services.sync.engine.bookmarks.buffer", false);
 pref("services.sync.engine.creditcards", false);
 pref("services.sync.engine.history", true);
 pref("services.sync.engine.passwords", true);
diff --git a/services/sync/tests/unit/test_bookmark_engine.js b/services/sync/tests/unit/test_bookmark_engine.js
index 8621a8bfeefa..7b4ffd4aa82b 100644
--- a/services/sync/tests/unit/test_bookmark_engine.js
+++ b/services/sync/tests/unit/test_bookmark_engine.js
@@ -38,11 +38,31 @@ add_task(async function setup() {
   await Service.engineManager.unregister("bookmarks");
 });
 
-add_task(async function test_delete_invalid_roots_from_server() {
+function add_bookmark_test(task) {
+  add_task(async function() {
+    _(`Running test ${task.name} with legacy bookmarks engine`);
+    let legacyEngine = new BookmarksEngine(Service);
+    await legacyEngine.initialize();
+    try {
+      await task(legacyEngine);
+    } finally {
+      await legacyEngine.finalize();
+    }
+
+    _(`Running test ${task.name} with buffered bookmarks engine`);
+    let bufferedEngine = new BufferedBookmarksEngine(Service);
+    await bufferedEngine.initialize();
+    try {
+      await task(bufferedEngine);
+    } finally {
+      await bufferedEngine.finalize();
+    }
+  });
+}
+
+add_bookmark_test(async function test_delete_invalid_roots_from_server(engine) {
   _("Ensure that we delete the Places and Reading List roots from the server.");
 
-  let engine  = new BookmarksEngine(Service);
-  await engine.initialize();
   let store   = engine._store;
   let server = await serverForFoo(engine);
   await SyncTestingInfrastructure(server);
@@ -81,9 +101,13 @@ add_task(async function test_delete_invalid_roots_from_server() {
 
     await sync_engine_and_validate_telem(engine, false);
 
-    ok(!(await store.itemExists("readinglist")), "Should not apply Reading List root");
-    ok(!(await store.itemExists(listBmk.id)), "Should not apply items in Reading List");
-    ok((await store.itemExists(newBmk.id)), "Should apply new bookmark");
+    await Assert.rejects(PlacesUtils.promiseItemId("readinglist"),
+      /no item found for the given GUID/, "Should not apply Reading List root");
+    await Assert.rejects(PlacesUtils.promiseItemId(listBmk.id),
+      /no item found for the given GUID/,
+      "Should not apply items in Reading List");
+    ok((await PlacesUtils.promiseItemId(newBmk.id)) > 0,
+      "Should apply new bookmark");
 
     deepEqual(collection.keys().sort(), ["menu", "mobile", "toolbar", "unfiled", newBmk.id].sort(),
       "Should remove Places root and reading list items from server; upload local roots");
@@ -122,11 +146,9 @@ add_task(async function bad_record_allIDs() {
   await promiseStopServer(server);
 });
 
-add_task(async function test_processIncoming_error_orderChildren() {
+add_bookmark_test(async function test_processIncoming_error_orderChildren(engine) {
   _("Ensure that _orderChildren() is called even when _processIncoming() throws an error.");
 
-  let engine = new BookmarksEngine(Service);
-  await engine.initialize();
   let store  = engine._store;
   let server = await serverForFoo(engine);
   await SyncTestingInfrastructure(server);
@@ -152,6 +174,15 @@ add_task(async function test_processIncoming_error_orderChildren() {
       title: "Get Thunderbird!",
     });
 
+    let toolbar_record = await store.createRecord("toolbar");
+    collection.insert("toolbar", encryptPayload(toolbar_record.cleartext));
+
+    let bmk1_record = await store.createRecord(bmk1.guid);
+    collection.insert(bmk1.guid, encryptPayload(bmk1_record.cleartext));
+
+    let bmk2_record = await store.createRecord(bmk2.guid);
+    collection.insert(bmk2.guid, encryptPayload(bmk2_record.cleartext));
+
     // Create a server record for folder1 where we flip the order of
     // the children.
     let folder1_record = await store.createRecord(folder1.guid);
@@ -169,7 +200,7 @@ add_task(async function test_processIncoming_error_orderChildren() {
 
     // Make the 10 minutes old so it will only be synced in the toFetch phase.
     bogus_record.modified = Date.now() / 1000 - 60 * 10;
-    engine.lastSync = Date.now() / 1000 - 60;
+    await engine.setLastSync(Date.now() / 1000 - 60);
     engine.toFetch = new SerializableSet([BOGUS_GUID]);
 
     let error;
@@ -200,25 +231,23 @@ add_task(async function test_processIncoming_error_orderChildren() {
   }
 });
 
-add_task(async function test_restorePromptsReupload() {
-  await test_restoreOrImport(true);
+add_bookmark_test(async function test_restorePromptsReupload(engine) {
+  await test_restoreOrImport(engine, { replace: true });
 });
 
-add_task(async function test_importPromptsReupload() {
-  await test_restoreOrImport(false);
+add_bookmark_test(async function test_importPromptsReupload(engine) {
+  await test_restoreOrImport(engine, { replace: false });
 });
 
-// Test a JSON restore or HTML import. Use JSON if `aReplace` is `true`, or
+// Test a JSON restore or HTML import. Use JSON if `replace` is `true`, or
 // HTML otherwise.
-async function test_restoreOrImport(aReplace) {
-  let verb = aReplace ? "restore" : "import";
-  let verbing = aReplace ? "restoring" : "importing";
-  let bookmarkUtils = aReplace ? BookmarkJSONUtils : BookmarkHTMLUtils;
+async function test_restoreOrImport(engine, { replace }) {
+  let verb = replace ? "restore" : "import";
+  let verbing = replace ? "restoring" : "importing";
+  let bookmarkUtils = replace ? BookmarkJSONUtils : BookmarkHTMLUtils;
 
   _(`Ensure that ${verbing} from a backup will reupload all records.`);
 
-  let engine = new BookmarksEngine(Service);
-  await engine.initialize();
   let store  = engine._store;
   let server = await serverForFoo(engine);
   await SyncTestingInfrastructure(server);
@@ -278,10 +307,10 @@ async function test_restoreOrImport(aReplace) {
     Assert.equal(wbos[0], bmk2.guid);
 
     _(`Now ${verb} from a backup.`);
-    await bookmarkUtils.importFromFile(backupFilePath, aReplace);
+    await bookmarkUtils.importFromFile(backupFilePath, replace);
 
     let bookmarksCollection = server.user("foo").collection("bookmarks");
-    if (aReplace) {
+    if (replace) {
       _("Verify that we wiped the server.");
       Assert.ok(!bookmarksCollection);
     } else {
@@ -290,29 +319,30 @@ async function test_restoreOrImport(aReplace) {
     }
 
     _("Ensure we have the bookmarks we expect locally.");
-    let guids = await fetchAllRecordIds();
-    _("GUIDs: " + JSON.stringify([...guids]));
-    let bookmarkGuids = new Map();
+    let recordIds = await fetchAllRecordIds();
+    _("GUIDs: " + JSON.stringify([...recordIds]));
+
+    let bookmarkRecordIds = new Map();
     let count = 0;
-    for (let guid of guids) {
+    for (let recordId of recordIds) {
       count++;
       let info = await PlacesUtils.bookmarks.fetch(
-        PlacesSyncUtils.bookmarks.recordIdToGuid(guid));
+        PlacesSyncUtils.bookmarks.recordIdToGuid(recordId));
       // Only one bookmark, so _all_ should be Firefox!
       if (info.type == PlacesUtils.bookmarks.TYPE_BOOKMARK) {
-        _(`Found URI ${info.url.href} for GUID ${guid}`);
-        bookmarkGuids.set(info.url.href, guid);
+        _(`Found URI ${info.url.href} for record ID ${recordId}`);
+        bookmarkRecordIds.set(info.url.href, recordId);
       }
     }
-    Assert.ok(bookmarkGuids.has("http://getfirefox.com/"));
-    if (!aReplace) {
-      Assert.ok(bookmarkGuids.has("http://getthunderbird.com/"));
+    Assert.ok(bookmarkRecordIds.has("http://getfirefox.com/"));
+    if (!replace) {
+      Assert.ok(bookmarkRecordIds.has("http://getthunderbird.com/"));
     }
 
     _("Have the correct number of IDs locally, too.");
     let expectedResults = ["menu", "toolbar", "mobile", "unfiled", folder1.guid,
                            bmk1.guid];
-    if (!aReplace) {
+    if (!replace) {
       expectedResults.push("toolbar", folder1.guid, bmk2.guid);
     }
     Assert.equal(count, expectedResults.length);
@@ -343,18 +373,18 @@ async function test_restoreOrImport(aReplace) {
                        });
 
     let expectedFX = {
-      id: bookmarkGuids.get("http://getfirefox.com/"),
+      id: bookmarkRecordIds.get("http://getfirefox.com/"),
       bmkUri: "http://getfirefox.com/",
       title: "Get Firefox!"
     };
     let expectedTB = {
-      id: bookmarkGuids.get("http://getthunderbird.com/"),
+      id: bookmarkRecordIds.get("http://getthunderbird.com/"),
       bmkUri: "http://getthunderbird.com/",
       title: "Get Thunderbird!"
     };
 
     let expectedBookmarks;
-    if (aReplace) {
+    if (replace) {
       expectedBookmarks = [expectedFX];
     } else {
       expectedBookmarks = [expectedTB, expectedFX];
@@ -366,7 +396,7 @@ async function test_restoreOrImport(aReplace) {
     let expectedFolder1 = { title: "Folder 1" };
 
     let expectedFolders;
-    if (aReplace) {
+    if (replace) {
       expectedFolders = [expectedFolder1];
     } else {
       expectedFolders = [expectedFolder1, expectedFolder1];
@@ -503,7 +533,7 @@ add_task(async function test_bookmark_guidMap_fail() {
   let itemPayload = itemRecord.cleartext;
   coll.insert(item.guid, encryptPayload(itemPayload));
 
-  engine.lastSync = 1; // So we don't back up.
+  await engine.setLastSync(1); // So we don't back up.
 
   // Make building the GUID map fail.
 
@@ -597,11 +627,9 @@ add_task(async function test_bookmark_tag_but_no_uri() {
   await store.update(record);
 });
 
-add_task(async function test_misreconciled_root() {
+add_bookmark_test(async function test_misreconciled_root(engine) {
   _("Ensure that we don't reconcile an arbitrary record with a root.");
 
-  let engine = new BookmarksEngine(Service);
-  await engine.initialize();
   let store = engine._store;
   let server = await serverForFoo(engine);
   await SyncTestingInfrastructure(server);
@@ -618,10 +646,9 @@ add_task(async function test_misreconciled_root() {
     PlacesUtils.bookmarks.toolbarGuid);
   Assert.notEqual(-1, toolbarIDBefore);
 
-  let parentGUIDBefore = toolbarBefore.parentid;
-  let parentIDBefore = await PlacesUtils.promiseItemId(
-    PlacesSyncUtils.bookmarks.recordIdToGuid(parentGUIDBefore));
-  Assert.notEqual(-1, parentIDBefore);
+  let parentRecordIDBefore = toolbarBefore.parentid;
+  let parentGUIDBefore = PlacesSyncUtils.bookmarks.recordIdToGuid(parentRecordIDBefore);
+  let parentIDBefore = await PlacesUtils.promiseItemId(parentGUIDBefore);
   Assert.equal("string", typeof(parentGUIDBefore));
 
   _("Current parent: " + parentGUIDBefore + " (" + parentIDBefore + ").");
@@ -639,15 +666,16 @@ add_task(async function test_misreconciled_root() {
   let rec = new FakeRecord(BookmarkFolder, to_apply);
 
   _("Applying record.");
-  store.applyIncoming(rec);
+  store.applyIncomingBatch([rec]);
 
   // Ensure that afterwards, toolbar is still there.
   // As of 2012-12-05, this only passes because Places doesn't use "toolbar" as
   // the real GUID, instead using a generated one. Sync does the translation.
   let toolbarAfter = await store.createRecord("toolbar", "bookmarks");
-  let parentGUIDAfter = toolbarAfter.parentid;
-  let parentIDAfter = await PlacesUtils.promiseItemId(
-    PlacesSyncUtils.bookmarks.recordIdToGuid(parentGUIDAfter));
+  let parentRecordIDAfter = toolbarAfter.parentid;
+  let parentGUIDAfter = PlacesSyncUtils.bookmarks.recordIdToGuid(
+    parentRecordIDAfter);
+  let parentIDAfter = await PlacesUtils.promiseItemId(parentGUIDAfter);
   Assert.equal((await PlacesUtils.promiseItemGuid(toolbarIDBefore)),
     PlacesUtils.bookmarks.toolbarGuid);
   Assert.equal(parentGUIDBefore, parentGUIDAfter);
@@ -659,11 +687,9 @@ add_task(async function test_misreconciled_root() {
   await promiseStopServer(server);
 });
 
-add_task(async function test_sync_dateAdded() {
+add_bookmark_test(async function test_sync_dateAdded(engine) {
   await Service.recordManager.clearCache();
   await PlacesSyncUtils.bookmarks.reset();
-  let engine = new BookmarksEngine(Service);
-  await engine.initialize();
   let store  = engine._store;
   let server = await serverForFoo(engine);
   await SyncTestingInfrastructure(server);
@@ -672,7 +698,7 @@ add_task(async function test_sync_dateAdded() {
 
   // TODO: Avoid random orange (bug 1374599), this is only necessary
   // intermittently - reset the last sync date so that we'll get all bookmarks.
-  engine.lastSync = 1;
+  await engine.setLastSync(1);
 
   Svc.Obs.notify("weave:engine:start-tracking"); // We skip usual startup...
 
diff --git a/services/sync/tests/unit/test_bookmark_repair.js b/services/sync/tests/unit/test_bookmark_repair.js
index 0f19bb81aa22..d17f84b4c081 100644
--- a/services/sync/tests/unit/test_bookmark_repair.js
+++ b/services/sync/tests/unit/test_bookmark_repair.js
@@ -5,6 +5,7 @@
 // many mocks)
 Cu.import("resource://gre/modules/Log.jsm");
 Cu.import("resource://gre/modules/osfile.jsm");
+Cu.import("resource://gre/modules/PlacesSyncUtils.jsm");
 Cu.import("resource://services-sync/bookmark_repair.js");
 Cu.import("resource://services-sync/constants.js");
 Cu.import("resource://services-sync/doctor.js");
@@ -12,15 +13,9 @@ Cu.import("resource://services-sync/service.js");
 Cu.import("resource://services-sync/engines/clients.js");
 Cu.import("resource://services-sync/engines/bookmarks.js");
 
-const LAST_BOOKMARK_SYNC_PREFS = [
-  "bookmarks.lastSync",
-  "bookmarks.lastSyncLocal",
-];
-
 const BOOKMARK_REPAIR_STATE_PREFS = [
   "client.GUID",
   "doctor.lastRepairAdvance",
-  ...LAST_BOOKMARK_SYNC_PREFS,
   ...Object.values(BookmarkRepairRequestor.PREF).map(name =>
     `repairs.bookmarks.${name}`
   ),
@@ -31,6 +26,9 @@ let bookmarksEngine;
 var recordedEvents = [];
 
 add_task(async function setup() {
+  await Service.engineManager.unregister("bookmarks");
+  await Service.engineManager.register(BufferedBookmarksEngine);
+
   clientsEngine = Service.clientsEngine;
   clientsEngine.ignoreLastModifiedOnProcessCommands = true;
   bookmarksEngine = Service.engineManager.get("bookmarks");
@@ -43,7 +41,9 @@ add_task(async function setup() {
 });
 
 function checkRecordedEvents(expected, message) {
-  deepEqual(recordedEvents, expected, message);
+  // Ignore event telemetry from the merger.
+  let repairEvents = recordedEvents.filter(event => event.object != "mirror");
+  deepEqual(repairEvents, expected, message);
   // and clear the list so future checks are easier to write.
   recordedEvents = [];
 }
@@ -130,7 +130,16 @@ add_task(async function test_bookmark_repair_integration() {
     checkRecordedEvents([], "Should not start repair after first sync");
 
     _("Back up last sync timestamps for remote client");
-    let restoreRemoteLastBookmarkSync = backupPrefs(LAST_BOOKMARK_SYNC_PREFS);
+    let buf = await bookmarksEngine._store.ensureOpenMirror();
+    let metaRows = await buf.db.execute(`
+      SELECT key, value FROM meta`);
+    let metaInfos = [];
+    for (let row of metaRows) {
+      metaInfos.push({
+        key: row.getResultByName("key"),
+        value: row.getResultByName("value"),
+      });
+    }
 
     _(`Delete ${bookmarkInfo.guid} locally and on server`);
     // Now we will reach into the server and hard-delete the bookmark
@@ -140,9 +149,29 @@ add_task(async function test_bookmark_repair_integration() {
     await PlacesUtils.bookmarks.remove(bookmarkInfo.guid, {
       source: PlacesUtils.bookmarks.SOURCE_SYNC,
     });
-    deepEqual((await bookmarksEngine.pullNewChanges()), {},
+    deepEqual((await PlacesSyncUtils.bookmarks.pullChanges()), {},
       `Should not upload tombstone for ${bookmarkInfo.guid}`);
 
+    // Remove the bookmark from the buffer, too.
+    let itemRows = await buf.db.execute(`
+      SELECT guid, kind, title, urlId
+      FROM items
+      WHERE guid = :guid`,
+      { guid: bookmarkInfo.guid });
+    equal(itemRows.length, 1, `Bookmark ${
+      bookmarkInfo.guid} should exist in buffer`);
+    let bufInfos = [];
+    for (let row of itemRows) {
+      bufInfos.push({
+        guid: row.getResultByName("guid"),
+        kind: row.getResultByName("kind"),
+        title: row.getResultByName("title"),
+        urlId: row.getResultByName("urlId"),
+      });
+    }
+    await buf.db.execute(`DELETE FROM items WHERE guid = :guid`,
+                         { guid: bookmarkInfo.guid });
+
     // sync again - we should have a few problems...
     _("Sync again to trigger repair");
     validationPromise = promiseValidationDone([
@@ -204,7 +233,14 @@ add_task(async function test_bookmark_repair_integration() {
     // repair instead of the sync.
     bookmarkInfo.source = PlacesUtils.bookmarks.SOURCE_SYNC;
     await PlacesUtils.bookmarks.insert(bookmarkInfo);
-    restoreRemoteLastBookmarkSync();
+    await buf.db.execute(`
+      INSERT INTO items(guid, urlId, kind, title)
+      VALUES(:guid, :urlId, :kind, :title)`,
+      bufInfos);
+    await buf.db.execute(`
+      REPLACE INTO meta(key, value)
+      VALUES(:key, :value)`,
+      metaInfos);
 
     _("Sync as remote client");
     await Service.sync();
@@ -350,9 +386,14 @@ add_task(async function test_repair_client_missing() {
     await PlacesUtils.bookmarks.remove(bookmarkInfo.guid, {
       source: PlacesUtils.bookmarks.SOURCE_SYNC,
     });
-    // sanity check we aren't going to sync this removal.
-    do_check_empty((await bookmarksEngine.pullNewChanges()));
-    // sanity check that the bookmark is not there anymore
+    // Delete the bookmark from the buffer, too.
+    let buf = await bookmarksEngine._store.ensureOpenMirror();
+    await buf.db.execute(`DELETE FROM items WHERE guid = :guid`,
+                         { guid: bookmarkInfo.guid });
+
+    // Ensure we won't upload a tombstone for the removed bookmark.
+    Assert.deepEqual((await PlacesSyncUtils.bookmarks.pullChanges()), {});
+    // Ensure the bookmark no longer exists in Places.
     Assert.equal(null, await PlacesUtils.bookmarks.fetch(bookmarkInfo.guid));
 
     // sync again - we should have a few problems...
@@ -481,6 +522,7 @@ add_task(async function test_repair_server_deleted() {
     // Now we will reach into the server and create a tombstone for that bookmark
     // but with a last-modified in the past - this way our sync isn't going to
     // pick up the record.
+    _(`Adding server tombstone for ${bookmarkInfo.guid}`);
     server.insertWBO("foo", "bookmarks", new ServerWBO(bookmarkInfo.guid, encryptPayload({
       id: bookmarkInfo.guid,
       deleted: true,
diff --git a/services/sync/tests/unit/test_bookmark_repair_responder.js b/services/sync/tests/unit/test_bookmark_repair_responder.js
index a9c14629d23f..290b25364dcc 100644
--- a/services/sync/tests/unit/test_bookmark_repair_responder.js
+++ b/services/sync/tests/unit/test_bookmark_repair_responder.js
@@ -15,7 +15,9 @@ Svc.Prefs.set("engine.bookmarks.validation.enabled", false);
 var recordedEvents = [];
 
 function checkRecordedEvents(expected) {
-  deepEqual(recordedEvents, expected);
+  // Ignore event telemetry from the merger.
+  let repairEvents = recordedEvents.filter(event => event.object != "mirror");
+  deepEqual(repairEvents, expected);
   // and clear the list so future checks are easier to write.
   recordedEvents = [];
 }
diff --git a/services/sync/tests/unit/test_bookmark_tracker.js b/services/sync/tests/unit/test_bookmark_tracker.js
index b96a59df0117..5dca565235ae 100644
--- a/services/sync/tests/unit/test_bookmark_tracker.js
+++ b/services/sync/tests/unit/test_bookmark_tracker.js
@@ -444,7 +444,7 @@ add_task(async function test_onItemAdded() {
     let syncFolderID = PlacesUtils.bookmarks.createFolder(
       PlacesUtils.bookmarks.bookmarksMenuFolder, "Sync Folder",
       PlacesUtils.bookmarks.DEFAULT_INDEX);
-    let syncFolderGUID = await engine._store.GUIDForId(syncFolderID);
+    let syncFolderGUID = await PlacesUtils.promiseItemGuid(syncFolderID);
     await verifyTrackedItems(["menu", syncFolderGUID]);
     Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE);
 
@@ -456,7 +456,7 @@ add_task(async function test_onItemAdded() {
       CommonUtils.makeURI("https://example.org/sync"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Sync Bookmark");
-    let syncBmkGUID = await engine._store.GUIDForId(syncBmkID);
+    let syncBmkGUID = await PlacesUtils.promiseItemGuid(syncBmkID);
     await verifyTrackedItems([syncFolderGUID, syncBmkGUID]);
     Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE);
 
@@ -467,7 +467,7 @@ add_task(async function test_onItemAdded() {
     let syncSepID = PlacesUtils.bookmarks.insertSeparator(
       PlacesUtils.bookmarks.bookmarksMenuFolder,
       PlacesUtils.bookmarks.getItemIndex(syncFolderID));
-    let syncSepGUID = await engine._store.GUIDForId(syncSepID);
+    let syncSepGUID = await PlacesUtils.promiseItemGuid(syncSepID);
     await verifyTrackedItems(["menu", syncSepGUID]);
     Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE);
   } finally {
@@ -568,7 +568,7 @@ add_task(async function test_onItemChanged_itemDates() {
       CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Firefox!");
-    let fx_guid = await engine._store.GUIDForId(fx_id);
+    let fx_guid = await PlacesUtils.promiseItemGuid(fx_id);
     _(`Firefox GUID: ${fx_guid}`);
 
     await startTracking();
@@ -592,40 +592,6 @@ add_task(async function test_onItemChanged_itemDates() {
   }
 });
 
-add_task(async function test_onItemChanged_changeBookmarkURI() {
-  _("Changes to bookmark URIs should be tracked");
-
-  try {
-    await stopTracking();
-
-    _("Insert a bookmark");
-    let bm = await PlacesUtils.bookmarks.insert({
-      parentGuid: PlacesUtils.bookmarks.menuGuid,
-      url: "http://getfirefox.com",
-      title: "Get Firefox!"
-    });
-    _(`Firefox GUID: ${bm.guid}`);
-
-    _("Set a tracked annotation to make sure we only notify once");
-    let id = await PlacesUtils.promiseItemId(bm.guid);
-    PlacesUtils.annotations.setItemAnnotation(
-      id, PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO, "A test description", 0,
-      PlacesUtils.annotations.EXPIRE_NEVER);
-
-    await startTracking();
-
-    _("Change the bookmark's URI");
-    bm.url = "https://www.mozilla.org/firefox";
-    bm = await PlacesUtils.bookmarks.update(bm);
-
-    await verifyTrackedItems([bm.guid]);
-    Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2);
-  } finally {
-    _("Clean up.");
-    await cleanup();
-  }
-});
-
 add_task(async function test_onItemTagged() {
   _("Items tagged using the synchronous API should be tracked");
 
@@ -636,7 +602,7 @@ add_task(async function test_onItemTagged() {
     let folder = PlacesUtils.bookmarks.createFolder(
       PlacesUtils.bookmarks.bookmarksMenuFolder, "Parent",
       PlacesUtils.bookmarks.DEFAULT_INDEX);
-    let folderGUID = await engine._store.GUIDForId(folder);
+    let folderGUID = await PlacesUtils.promiseItemGuid(folder);
     _("Folder ID: " + folder);
     _("Folder GUID: " + folderGUID);
 
@@ -645,7 +611,7 @@ add_task(async function test_onItemTagged() {
     let b = PlacesUtils.bookmarks.insertBookmark(
       folder, uri,
       PlacesUtils.bookmarks.DEFAULT_INDEX, "Get Firefox!");
-    let bGUID = await engine._store.GUIDForId(b);
+    let bGUID = await PlacesUtils.promiseItemGuid(b);
     _("New item is " + b);
     _("GUID: " + bGUID);
 
@@ -674,12 +640,12 @@ add_task(async function test_onItemUntagged() {
     let fx1ID = PlacesUtils.bookmarks.insertBookmark(
       PlacesUtils.bookmarks.bookmarksMenuFolder, uri,
       PlacesUtils.bookmarks.DEFAULT_INDEX, "Get Firefox!");
-    let fx1GUID = await engine._store.GUIDForId(fx1ID);
+    let fx1GUID = await PlacesUtils.promiseItemGuid(fx1ID);
     // Different parent and title; same URL.
     let fx2ID = PlacesUtils.bookmarks.insertBookmark(
       PlacesUtils.bookmarks.toolbarFolder, uri,
       PlacesUtils.bookmarks.DEFAULT_INDEX, "Download Firefox");
-    let fx2GUID = await engine._store.GUIDForId(fx2ID);
+    let fx2GUID = await PlacesUtils.promiseItemGuid(fx2ID);
     PlacesUtils.tagging.tagURI(uri, ["foo"]);
 
     await startTracking();
@@ -809,7 +775,7 @@ add_task(async function test_onItemKeywordChanged() {
     let b = PlacesUtils.bookmarks.insertBookmark(
       folder, uri,
       PlacesUtils.bookmarks.DEFAULT_INDEX, "Get Firefox!");
-    let bGUID = await engine._store.GUIDForId(b);
+    let bGUID = await PlacesUtils.promiseItemGuid(b);
     _("New item is " + b);
     _("GUID: " + bGUID);
 
@@ -914,7 +880,7 @@ add_task(async function test_onItemPostDataChanged() {
       CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Firefox!");
-    let fx_guid = await engine._store.GUIDForId(fx_id);
+    let fx_guid = await PlacesUtils.promiseItemGuid(fx_id);
     _(`Firefox GUID: ${fx_guid}`);
 
     await startTracking();
@@ -943,7 +909,7 @@ add_task(async function test_onItemAnnoChanged() {
     let b = PlacesUtils.bookmarks.insertBookmark(
       folder, CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX, "Get Firefox!");
-    let bGUID = await engine._store.GUIDForId(b);
+    let bGUID = await PlacesUtils.promiseItemGuid(b);
     _("New item is " + b);
     _("GUID: " + bGUID);
 
@@ -977,7 +943,7 @@ add_task(async function test_onItemAdded_filtered_root() {
       PlacesUtils.bookmarks.placesRoot,
       "New root",
       PlacesUtils.bookmarks.DEFAULT_INDEX);
-    let rootGUID = await engine._store.GUIDForId(rootID);
+    let rootGUID = await PlacesUtils.promiseItemGuid(rootID);
     _(`New root GUID: ${rootGUID}`);
 
     _("Insert a bookmark underneath the new root");
@@ -986,7 +952,7 @@ add_task(async function test_onItemAdded_filtered_root() {
       CommonUtils.makeURI("http://getthunderbird.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Thunderbird!");
-    let untrackedBmkGUID = await engine._store.GUIDForId(untrackedBmkID);
+    let untrackedBmkGUID = await PlacesUtils.promiseItemGuid(untrackedBmkID);
     _(`New untracked bookmark GUID: ${untrackedBmkGUID}`);
 
     _("Insert a bookmark underneath the Places root");
@@ -994,7 +960,7 @@ add_task(async function test_onItemAdded_filtered_root() {
       PlacesUtils.bookmarks.placesRoot,
       CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX, "Get Firefox!");
-    let rootBmkGUID = await engine._store.GUIDForId(rootBmkID);
+    let rootBmkGUID = await PlacesUtils.promiseItemGuid(rootBmkID);
     _(`New Places root bookmark GUID: ${rootBmkGUID}`);
 
     _("New root and bookmark should be ignored");
@@ -1017,7 +983,7 @@ add_task(async function test_onItemDeleted_filtered_root() {
       PlacesUtils.bookmarks.placesRoot,
       CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX, "Get Firefox!");
-    let rootBmkGUID = await engine._store.GUIDForId(rootBmkID);
+    let rootBmkGUID = await PlacesUtils.promiseItemGuid(rootBmkID);
     _(`New Places root bookmark GUID: ${rootBmkGUID}`);
 
     await startTracking();
@@ -1170,14 +1136,14 @@ add_task(async function test_onItemMoved() {
       CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Firefox!");
-    let fx_guid = await engine._store.GUIDForId(fx_id);
+    let fx_guid = await PlacesUtils.promiseItemGuid(fx_id);
     _("Firefox GUID: " + fx_guid);
     let tb_id = PlacesUtils.bookmarks.insertBookmark(
       PlacesUtils.bookmarks.bookmarksMenuFolder,
       CommonUtils.makeURI("http://getthunderbird.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Thunderbird!");
-    let tb_guid = await engine._store.GUIDForId(tb_id);
+    let tb_guid = await PlacesUtils.promiseItemGuid(tb_id);
     _("Thunderbird GUID: " + tb_guid);
 
     await startTracking();
@@ -1306,21 +1272,21 @@ add_task(async function test_onItemDeleted_removeFolderTransaction() {
       PlacesUtils.bookmarks.bookmarksMenuFolder,
       "Test folder",
       PlacesUtils.bookmarks.DEFAULT_INDEX);
-    let folder_guid = await engine._store.GUIDForId(folder_id);
+    let folder_guid = await PlacesUtils.promiseItemGuid(folder_id);
     _(`Folder GUID: ${folder_guid}`);
     let fx_id = PlacesUtils.bookmarks.insertBookmark(
       folder_id,
       CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Firefox!");
-    let fx_guid = await engine._store.GUIDForId(fx_id);
+    let fx_guid = await PlacesUtils.promiseItemGuid(fx_id);
     _(`Firefox GUID: ${fx_guid}`);
     let tb_id = PlacesUtils.bookmarks.insertBookmark(
       folder_id,
       CommonUtils.makeURI("http://getthunderbird.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Thunderbird!");
-    let tb_guid = await engine._store.GUIDForId(tb_id);
+    let tb_guid = await PlacesUtils.promiseItemGuid(tb_id);
     _(`Thunderbird GUID: ${tb_guid}`);
 
     await startTracking();
@@ -1364,14 +1330,14 @@ add_task(async function test_treeMoved() {
       PlacesUtils.bookmarks.bookmarksMenuFolder,
       "First test folder",
       PlacesUtils.bookmarks.DEFAULT_INDEX);
-    let folder1_guid = await engine._store.GUIDForId(folder1_id);
+    let folder1_guid = await PlacesUtils.promiseItemGuid(folder1_id);
 
     // A second folder in the first.
     let folder2_id = PlacesUtils.bookmarks.createFolder(
       folder1_id,
       "Second test folder",
       PlacesUtils.bookmarks.DEFAULT_INDEX);
-    let folder2_guid = await engine._store.GUIDForId(folder2_id);
+    let folder2_guid = await PlacesUtils.promiseItemGuid(folder2_id);
 
     // Create a couple of bookmarks in the second folder.
     PlacesUtils.bookmarks.insertBookmark(
@@ -1413,7 +1379,7 @@ add_task(async function test_onItemDeleted() {
       CommonUtils.makeURI("http://getthunderbird.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Thunderbird!");
-    let tb_guid = await engine._store.GUIDForId(tb_id);
+    let tb_guid = await PlacesUtils.promiseItemGuid(tb_id);
 
     await startTracking();
 
@@ -1553,7 +1519,7 @@ add_task(async function test_onItemDeleted_removeFolderChildren() {
       CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Firefox!");
-    let fx_guid = await engine._store.GUIDForId(fx_id);
+    let fx_guid = await PlacesUtils.promiseItemGuid(fx_id);
     _(`Firefox GUID: ${fx_guid}`);
 
     let tb_id = PlacesUtils.bookmarks.insertBookmark(
@@ -1561,7 +1527,7 @@ add_task(async function test_onItemDeleted_removeFolderChildren() {
       CommonUtils.makeURI("http://getthunderbird.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Thunderbird!");
-    let tb_guid = await engine._store.GUIDForId(tb_id);
+    let tb_guid = await PlacesUtils.promiseItemGuid(tb_id);
     _(`Thunderbird GUID: ${tb_guid}`);
 
     let moz_id = PlacesUtils.bookmarks.insertBookmark(
@@ -1570,7 +1536,7 @@ add_task(async function test_onItemDeleted_removeFolderChildren() {
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Mozilla"
     );
-    let moz_guid = await engine._store.GUIDForId(moz_id);
+    let moz_guid = await PlacesUtils.promiseItemGuid(moz_id);
     _(`Mozilla GUID: ${moz_guid}`);
 
     await startTracking();
@@ -1595,14 +1561,14 @@ add_task(async function test_onItemDeleted_tree() {
       PlacesUtils.bookmarks.bookmarksMenuFolder,
       "First test folder",
       PlacesUtils.bookmarks.DEFAULT_INDEX);
-    let folder1_guid = await engine._store.GUIDForId(folder1_id);
+    let folder1_guid = await PlacesUtils.promiseItemGuid(folder1_id);
 
     // A second folder in the first.
     let folder2_id = PlacesUtils.bookmarks.createFolder(
       folder1_id,
       "Second test folder",
       PlacesUtils.bookmarks.DEFAULT_INDEX);
-    let folder2_guid = await engine._store.GUIDForId(folder2_id);
+    let folder2_guid = await PlacesUtils.promiseItemGuid(folder2_id);
 
     // Create a couple of bookmarks in the second folder.
     let fx_id = PlacesUtils.bookmarks.insertBookmark(
@@ -1610,13 +1576,13 @@ add_task(async function test_onItemDeleted_tree() {
       CommonUtils.makeURI("http://getfirefox.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Firefox!");
-    let fx_guid = await engine._store.GUIDForId(fx_id);
+    let fx_guid = await PlacesUtils.promiseItemGuid(fx_id);
     let tb_id = PlacesUtils.bookmarks.insertBookmark(
       folder2_id,
       CommonUtils.makeURI("http://getthunderbird.com"),
       PlacesUtils.bookmarks.DEFAULT_INDEX,
       "Get Thunderbird!");
-    let tb_guid = await engine._store.GUIDForId(tb_id);
+    let tb_guid = await PlacesUtils.promiseItemGuid(tb_id);
 
     await startTracking();
 
@@ -1630,170 +1596,3 @@ add_task(async function test_onItemDeleted_tree() {
     await cleanup();
   }
 });
-
-add_task(async function test_skip_migration() {
-  await insertBookmarksToMigrate();
-
-  let originalTombstones = await PlacesTestUtils.fetchSyncTombstones();
-  let originalFields = await PlacesTestUtils.fetchBookmarkSyncFields(
-    "0gtWTOgYcoJD", "0dbpnMdxKxfg", "r5ouWdPB3l28", "YK5Bdq5MIqL6");
-
-  let filePath = OS.Path.join(OS.Constants.Path.profileDir, "weave", "changes",
-    "bookmarks.json");
-
-  _("No tracker file");
-  {
-    await Utils.jsonRemove("changes/bookmarks", tracker);
-    ok(!(await OS.File.exists(filePath)), "Tracker file should not exist");
-
-    await tracker._migrateOldEntries();
-
-    let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
-      "0gtWTOgYcoJD", "0dbpnMdxKxfg", "r5ouWdPB3l28", "YK5Bdq5MIqL6");
-    deepEqual(fields, originalFields,
-      "Sync fields should not change if tracker file is missing");
-    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
-    deepEqual(tombstones, originalTombstones,
-      "Tombstones should not change if tracker file is missing");
-  }
-
-  _("Existing tracker file; engine disabled");
-  {
-    await Utils.jsonSave("changes/bookmarks", tracker, {});
-    ok(await OS.File.exists(filePath),
-      "Tracker file should exist before disabled engine migration");
-
-    engine.disabled = true;
-    await tracker._migrateOldEntries();
-    engine.disabled = false;
-
-    let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
-      "0gtWTOgYcoJD", "0dbpnMdxKxfg", "r5ouWdPB3l28", "YK5Bdq5MIqL6");
-    deepEqual(fields, originalFields,
-      "Sync fields should not change on disabled engine migration");
-    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
-    deepEqual(tombstones, originalTombstones,
-      "Tombstones should not change if tracker file is missing");
-
-    ok(!(await OS.File.exists(filePath)),
-      "Tracker file should be deleted after disabled engine migration");
-  }
-
-  _("Existing tracker file; first sync");
-  {
-    await Utils.jsonSave("changes/bookmarks", tracker, {});
-    ok(await OS.File.exists(filePath),
-      "Tracker file should exist before first sync migration");
-
-    engine.lastSync = 0;
-    await tracker._migrateOldEntries();
-
-    let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
-      "0gtWTOgYcoJD", "0dbpnMdxKxfg", "r5ouWdPB3l28", "YK5Bdq5MIqL6");
-    deepEqual(fields, originalFields,
-      "Sync fields should not change on first sync migration");
-    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
-    deepEqual(tombstones, originalTombstones,
-      "Tombstones should not change if tracker file is missing");
-
-    ok(!(await OS.File.exists(filePath)),
-      "Tracker file should be deleted after first sync migration");
-  }
-
-  await cleanup();
-});
-
-add_task(async function test_migrate_empty_tracker() {
-  _("Migration with empty tracker file");
-  await insertBookmarksToMigrate();
-
-  await Utils.jsonSave("changes/bookmarks", tracker, {});
-
-  engine.lastSync = Date.now() / 1000;
-  await tracker._migrateOldEntries();
-
-  let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
-    "0gtWTOgYcoJD", "0dbpnMdxKxfg", "r5ouWdPB3l28", "YK5Bdq5MIqL6");
-  for (let field of fields) {
-    equal(field.syncStatus, PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
-      `Sync status of migrated bookmark ${field.guid} should be NORMAL`);
-    strictEqual(field.syncChangeCounter, 0,
-      `Change counter of migrated bookmark ${field.guid} should be 0`);
-  }
-
-  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
-  deepEqual(tombstones, [], "Migration should delete old tombstones");
-
-  let filePath = OS.Path.join(OS.Constants.Path.profileDir, "weave", "changes",
-    "bookmarks.json");
-  ok(!(await OS.File.exists(filePath)),
-    "Tracker file should be deleted after empty tracker migration");
-
-  await cleanup();
-});
-
-add_task(async function test_migrate_existing_tracker() {
-  _("Migration with existing tracker entries");
-  await insertBookmarksToMigrate();
-
-  let mozBmk = await PlacesUtils.bookmarks.fetch("0gtWTOgYcoJD");
-  let fxBmk = await PlacesUtils.bookmarks.fetch("0dbpnMdxKxfg");
-  let mozChangeTime = Math.floor(mozBmk.lastModified / 1000) - 60;
-  let fxChangeTime = Math.floor(fxBmk.lastModified / 1000) + 60;
-  await Utils.jsonSave("changes/bookmarks", tracker, {
-    "0gtWTOgYcoJD": mozChangeTime,
-    "0dbpnMdxKxfg": {
-      modified: fxChangeTime,
-      deleted: false,
-    },
-    "3kdIPWHs9hHC": {
-      modified: 1479494951,
-      deleted: true,
-    },
-    "l7DlMy2lL1jL": 1479496460,
-  });
-
-  engine.lastSync = Date.now() / 1000;
-  await tracker._migrateOldEntries();
-
-  let changedFields = await PlacesTestUtils.fetchBookmarkSyncFields(
-    "0gtWTOgYcoJD", "0dbpnMdxKxfg");
-  for (let field of changedFields) {
-    if (field.guid == "0gtWTOgYcoJD") {
-      ok(field.lastModified.getTime(), mozBmk.lastModified.getTime(),
-        `Modified time for ${field.guid} should not be reset to older change time`);
-    } else if (field.guid == "0dbpnMdxKxfg") {
-      equal(field.lastModified.getTime(), fxChangeTime * 1000,
-        `Modified time for ${field.guid} should be updated to newer change time`);
-    }
-    equal(field.syncStatus, PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
-      `Sync status of migrated bookmark ${field.guid} should be NORMAL`);
-    ok(field.syncChangeCounter > 0,
-      `Change counter of migrated bookmark ${field.guid} should be > 0`);
-  }
-
-  let unchangedFields = await PlacesTestUtils.fetchBookmarkSyncFields(
-    "r5ouWdPB3l28", "YK5Bdq5MIqL6");
-  for (let field of unchangedFields) {
-    equal(field.syncStatus, PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
-      `Sync status of unchanged bookmark ${field.guid} should be NORMAL`);
-    strictEqual(field.syncChangeCounter, 0,
-      `Change counter of unchanged bookmark ${field.guid} should be 0`);
-  }
-
-  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
-  await deepEqual(tombstones, [{
-    guid: "3kdIPWHs9hHC",
-    dateRemoved: new Date(1479494951 * 1000),
-  }, {
-    guid: "l7DlMy2lL1jL",
-    dateRemoved: new Date(1479496460 * 1000),
-  }], "Should write tombstones for deleted tracked items");
-
-  let filePath = OS.Path.join(OS.Constants.Path.profileDir, "weave", "changes",
-    "bookmarks.json");
-  ok(!(await OS.File.exists(filePath)),
-    "Tracker file should be deleted after existing tracker migration");
-
-  await cleanup();
-});
diff --git a/services/sync/tests/unit/test_telemetry.js b/services/sync/tests/unit/test_telemetry.js
index d5720c18b81c..e63c84cfa64f 100644
--- a/services/sync/tests/unit/test_telemetry.js
+++ b/services/sync/tests/unit/test_telemetry.js
@@ -182,6 +182,7 @@ add_task(async function test_uploading() {
       title: "New Title",
     });
 
+    await store.wipe();
     await engine.resetClient();
 
     ping = await sync_engine_and_validate_telem(engine, false);
diff --git a/tools/lint/eslint/modules.json b/tools/lint/eslint/modules.json
index f1ebea027831..399254ab2518 100644
--- a/tools/lint/eslint/modules.json
+++ b/tools/lint/eslint/modules.json
@@ -19,7 +19,7 @@
   "bogus_element_type.jsm": [],
   "bookmark_repair.js": ["BookmarkRepairRequestor", "BookmarkRepairResponder"],
   "bookmark_validator.js": ["BookmarkValidator", "BookmarkProblemData"],
-  "bookmarks.js": ["BookmarksEngine", "PlacesItem", "Bookmark", "BookmarkFolder", "BookmarkQuery", "Livemark", "BookmarkSeparator"],
+  "bookmarks.js": ["BookmarksEngine", "PlacesItem", "Bookmark", "BookmarkFolder", "BookmarkQuery", "Livemark", "BookmarkSeparator", "BufferedBookmarksEngine"],
   "bookmarks.jsm": ["PlacesItem", "Bookmark", "Separator", "Livemark", "BookmarkFolder", "DumpBookmarks"],
   "BootstrapMonitor.jsm": ["monitor"],
   "browser-loader.js": ["BrowserLoader"],

From ef4b2d883d3ef1930725c28f0bd33752b471c1e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Fri, 26 Jan 2018 20:18:48 -0600
Subject: [PATCH 44/46] servo: Merge #19877 -  style: Look at the snapshots
 when invalidating due to stylesheet changes (from
 emilio:remove-sheet-oh-noes); r=bz

Otherwise removal of stylesheets may get out of sync with other DOM changes, and
we may fail to invalidate the style of the affected elements.

Bug: 1432850
Reviewed-by: bz
MozReview-Commit-ID: DrMTgLzQcnk
Source-Repo: https://github.com/servo/servo
Source-Revision: bc7dd644f6f50bd6eb3fba1547808775ee9d7620

--HG--
extra : subtree_source : https%3A//hg.mozilla.org/projects/converted-servo-linear
extra : subtree_revision : a906350d0b2590d4634b95ddccb3a8294ec1cbfe
---
 servo/components/layout_thread/lib.rs         |  4 +-
 servo/components/style/gecko/data.rs          |  3 +
 .../style/gecko/generated/bindings.rs         |  8 ++
 servo/components/style/gecko/snapshot.rs      |  3 +-
 .../invalidation/element/element_wrapper.rs   |  3 +-
 .../style/invalidation/stylesheets.rs         | 92 ++++++++++++++-----
 servo/components/style/stylesheet_set.rs      |  6 +-
 servo/components/style/stylist.rs             |  5 +-
 servo/ports/geckolib/glue.rs                  | 16 ++--
 9 files changed, 103 insertions(+), 37 deletions(-)

diff --git a/servo/components/layout_thread/lib.rs b/servo/components/layout_thread/lib.rs
index fcd790fe6ec1..66f328f4465b 100644
--- a/servo/components/layout_thread/lib.rs
+++ b/servo/components/layout_thread/lib.rs
@@ -1194,8 +1194,6 @@ impl LayoutThread {
                 debug!("Doc sheets changed, flushing author sheets too");
                 self.stylist.force_stylesheet_origins_dirty(Origin::Author.into());
             }
-
-            self.stylist.flush(&guards, Some(element));
         }
 
         if viewport_size_changed {
@@ -1246,6 +1244,8 @@ impl LayoutThread {
             debug!("Noting restyle for {:?}: {:?}", el, style_data);
         }
 
+        self.stylist.flush(&guards, Some(element), Some(&map));
+
         // Create a layout context for use throughout the following passes.
         let mut layout_context =
             self.build_layout_context(guards.clone(), true, &map);
diff --git a/servo/components/style/gecko/data.rs b/servo/components/style/gecko/data.rs
index d3f5e303f759..b0e404151578 100644
--- a/servo/components/style/gecko/data.rs
+++ b/servo/components/style/gecko/data.rs
@@ -15,6 +15,7 @@ use invalidation::media_queries::{MediaListKey, ToMediaListKey};
 use malloc_size_of::MallocSizeOfOps;
 use media_queries::{Device, MediaList};
 use properties::ComputedValues;
+use selector_parser::SnapshotMap;
 use servo_arc::Arc;
 use shared_lock::{Locked, StylesheetGuards, SharedRwLockReadGuard};
 use std::sync::atomic::{AtomicUsize, Ordering};
@@ -202,6 +203,7 @@ impl PerDocumentStyleDataImpl {
         &mut self,
         guard: &SharedRwLockReadGuard,
         document_element: Option<E>,
+        snapshots: Option<&SnapshotMap>,
     ) -> bool
     where
         E: TElement,
@@ -209,6 +211,7 @@ impl PerDocumentStyleDataImpl {
         self.stylist.flush(
             &StylesheetGuards::same(guard),
             document_element,
+            snapshots,
         )
     }
 
diff --git a/servo/components/style/gecko/generated/bindings.rs b/servo/components/style/gecko/generated/bindings.rs
index ff7248e166c0..81200ed8b532 100644
--- a/servo/components/style/gecko/generated/bindings.rs
+++ b/servo/components/style/gecko/generated/bindings.rs
@@ -2008,6 +2008,13 @@ extern "C" {
 extern "C" {
     pub fn Servo_Element_IsPrimaryStyleReusedViaRuleNode(element: RawGeckoElementBorrowed) -> bool;
 }
+extern "C" {
+    pub fn Servo_InvalidateStyleForDocStateChanges(
+        root: RawGeckoElementBorrowed,
+        sets: *const nsTArray<RawServoStyleSetBorrowed>,
+        aStatesChanged: u64,
+    );
+}
 extern "C" {
     pub fn Servo_StyleSheet_FromUTF8Bytes(
         loader: *mut Loader,
@@ -2112,6 +2119,7 @@ extern "C" {
     pub fn Servo_StyleSet_FlushStyleSheets(
         set: RawServoStyleSetBorrowed,
         doc_elem: RawGeckoElementBorrowedOrNull,
+        snapshots: *const ServoElementSnapshotTable,
     );
 }
 extern "C" {
diff --git a/servo/components/style/gecko/snapshot.rs b/servo/components/style/gecko/snapshot.rs
index 862f8ab545be..844c36368563 100644
--- a/servo/components/style/gecko/snapshot.rs
+++ b/servo/components/style/gecko/snapshot.rs
@@ -193,7 +193,8 @@ impl ElementSnapshot for GeckoElementSnapshot {
 
     #[inline]
     fn each_class<F>(&self, callback: F)
-        where F: FnMut(&Atom)
+    where
+        F: FnMut(&Atom)
     {
         if !self.has_any(Flags::MaybeClass) {
             return;
diff --git a/servo/components/style/invalidation/element/element_wrapper.rs b/servo/components/style/invalidation/element/element_wrapper.rs
index afd5bbcb2cbc..22ca94a6e147 100644
--- a/servo/components/style/invalidation/element/element_wrapper.rs
+++ b/servo/components/style/invalidation/element/element_wrapper.rs
@@ -71,7 +71,8 @@ pub struct ElementWrapper<'a, E>
 }
 
 impl<'a, E> ElementWrapper<'a, E>
-    where E: TElement,
+where
+    E: TElement,
 {
     /// Trivially constructs an `ElementWrapper`.
     pub fn new(el: E, snapshot_map: &'a SnapshotMap) -> Self {
diff --git a/servo/components/style/invalidation/stylesheets.rs b/servo/components/style/invalidation/stylesheets.rs
index 01cff601560c..532968781641 100644
--- a/servo/components/style/invalidation/stylesheets.rs
+++ b/servo/components/style/invalidation/stylesheets.rs
@@ -8,12 +8,14 @@
 #![deny(unsafe_code)]
 
 use Atom;
+use CaseSensitivityExt;
 use LocalName as SelectorLocalName;
 use dom::{TElement, TNode};
 use fnv::FnvHashSet;
+use invalidation::element::element_wrapper::{ElementSnapshot, ElementWrapper};
 use invalidation::element::restyle_hints::RestyleHint;
 use media_queries::Device;
-use selector_parser::SelectorImpl;
+use selector_parser::{SelectorImpl, Snapshot, SnapshotMap};
 use selectors::attr::CaseSensitivity;
 use selectors::parser::{Component, LocalName, Selector};
 use shared_lock::SharedRwLockReadGuard;
@@ -43,21 +45,40 @@ impl Invalidation {
         matches!(*self, Invalidation::ID(..) | Invalidation::Class(..))
     }
 
-    fn matches<E>(&self, element: E) -> bool
-        where E: TElement,
+    fn matches<E>(&self, element: E, snapshot: Option<&Snapshot>) -> bool
+    where
+        E: TElement,
     {
+        // FIXME This should look at the quirks mode of the document to
+        // determine case sensitivity.
+        //
+        // FIXME(emilio): Actually write a test and fix this.
+        let case_sensitivity = CaseSensitivity::CaseSensitive;
         match *self {
             Invalidation::Class(ref class) => {
-                // FIXME This should look at the quirks mode of the document to
-                // determine case sensitivity.
-                element.has_class(class, CaseSensitivity::CaseSensitive)
+                if element.has_class(class, case_sensitivity) {
+                    return true;
+                }
+
+                if let Some(snapshot) = snapshot {
+                    if snapshot.has_class(class, case_sensitivity) {
+                        return true;
+                    }
+                }
             }
             Invalidation::ID(ref id) => {
-                match element.get_id() {
-                    // FIXME This should look at the quirks mode of the document
-                    // to determine case sensitivity.
-                    Some(element_id) => element_id == *id,
-                    None => false,
+                if let Some(ref element_id) = element.get_id() {
+                    if case_sensitivity.eq_atom(element_id, id) {
+                        return true;
+                    }
+                }
+
+                if let Some(snapshot) = snapshot {
+                    if let Some(ref old_id) = snapshot.id_attr() {
+                        if case_sensitivity.eq_atom(old_id, id) {
+                            return true;
+                        }
+                    }
                 }
             }
             Invalidation::LocalName { ref name, ref lower_name } => {
@@ -65,9 +86,11 @@ impl Invalidation {
                 // of testing against both names, but it's probably not worth
                 // it.
                 let local_name = element.get_local_name();
-                *local_name == **name || *local_name == **lower_name
+                return *local_name == **name || *local_name == **lower_name
             }
         }
+
+        false
     }
 }
 
@@ -145,11 +168,17 @@ impl StylesheetInvalidationSet {
     /// `document_element` is provided.
     ///
     /// Returns true if any invalidations ocurred.
-    pub fn flush<E>(&mut self, document_element: Option<E>) -> bool
-        where E: TElement,
+    pub fn flush<E>(
+        &mut self,
+        document_element: Option<E>,
+        snapshots: Option<&SnapshotMap>,
+    ) -> bool
+    where
+        E: TElement,
     {
+        debug!("Stylist::flush({:?}, snapshots: {})", document_element, snapshots.is_some());
         let have_invalidations = match document_element {
-            Some(e) => self.process_invalidations(e),
+            Some(e) => self.process_invalidations(e, snapshots),
             None => false,
         };
         self.clear();
@@ -163,9 +192,17 @@ impl StylesheetInvalidationSet {
         self.fully_invalid = false;
     }
 
-    fn process_invalidations<E>(&self, element: E) -> bool
-        where E: TElement,
+    fn process_invalidations<E>(&self, element: E, snapshots: Option<&SnapshotMap>) -> bool
+    where
+        E: TElement,
     {
+        debug!(
+           "Stylist::process_invalidations({:?}, {:?}, {:?})",
+           element,
+           self.invalid_scopes,
+           self.invalid_elements,
+        );
+
         {
             let mut data = match element.mutate_data() {
                 Some(data) => data,
@@ -185,7 +222,7 @@ impl StylesheetInvalidationSet {
             return false;
         }
 
-        self.process_invalidations_in_subtree(element)
+        self.process_invalidations_in_subtree(element, snapshots)
     }
 
     /// Process style invalidations in a given subtree. This traverses the
@@ -194,9 +231,15 @@ impl StylesheetInvalidationSet {
     ///
     /// Returns whether it invalidated at least one element's style.
     #[allow(unsafe_code)]
-    fn process_invalidations_in_subtree<E>(&self, element: E) -> bool
-        where E: TElement,
+    fn process_invalidations_in_subtree<E>(
+        &self,
+        element: E,
+        snapshots: Option<&SnapshotMap>,
+    ) -> bool
+    where
+        E: TElement,
     {
+        debug!("process_invalidations_in_subtree({:?})", element);
         let mut data = match element.mutate_data() {
             Some(data) => data,
             None => return false,
@@ -212,8 +255,10 @@ impl StylesheetInvalidationSet {
             return false;
         }
 
+        let element_wrapper = snapshots.map(|s| ElementWrapper::new(element, s));
+        let snapshot = element_wrapper.as_ref().and_then(|e| e.snapshot());
         for invalidation in &self.invalid_scopes {
-            if invalidation.matches(element) {
+            if invalidation.matches(element, snapshot) {
                 debug!("process_invalidations_in_subtree: {:?} matched subtree {:?}",
                        element, invalidation);
                 data.hint.insert(RestyleHint::restyle_subtree());
@@ -225,7 +270,7 @@ impl StylesheetInvalidationSet {
 
         if !data.hint.contains(RestyleHint::RESTYLE_SELF) {
             for invalidation in &self.invalid_elements {
-                if invalidation.matches(element) {
+                if invalidation.matches(element, snapshot) {
                     debug!("process_invalidations_in_subtree: {:?} matched self {:?}",
                            element, invalidation);
                     data.hint.insert(RestyleHint::RESTYLE_SELF);
@@ -243,7 +288,8 @@ impl StylesheetInvalidationSet {
                 None => continue,
             };
 
-            any_children_invalid |= self.process_invalidations_in_subtree(child);
+            any_children_invalid |=
+                self.process_invalidations_in_subtree(child, snapshots);
         }
 
         if any_children_invalid {
diff --git a/servo/components/style/stylesheet_set.rs b/servo/components/style/stylesheet_set.rs
index c3cc0c7799de..b50c18cf14e7 100644
--- a/servo/components/style/stylesheet_set.rs
+++ b/servo/components/style/stylesheet_set.rs
@@ -7,6 +7,7 @@
 use dom::TElement;
 use invalidation::stylesheets::StylesheetInvalidationSet;
 use media_queries::Device;
+use selector_parser::SnapshotMap;
 use shared_lock::SharedRwLockReadGuard;
 use std::slice;
 use stylesheets::{Origin, OriginSet, OriginSetIterator, PerOrigin, StylesheetInDocument};
@@ -502,6 +503,7 @@ where
     pub fn flush<'a, E>(
         &'a mut self,
         document_element: Option<E>,
+        snapshots: Option<&SnapshotMap>,
     ) -> StylesheetFlusher<'a, S>
     where
         E: TElement,
@@ -510,7 +512,9 @@ where
 
         debug!("StylesheetSet::flush");
 
-        let had_invalidations = self.invalidations.flush(document_element);
+        let had_invalidations =
+            self.invalidations.flush(document_element, snapshots);
+
         let origins_dirty =
             mem::replace(&mut self.origins_dirty, OriginSet::empty());
 
diff --git a/servo/components/style/stylist.rs b/servo/components/style/stylist.rs
index 6ed2b36ac89d..4abe0aca207c 100644
--- a/servo/components/style/stylist.rs
+++ b/servo/components/style/stylist.rs
@@ -25,7 +25,7 @@ use properties::{AnimationRules, PropertyDeclarationBlock};
 use rule_cache::{RuleCache, RuleCacheConditions};
 use rule_tree::{CascadeLevel, RuleTree, StrongRuleNode, StyleSource};
 use selector_map::{PrecomputedHashMap, SelectorMap, SelectorMapEntry};
-use selector_parser::{SelectorImpl, PerPseudoElementMap, PseudoElement};
+use selector_parser::{SelectorImpl, SnapshotMap, PerPseudoElementMap, PseudoElement};
 use selectors::NthIndexCache;
 use selectors::attr::{CaseSensitivity, NamespaceConstraint};
 use selectors::bloom::{BloomFilter, NonCountingBloomFilter};
@@ -504,6 +504,7 @@ impl Stylist {
         &mut self,
         guards: &StylesheetGuards,
         document_element: Option<E>,
+        snapshots: Option<&SnapshotMap>,
     ) -> bool
     where
         E: TElement,
@@ -548,7 +549,7 @@ impl Stylist {
             }
         }
 
-        let flusher = self.stylesheets.flush(document_element);
+        let flusher = self.stylesheets.flush(document_element, snapshots);
 
         let had_invalidations = flusher.had_invalidations();
 
diff --git a/servo/ports/geckolib/glue.rs b/servo/ports/geckolib/glue.rs
index ca344ac922d1..701c594ef568 100644
--- a/servo/ports/geckolib/glue.rs
+++ b/servo/ports/geckolib/glue.rs
@@ -1219,21 +1219,23 @@ pub extern "C" fn Servo_StyleSet_RemoveStyleSheet(
 }
 
 #[no_mangle]
-pub extern "C" fn Servo_StyleSet_FlushStyleSheets(
+pub unsafe extern "C" fn Servo_StyleSet_FlushStyleSheets(
     raw_data: RawServoStyleSetBorrowed,
     doc_element: RawGeckoElementBorrowedOrNull,
+    snapshots: *const ServoElementSnapshotTable,
 ) {
     let global_style_data = &*GLOBAL_STYLE_DATA;
     let guard = global_style_data.shared_lock.read();
     let mut data = PerDocumentStyleData::from_ffi(raw_data).borrow_mut();
     let doc_element = doc_element.map(GeckoElement);
-    let have_invalidations = data.flush_stylesheets(&guard, doc_element);
+
+    let have_invalidations =
+        data.flush_stylesheets(&guard, doc_element, snapshots.as_ref());
+
     if have_invalidations && doc_element.is_some() {
-        // The invalidation machinery propagates the bits up, but we still
-        // need to tell the gecko restyle root machinery about it.
-        unsafe {
-            bindings::Gecko_NoteDirtySubtreeForInvalidation(doc_element.unwrap().0);
-        }
+        // The invalidation machinery propagates the bits up, but we still need
+        // to tell the Gecko restyle root machinery about it.
+        bindings::Gecko_NoteDirtySubtreeForInvalidation(doc_element.unwrap().0);
     }
 }
 

From 09b3922d79fd79214a482e765c325aea53485b18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Thu, 25 Jan 2018 02:53:49 +0100
Subject: [PATCH 45/46] Bug 1432850: Look at the snapshots when invalidating
 due to stylesheet changes. r=bz

The selectorText test happens to pass right now because well, we don't implement
the setter yet[1], but would fail if we implemented an specific invalidation in
the way I'd have done it yesterday.

[1]: https://bugzilla.mozilla.org/show_bug.cgi?id=37468

MozReview-Commit-ID: DrMTgLzQcnk
---
 layout/style/ServoBindingList.h               |  6 ++-
 layout/style/ServoStyleSet.cpp                |  6 ++-
 testing/web-platform/meta/MANIFEST.json       | 45 +++++++++++++++++++
 .../selectorText-dynamic-001.html             | 24 ++++++++++
 .../invalidation/sheet-going-away-001.html    | 26 +++++++++++
 .../sheet-going-away-002-ref.html             |  7 +++
 .../invalidation/sheet-going-away-002.html    | 23 ++++++++++
 7 files changed, 134 insertions(+), 3 deletions(-)
 create mode 100644 testing/web-platform/tests/css/selectors/invalidation/selectorText-dynamic-001.html
 create mode 100644 testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-001.html
 create mode 100644 testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-002-ref.html
 create mode 100644 testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-002.html

diff --git a/layout/style/ServoBindingList.h b/layout/style/ServoBindingList.h
index 4fa6637243d8..b2f56b3ea0a0 100644
--- a/layout/style/ServoBindingList.h
+++ b/layout/style/ServoBindingList.h
@@ -109,8 +109,10 @@ SERVO_BINDING_FUNC(Servo_StyleSet_InsertStyleSheetBefore, void,
                    RawServoStyleSetBorrowed set,
                    const mozilla::ServoStyleSheet* gecko_sheet,
                    const mozilla::ServoStyleSheet* before)
-SERVO_BINDING_FUNC(Servo_StyleSet_FlushStyleSheets, void, RawServoStyleSetBorrowed set,
-                   RawGeckoElementBorrowedOrNull doc_elem)
+SERVO_BINDING_FUNC(Servo_StyleSet_FlushStyleSheets, void,
+                   RawServoStyleSetBorrowed set,
+                   RawGeckoElementBorrowedOrNull doc_elem,
+                   const mozilla::ServoElementSnapshotTable* snapshots)
 SERVO_BINDING_FUNC(Servo_StyleSet_NoteStyleSheetsChanged, void,
                    RawServoStyleSetBorrowed set,
                    bool author_style_disabled,
diff --git a/layout/style/ServoStyleSet.cpp b/layout/style/ServoStyleSet.cpp
index fbfc304b9393..bc1ec9ed7380 100644
--- a/layout/style/ServoStyleSet.cpp
+++ b/layout/style/ServoStyleSet.cpp
@@ -1435,7 +1435,11 @@ ServoStyleSet::UpdateStylist()
     // since they are loaded and unloaded synchronously, and they don't have to
     // deal with dynamic content changes.
     Element* root = IsMaster() ? mDocument->GetRootElement() : nullptr;
-    Servo_StyleSet_FlushStyleSheets(mRawSet.get(), root);
+    const ServoElementSnapshotTable* snapshots = nullptr;
+    if (nsPresContext* pc = GetPresContext()) {
+      snapshots = &pc->RestyleManager()->AsServo()->Snapshots();
+    }
+    Servo_StyleSet_FlushStyleSheets(mRawSet.get(), root, snapshots);
   }
 
   if (MOZ_UNLIKELY(mStylistState & StylistState::XBLStyleSheetsDirty)) {
diff --git a/testing/web-platform/meta/MANIFEST.json b/testing/web-platform/meta/MANIFEST.json
index fca171be8e92..0ab7f3f57b45 100644
--- a/testing/web-platform/meta/MANIFEST.json
+++ b/testing/web-platform/meta/MANIFEST.json
@@ -158799,6 +158799,18 @@
      {}
     ]
    ],
+   "css/selectors/invalidation/sheet-going-away-002.html": [
+    [
+     "/css/selectors/invalidation/sheet-going-away-002.html",
+     [
+      [
+       "/css/selectors/invalidation/sheet-going-away-002-ref.html",
+       "=="
+      ]
+     ],
+     {}
+    ]
+   ],
    "css/selectors/of-type-selectors.xhtml": [
     [
      "/css/selectors/of-type-selectors.xhtml",
@@ -259875,6 +259887,11 @@
      {}
     ]
    ],
+   "css/selectors/invalidation/sheet-going-away-002-ref.html": [
+    [
+     {}
+    ]
+   ],
    "css/selectors/of-type-selectors-ref.xhtml": [
     [
      {}
@@ -310300,6 +310317,18 @@
      {}
     ]
    ],
+   "css/selectors/invalidation/selectorText-dynamic-001.html": [
+    [
+     "/css/selectors/invalidation/selectorText-dynamic-001.html",
+     {}
+    ]
+   ],
+   "css/selectors/invalidation/sheet-going-away-001.html": [
+    [
+     "/css/selectors/invalidation/sheet-going-away-001.html",
+     {}
+    ]
+   ],
    "css/selectors/missing-right-token.html": [
     [
      "/css/selectors/missing-right-token.html",
@@ -520947,6 +520976,22 @@
    "9593a7d2dddc79525edb801748a28b1a5a1837c7",
    "testharness"
   ],
+  "css/selectors/invalidation/selectorText-dynamic-001.html": [
+   "ab36d3d2b9f8e3b610f60a4b8859c95bf239eb43",
+   "testharness"
+  ],
+  "css/selectors/invalidation/sheet-going-away-001.html": [
+   "ac8bac5fe4663c70de2ee449fd0cf432d7a82eff",
+   "testharness"
+  ],
+  "css/selectors/invalidation/sheet-going-away-002-ref.html": [
+   "5616ec15bb322f49c4b28761df2cfb40fdafc226",
+   "support"
+  ],
+  "css/selectors/invalidation/sheet-going-away-002.html": [
+   "d23ac4f08cb0c70d06bfac4c4be121a8de8af48f",
+   "reftest"
+  ],
   "css/selectors/missing-right-token.html": [
    "d961e801f7df57161cd8c7b5a4b26ae24013c3e9",
    "testharness"
diff --git a/testing/web-platform/tests/css/selectors/invalidation/selectorText-dynamic-001.html b/testing/web-platform/tests/css/selectors/invalidation/selectorText-dynamic-001.html
new file mode 100644
index 000000000000..c705c2b3e85f
--- /dev/null
+++ b/testing/web-platform/tests/css/selectors/invalidation/selectorText-dynamic-001.html
@@ -0,0 +1,24 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>CSS Test: invalidation of class changes when the selector in a rule has changed</title>
+<link rel="author" title="Emilio Cobos Álvarez" href="mailto:emilio@crisal.io">
+<link rel="help" href="https://drafts.csswg.org/selectors-4/">
+<link rel="help" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1432850">
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<style>
+  body { background: green; }
+  .red { background: red; }
+</style>
+<body class="red">
+Should have a green background.
+<script>
+test(() => {
+  document.body.offsetTop;
+  assert_equals(getComputedStyle(document.body).backgroundColor, "rgb(255, 0, 0)");
+  document.body.className = "";
+  document.styleSheets[0].cssRules[1].selectorText = ".bar";
+  assert_equals(getComputedStyle(document.body).backgroundColor, "rgb(0, 128, 0)");
+}, "Style should be recomputed correctly when the selector it depends on changes");
+</script>
+</body>
diff --git a/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-001.html b/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-001.html
new file mode 100644
index 000000000000..7b704eca97ff
--- /dev/null
+++ b/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-001.html
@@ -0,0 +1,26 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>CSS Test: invalidation of class changes when the sheet the style depends on goes away</title>
+<link rel="author" title="Emilio Cobos Álvarez" href="mailto:emilio@crisal.io">
+<link rel="help" href="https://drafts.csswg.org/selectors-4/">
+<link rel="help" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1432850">
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<style>
+  body { background: green; }
+</style>
+<style id="style">
+  .red { background: red; }
+</style>
+<body class="red">
+Should have a green background.
+<script>
+test(() => {
+  document.body.offsetTop;
+  assert_equals(getComputedStyle(document.body).backgroundColor, "rgb(255, 0, 0)");
+  document.body.className = "";
+  style.remove();
+  assert_equals(getComputedStyle(document.body).backgroundColor, "rgb(0, 128, 0)");
+}, "Style should be recomputed correctly when the stylesheet it depends on goes away");
+</script>
+</body>
diff --git a/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-002-ref.html b/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-002-ref.html
new file mode 100644
index 000000000000..678416177367
--- /dev/null
+++ b/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-002-ref.html
@@ -0,0 +1,7 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>CSS Test Reference</title>
+<link rel="author" title="Emilio Cobos Álvarez" href="mailto:emilio@crisal.io">
+<p style="color: green">
+  Should be green.
+</p>
diff --git a/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-002.html b/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-002.html
new file mode 100644
index 000000000000..41778239518d
--- /dev/null
+++ b/testing/web-platform/tests/css/selectors/invalidation/sheet-going-away-002.html
@@ -0,0 +1,23 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>CSS Test: invalidation of class changes when the sheet the style depends on goes away</title>
+<link rel="author" title="Emilio Cobos Álvarez" href="mailto:emilio@crisal.io">
+<link rel="help" href="https://drafts.csswg.org/selectors-4/">
+<link rel="help" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1432850">
+<link rel="match" href="sheet-going-away-002-ref.html">
+<style>
+  p { color: green; }
+</style>
+<style id="style">
+  .red p { color: red; }
+</style>
+<body class="red">
+<p>
+  Should be green.
+</p>
+<script>
+document.body.offsetTop;
+document.body.className = "";
+style.remove();
+</script>
+</body>

From 0cde8e00c634709fdfd7c2b0e55ba457f7fa9fc1 Mon Sep 17 00:00:00 2001
From: Gerald Squelart <gsquelart@mozilla.com>
Date: Thu, 25 Jan 2018 16:21:23 +1100
Subject: [PATCH 46/46] Bug 1419508 - Return early from CreateMaskLayer if
 there is no visible data - r=mattwoodrow

MozReview-Commit-ID: 9jWGXA2eGvv

--HG--
extra : rebase_source : 1e14c7baa20cd3441de7fa4b7b9c4de574121a24
---
 layout/painting/FrameLayerBuilder.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/layout/painting/FrameLayerBuilder.cpp b/layout/painting/FrameLayerBuilder.cpp
index 3c5d04260f7e..48ec5db56251 100644
--- a/layout/painting/FrameLayerBuilder.cpp
+++ b/layout/painting/FrameLayerBuilder.cpp
@@ -6358,6 +6358,10 @@ ContainerState::CreateMaskLayer(Layer *aLayer,
   gfx::Rect boundingRect = CalculateBounds(newData.mRoundedClipRects,
                                            newData.mAppUnitsPerDevPixel);
   boundingRect.Scale(mParameters.mXScale, mParameters.mYScale);
+  if (boundingRect.IsEmpty()) {
+    // Return early if we know that there is effectively no visible data.
+    return nullptr;
+  }
 
   uint32_t maxSize = mManager->GetMaxTextureSize();
   NS_ASSERTION(maxSize > 0, "Invalid max texture size");